diff --git a/apps/dav/lib/Upload/ChunkingV2Plugin.php b/apps/dav/lib/Upload/ChunkingV2Plugin.php index 117bdb835d265..54d73d0169c70 100644 --- a/apps/dav/lib/Upload/ChunkingV2Plugin.php +++ b/apps/dav/lib/Upload/ChunkingV2Plugin.php @@ -41,26 +41,27 @@ use Sabre\HTTP\ResponseInterface; use Sabre\Uri; +/** + * Sabre server plugin that coordinates chunked WebDAV uploads and finalization. + * + * Uploads are written either directly to the target file or to a staging file + * in the upload folder, depending on the target storage backend. + */ class ChunkingV2Plugin extends ServerPlugin { - /** @var Server */ - private $server; - /** @var UploadFolder */ - private $uploadFolder; - /** @var ICache */ - private $cache; - + private Server $server; + private UploadFolder $uploadFolder; + private ICache $cache; private ?string $uploadId = null; private ?string $uploadPath = null; private const TEMP_TARGET = '.target'; + private const DESTINATION_HEADER = 'Destination'; public const CACHE_KEY = 'chunking-v2'; public const UPLOAD_TARGET_PATH = 'upload-target-path'; public const UPLOAD_TARGET_ID = 'upload-target-id'; public const UPLOAD_ID = 'upload-id'; - private const DESTINATION_HEADER = 'Destination'; - public function __construct(ICacheFactory $cacheFactory) { $this->cache = $cacheFactory->createDistributed(self::CACHE_KEY); } @@ -89,34 +90,118 @@ protected function beforeGet(RequestInterface $request) { } /** - * @param string $path - * @param bool $createIfNotExists - * @return FutureFile|UploadFile|ICollection|INode + * Resolve the file and storage/path tuple used for chunk writes. + * + * Direct writes are only used when the destination file already exists on the + * same storage as the upload folder. Otherwise chunks are written to the + * staging file in the upload folder and finalized with copy/move afterwards. + * + * FIXME: Verify 'file' return type; old code had probably overly broad `@return FutureFile|UploadFile|ICollection|INode` + * + * @return array{ + * file: File, + * storage: \OCP\Files\Storage\IStorage, + * storagePath: string, + * isDirect: bool + * } + */ + private function resolveChunkWriteTarget(string $path, bool $createIfNotExists = false): array { + if (!$this->uploadFolder instanceof UploadFolder) { + throw new \LogicException('Upload folder not initialized'); + } + + $directTarget = $this->tryResolveDirectChunkWriteTarget($path); + if ($directTarget !== null) { + return $directTarget; + } + + return $this->resolveStagingChunkWriteTarget($createIfNotExists); + } + + /** + * Try to resolve the destination as a direct chunk-write target. + * + * This only succeeds when the destination exists as a file and is backed by + * the same storage as the upload folder. + * + * @return array{ + * file: File, + * storage: \OCP\Files\Storage\IStorage, + * storagePath: string, + * isDirect: bool + * }|null */ - private function getUploadFile(string $path, bool $createIfNotExists = false) { + private function tryResolveDirectChunkWriteTarget(string $path): ?array { + $uploadStorage = $this->uploadFolder->getStorage(); + try { $actualFile = $this->server->tree->getNodeForPath($path); - // Only directly upload to the target file if it is on the same storage - // There may be further potential to optimize here by also uploading - // to other storages directly. This would require to also carefully pick - // the storage/path used in getStorage() - if ($actualFile instanceof File && $this->uploadFolder->getStorage()->getId() === $actualFile->getNode()->getStorage()->getId()) { - return $actualFile; - } } catch (NotFound $e) { - // If there is no target file we upload to the upload folder first + // No target file exists yet; fall back to staging. + return null; } - // Use file in the upload directory that will be copied or moved afterwards - if ($createIfNotExists) { + if (!$actualFile instanceof File) { + return null; + } + + $actualNode = $actualFile->getNode(); + // This is a conservative optimization: only bypass staging when the + // destination already lives on the same storage as the upload folder. + // Broader direct-write support may be possible, but only if this + // resolver is extended to return the correct storage/path pair for + // that backend. + if ($uploadStorage->getId() !== $actualNode->getStorage()->getId()) { + return null; + } + + return [ + 'file' => $actualFile, + 'storage' => $uploadStorage, + 'storagePath' => $actualFile->getInternalPath(), + 'isDirect' => true, + ]; + } + + /** + * Resolve the staging file used when direct chunk writes are not possible. + * + * The staging file lives in the upload folder and is copied or moved to the + * final destination after the chunked write has been completed. + * + * @return array{ + * file: File, + * storage: \OCP\Files\Storage\IStorage, + * storagePath: string, + * isDirect: bool + * } + */ + private function resolveStagingChunkWriteTarget(bool $createIfNotExists): array { + $uploadStorage = $this->uploadFolder->getStorage(); + + if ($createIfNotExists && !$this->uploadFolder->childExists(self::TEMP_TARGET)) { $this->uploadFolder->createFile(self::TEMP_TARGET); } + // Use file in the upload directory that will be copied or moved afterwards /** @var UploadFile $uploadFile */ $uploadFile = $this->uploadFolder->getChild(self::TEMP_TARGET); - return $uploadFile->getFile(); + $targetFile = $uploadFile->getFile(); + + return [ + 'file' => $targetFile, + 'storage' => $uploadStorage, + 'storagePath' => $targetFile->getInternalPath(), + 'isDirect' => false, + ]; } + /** + * Start a chunked upload after the upload collection is created. + * + * Resolves the upload target, starts the backend chunked-write session, + * and stores upload metadata in distributed cache for subsequent chunk requests. + */ public function afterMkcol(RequestInterface $request, ResponseInterface $response): bool { try { $this->prepareUpload($request->getPath()); @@ -125,9 +210,14 @@ public function afterMkcol(RequestInterface $request, ResponseInterface $respons return true; } - $this->uploadPath = $this->server->calculateUri($this->server->httpRequest->getHeader(self::DESTINATION_HEADER)); - $targetFile = $this->getUploadFile($this->uploadPath, true); - [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); + $headerDestination = $this->server->httpRequest->getHeader(self::DESTINATION_HEADER); + $this->uploadPath = $this->server->calculateUri($headerDestination); + + [ + 'file' => $targetFile, + 'storage' => $storage, + 'storagePath' => $storagePath, + ] = $this->resolveChunkWriteTarget($this->uploadPath, true); $this->uploadId = $storage->startChunkedWrite($storagePath); @@ -141,15 +231,31 @@ public function afterMkcol(RequestInterface $request, ResponseInterface $respons return true; } + /** + * Store one uploaded chunk in the active chunked-write session. + * + * For staged uploads, this also updates the temporary target file metadata so + * size and propagation stay consistent across chunk requests. + * + * @throws BadRequest + * @throws InsufficientStorage + */ public function beforePut(RequestInterface $request, ResponseInterface $response): bool { try { - $this->prepareUpload(dirname($request->getPath())); + if ($this->prepareUpload(dirname($request->getPath())) === null) { + return true; + } $this->checkPrerequisites(); } catch (StorageInvalidException|BadRequest|NotFound $e) { return true; } - [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); + [ + 'file' => $targetFile, + 'storage' => $storage, + 'storagePath' => $storagePath, + 'isDirect' => $isDirect, + ] = $this->resolveChunkWriteTarget($this->uploadPath); $chunkName = basename($request->getPath()); $partId = is_numeric($chunkName) ? (int)$chunkName : -1; @@ -157,16 +263,18 @@ public function beforePut(RequestInterface $request, ResponseInterface $response throw new BadRequest('Invalid chunk name, must be numeric between 1 and 10000'); } - $uploadFile = $this->getUploadFile($this->uploadPath); $tempTargetFile = null; - $additionalSize = (int)$request->getHeader('Content-Length'); - if ($this->uploadFolder->childExists(self::TEMP_TARGET) && $this->uploadPath) { + + // For staged uploads, track the temporary target size separately so we can + // reject requests that would exceed the destination's available free space. + if (!$isDirect && $this->uploadPath) { /** @var UploadFile $tempTargetFile */ $tempTargetFile = $this->uploadFolder->getChild(self::TEMP_TARGET); [$destinationDir, $destinationName] = Uri\split($this->uploadPath); /** @var Directory $destinationParent */ $destinationParent = $this->server->tree->getNodeForPath($destinationDir); + $free = $destinationParent->getNode()->getFreeSpace(); $newSize = $tempTargetFile->getSize() + $additionalSize; if ($free >= 0 && ($tempTargetFile->getSize() > $free || $newSize > $free)) { @@ -175,46 +283,78 @@ public function beforePut(RequestInterface $request, ResponseInterface $response } $stream = $request->getBodyAsStream(); - $storage->putChunkedWritePart($storagePath, $this->uploadId, (string)$partId, $stream, $additionalSize); + $storage->putChunkedWritePart( + $storagePath, + $this->uploadId, + (string)$partId, + $stream, + $additionalSize + ); + + $storage->getCache()->update( + $targetFile->getId(), + ['size' => $targetFile->getSize() + $additionalSize] + ); - $storage->getCache()->update($uploadFile->getId(), ['size' => $uploadFile->getSize() + $additionalSize]); if ($tempTargetFile) { - $storage->getPropagator()->propagateChange($tempTargetFile->getInternalPath(), time(), $additionalSize); + $storage->getPropagator()->propagateChange( + $tempTargetFile->getInternalPath(), + time(), + $additionalSize + ); } $response->setStatus(201); return false; } + /** + * Finalize a chunked upload when the upload collection is moved to its target. + * + * Completes the backend chunked write, applies file metadata, and removes the + * temporary upload folder when the source is still a future file. + */ public function beforeMove($sourcePath, $destination): bool { try { - $this->prepareUpload(dirname($sourcePath)); + if ($this->prepareUpload(dirname($sourcePath)) === null) { + return true; + } $this->checkPrerequisites(); } catch (StorageInvalidException|BadRequest|NotFound|PreconditionFailed $e) { return true; } - [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); - $targetFile = $this->getUploadFile($this->uploadPath); + ['file' => $targetFile, 'storage' => $storage] = $this->resolveChunkWriteTarget($this->uploadPath); [$destinationDir, $destinationName] = Uri\split($destination); /** @var Directory $destinationParent */ $destinationParent = $this->server->tree->getNodeForPath($destinationDir); $destinationExists = $destinationParent->childExists($destinationName); - // allow sync clients to send the modification and creation time along in a header $updateFileInfo = []; - if ($this->server->httpRequest->getHeader('X-OC-MTime') !== null) { - $updateFileInfo['mtime'] = $this->sanitizeMtime($this->server->httpRequest->getHeader('X-OC-MTime')); + + // allow sync clients to send the modification time along in a header + $headerMTime = $this->server->httpRequest->getHeader('X-OC-MTime'); + if ($headerMTime !== null) { + $updateFileInfo['mtime'] = $this->sanitizeMtime($headerMTime); $this->server->httpResponse->setHeader('X-OC-MTime', 'accepted'); } - if ($this->server->httpRequest->getHeader('X-OC-CTime') !== null) { - $updateFileInfo['creation_time'] = $this->sanitizeMtime($this->server->httpRequest->getHeader('X-OC-CTime')); + + // allow sync clients to send the creation time along in a header + $headerCTime = $this->server->httpRequest->getHeader('X-OC-CTime'); + if ($headerCTime !== null) { + $updateFileInfo['creation_time'] = $this->sanitizeMtime($headerCTime); $this->server->httpResponse->setHeader('X-OC-CTime', 'accepted'); } + $updateFileInfo['mimetype'] = \OCP\Server::get(IMimeTypeDetector::class)->detectPath($destinationName); - if ($storage->instanceOfStorage(ObjectStoreStorage::class) && $storage->getObjectStore() instanceof IObjectStoreMultiPartUpload) { + // For multipart object-store uploads, determine the final size from uploaded + // parts before completing the write so free-space checks reflect the real size. + if ( + $storage->instanceOfStorage(ObjectStoreStorage::class) + && $storage->getObjectStore() instanceof IObjectStoreMultiPartUpload + ) { /** @var ObjectStoreStorage $storage */ /** @var IObjectStoreMultiPartUpload $objectStore */ $objectStore = $storage->getObjectStore(); @@ -253,18 +393,26 @@ public function beforeMove($sourcePath, $destination): bool { public function beforeDelete(RequestInterface $request, ResponseInterface $response) { try { - $this->prepareUpload(dirname($request->getPath())); + if ($this->prepareUpload(dirname($request->getPath()))) { + return true; + } $this->checkPrerequisites(); } catch (StorageInvalidException|BadRequest|NotFound $e) { return true; } - [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); + ['storage' => $storage, 'storagePath' => $storagePath] = $this->resolveChunkWriteTarget($this->uploadPath); $storage->cancelChunkedWrite($storagePath, $this->uploadId); + return true; } /** + * Validate that the current request can use chunking v2. + * + * This checks cache availability, upload-folder capabilities, destination + * headers, and cached upload metadata from previous requests. + * * @throws BadRequest * @throws PreconditionFailed * @throws StorageInvalidException @@ -272,89 +420,123 @@ public function beforeDelete(RequestInterface $request, ResponseInterface $respo private function checkPrerequisites(bool $checkUploadMetadata = true): void { $distributedCacheConfig = \OCP\Server::get(IConfig::class)->getSystemValue('memcache.distributed', null); - if ($distributedCacheConfig === null || (!$this->cache instanceof Redis && !$this->cache instanceof Memcached)) { + if ( + $distributedCacheConfig === null + || (!$this->cache instanceof Redis && !$this->cache instanceof Memcached) + ) { throw new BadRequest('Skipping chunking v2 since no proper distributed cache is available'); } - if (!$this->uploadFolder instanceof UploadFolder || empty($this->server->httpRequest->getHeader(self::DESTINATION_HEADER))) { + + if ( + !$this->uploadFolder instanceof UploadFolder + || empty($this->server->httpRequest->getHeader(self::DESTINATION_HEADER)) + ) { throw new BadRequest('Skipping chunked file writing as the destination header was not passed'); } + if (!$this->uploadFolder->getStorage()->instanceOfStorage(IChunkedFileWrite::class)) { throw new StorageInvalidException('Storage does not support chunked file writing'); } - if ($this->uploadFolder->getStorage()->instanceOfStorage(ObjectStoreStorage::class) && !$this->uploadFolder->getStorage()->getObjectStore() instanceof IObjectStoreMultiPartUpload) { + + if ( + $this->uploadFolder->getStorage()->instanceOfStorage(ObjectStoreStorage::class) + && !$this->uploadFolder->getStorage()->getObjectStore() instanceof IObjectStoreMultiPartUpload + ) { throw new StorageInvalidException('Storage does not support multi part uploads'); } if ($checkUploadMetadata) { if ($this->uploadId === null || $this->uploadPath === null) { - throw new PreconditionFailed('Missing metadata for chunked upload. The distributed cache does not hold the information of previous requests.'); + throw new PreconditionFailed( + 'Missing metadata for chunked upload. The distributed cache does not hold the information of previous requests.' + ); } } } - /** - * @return array [IStorage, string] - */ - private function getUploadStorage(string $targetPath): array { - $storage = $this->uploadFolder->getStorage(); - $targetFile = $this->getUploadFile($targetPath); - return [$storage, $targetFile->getInternalPath()]; - } - protected function sanitizeMtime(string $mtimeFromRequest): int { if (!is_numeric($mtimeFromRequest)) { - throw new InvalidArgumentException('X-OC-MTime header must be an integer (unix timestamp).'); + throw new InvalidArgumentException('X-OC-MTime/CTime header must be an integer (unix timestamp).'); } return (int)$mtimeFromRequest; } /** + * Load the upload folder and cached metadata for the current upload request. + * + * Populates the upload folder as well as the cached upload ID and destination + * path needed by subsequent chunk, move, and delete requests. + * + * FIXME: make private? + * * @throws NotFound */ - public function prepareUpload($path): void { - $this->uploadFolder = $this->server->tree->getNodeForPath($path); + public function prepareUpload($path): ?UploadFolder { + $node = $this->server->tree->getNodeForPath($path); + if (!$node instanceof UploadFolder) { + return null; + } + + $this->uploadFolder = $node; $uploadMetadata = $this->cache->get($this->uploadFolder->getName()); $this->uploadId = $uploadMetadata[self::UPLOAD_ID] ?? null; $this->uploadPath = $uploadMetadata[self::UPLOAD_TARGET_PATH] ?? null; + + return $node; } + /** + * Complete the backend chunked write and materialize the final target file. + * + * When the upload was staged in the upload folder, the completed file is + * copied or moved to the requested destination after the backend write + * finishes. + */ private function completeChunkedWrite(string $targetAbsolutePath): void { - $uploadFile = $this->getUploadFile($this->uploadPath)->getNode(); - [$storage, $storagePath] = $this->getUploadStorage($this->uploadPath); + [ + 'file' => $targetFile, + 'storage' => $storage, + 'storagePath' => $storagePath, + 'isDirect' => $isDirect, + ] = $this->resolveChunkWriteTarget($this->uploadPath); + + $targetNode = $targetFile->getNode(); $rootFolder = \OCP\Server::get(IRootFolder::class); $exists = $rootFolder->nodeExists($targetAbsolutePath); - $uploadFile->lock(ILockingProvider::LOCK_SHARED); + $targetNode->lock(ILockingProvider::LOCK_SHARED); $this->emitPreHooks($targetAbsolutePath, $exists); try { - $uploadFile->changeLock(ILockingProvider::LOCK_EXCLUSIVE); + $targetNode->changeLock(ILockingProvider::LOCK_EXCLUSIVE); $storage->completeChunkedWrite($storagePath, $this->uploadId); - $uploadFile->changeLock(ILockingProvider::LOCK_SHARED); + $targetNode->changeLock(ILockingProvider::LOCK_SHARED); } catch (Exception $e) { - $uploadFile->unlock(ILockingProvider::LOCK_EXCLUSIVE); + $targetNode->unlock(ILockingProvider::LOCK_EXCLUSIVE); throw $e; } // If the file was not uploaded to the user storage directly we need to copy/move it try { - $uploadFileAbsolutePath = $uploadFile->getFileInfo()->getPath(); - if ($uploadFileAbsolutePath !== $targetAbsolutePath) { - $uploadFile = $rootFolder->get($uploadFile->getFileInfo()->getPath()); + if (!$isDirect) { + $stagingNode = $rootFolder->get($targetNode->getFileInfo()->getPath()); if ($exists) { - $uploadFile->copy($targetAbsolutePath); + $stagingNode->copy($targetAbsolutePath); } else { - $uploadFile->move($targetAbsolutePath); + $stagingNode->move($targetAbsolutePath); } } $this->emitPostHooks($targetAbsolutePath, $exists); } catch (Exception $e) { - $uploadFile->unlock(ILockingProvider::LOCK_SHARED); + $targetNode->unlock(ILockingProvider::LOCK_SHARED); throw $e; } } + /** + * Emit filesystem hooks before the completed upload is materialized. + */ private function emitPreHooks(string $target, bool $exists): void { $hookPath = $this->getHookPath($target); if (!$exists) { @@ -371,6 +553,9 @@ private function emitPreHooks(string $target, bool $exists): void { ]); } + /** + * Emit filesystem hooks after the completed upload has been materialized. + */ private function emitPostHooks(string $target, bool $exists): void { $hookPath = $this->getHookPath($target); if (!$exists) {