[FEATURE] Add "base folder" configuration setting

The base folder allows us to use the same bucket for multiple projects by forcing every project/TYPO3 instance into an own prefix: ``` mybucket/ + project1/ | + file23.jpg | + subfolder/ | | + subfile42.png | + _processed_/ | + ... + project2/ + file2.jpg + _processed_/ + ... ``` The prefix is not visible in the TYPO3 UI (file list, file information) and gets added transparently to S3 requests, and is removed from S3 responses as well. My initial version used a simpler approach by just overriding `getRootLevelFolder()` and `getParentFolderIdentifierOfIdentifier()`, but this led to inconsistencies in the UI, and the processing folder could not be moved into the base folder because of inconsistencies inside the TYPO3 API.
andersundsehr · Sep 6, 2024 · cdd622c · cdd622c
1 parent f16e2ba
commit cdd622c
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 16 deletions.
diff --git a/Classes/Driver/AmazonS3Driver.php b/Classes/Driver/AmazonS3Driver.php
@@ -80,6 +80,14 @@ class AmazonS3Driver extends AbstractHierarchicalFilesystemDriver implements Str
      */
     protected $baseUrl = '';
 
+    /**
+     * Folder that is used as root folder.
+     * Must be empty or have a trailing slash.
+     *
+     * @var string
+     */
+    protected $baseFolder = '';
+
     /**
      * Stream wrapper protocol: Will be set in the constructor
      *
@@ -219,6 +227,7 @@ public function processConfiguration()
     public function initialize()
     {
         $this->initializeBaseUrl()
+            ->initializeBaseFolder()
             ->initializeSettings()
             ->initializeClient();
         $this->resetRequestCache();
@@ -239,7 +248,7 @@ public function getPublicUrl($identifier)
     {
         $uriParts = GeneralUtility::trimExplode('/', ltrim($identifier, '/'), true);
         $uriParts = array_map('rawurlencode', $uriParts);
-        return $this->baseUrl . '/' . implode('/', $uriParts);
+        return $this->baseUrl . '/' . $this->addBaseFolder(implode('/', $uriParts));
     }
 
     /**
@@ -395,14 +404,14 @@ public function addFile($localFilePath, $targetFolderIdentifier, $newFileName =
             if (filesize($localFilePath) === 0) { // Multipart uploader would fail to upload empty files
                 $this->s3Client->upload(
                     $this->configuration['bucket'],
-                    $targetIdentifier,
+                    $this->addBaseFolder($targetIdentifier),
                     ''
                 );
             } else {
                 $multipartUploadAdapter = GeneralUtility::makeInstance(MultipartUploaderAdapter::class, $this->s3Client);
                 $multipartUploadAdapter->upload(
                     $localFilePath,
-                    $targetIdentifier,
+                    $this->addBaseFolder($targetIdentifier),
                     $this->configuration['bucket'],
                     $this->getCacheControl($targetIdentifier)
                 );
@@ -525,7 +534,7 @@ public function getFileForLocalProcessing($fileIdentifier, $writable = true)
         $temporaryPath = $this->getTemporaryPathForFile($fileIdentifier);
         $this->s3Client->getObject([
             'Bucket' => $this->configuration['bucket'],
-            'Key' => $fileIdentifier,
+            'Key' => $this->addBaseFolder($fileIdentifier),
             'SaveAs' => $temporaryPath,
         ]);
         if (!is_file($temporaryPath)) {
@@ -600,7 +609,7 @@ public function getFileContents($fileIdentifier)
     {
         $result = $this->s3Client->getObject([
             'Bucket' => $this->configuration['bucket'],
-            'Key' => $fileIdentifier
+            'Key' => $this->addBaseFolder($fileIdentifier)
         ]);
         return (string)$result['Body'];
     }
@@ -1084,6 +1093,19 @@ protected function initializeBaseUrl()
         return $this;
     }
 
+    /**
+     * Set the $baseFolder variable from configuration
+     */
+    protected function initializeBaseFolder(): self
+    {
+        $baseFolder = $this->configuration['baseFolder'] ?? '';
+        if ($baseFolder != '') {
+            $baseFolder = rtrim($baseFolder, '/') . '/';
+        }
+        $this->baseFolder = $baseFolder;
+        return $this;
+    }
+
     /**
      * initializeSettings
      *
@@ -1184,6 +1206,30 @@ protected function testConnection()
         }
     }
 
+    /**
+     * Prefix the given file/path identifier with the base folder.
+     * Used by internal functions that directly speak with S3Client
+     */
+    protected function addBaseFolder(string $identifier): string
+    {
+        if ($this->baseFolder) {
+            $identifier = str_replace('//', '/', $this->baseFolder . $identifier);
+        }
+        return $identifier;
+    }
+
+    /**
+     * Remove base folder prefix from a given S3 path.
+     * Used when returning information from S3.
+     */
+    protected function removeBaseFolder(string $identifier): string
+    {
+        if ($this->baseFolder) {
+            $identifier = substr($identifier, strlen($this->baseFolder));
+        }
+        return $identifier;
+    }
+
     /**
      * @return \TYPO3\CMS\Core\Messaging\FlashMessageQueue
      */
@@ -1238,7 +1284,7 @@ protected function getMetaInfo($identifier)
             try {
                 $metadata = $this->s3Client->headObject([
                     'Bucket' => $this->configuration['bucket'],
-                    'Key' => $identifier
+                    'Key' => $this->addBaseFolder($identifier)
                 ])->toArray();
                 $metaInfoDownloadAdapter = GeneralUtility::makeInstance(MetaInfoDownloadAdapter::class);
                 $metaInfo = $metaInfoDownloadAdapter->getMetaInfoFromResponse($this, $identifier, $metadata);
@@ -1325,7 +1371,7 @@ protected function getObjectPermissions($identifier)
                 try {
                     $response = $this->s3Client->getObjectAcl([
                         'Bucket' => $this->configuration['bucket'],
-                        'Key' => $identifier
+                        'Key' => $this->addBaseFolder($identifier)
                     ])->toArray();
 
                     // Until the SDK provides any useful information about folder permissions, we take full access for granted as long as one user with full access exists.
@@ -1363,10 +1409,10 @@ protected function getObjectPermissions($identifier)
      */
     protected function deleteObject(string $identifier): bool
     {
-        $this->s3Client->deleteObject(['Bucket' => $this->configuration['bucket'], 'Key' => $identifier]);
+        $this->s3Client->deleteObject(['Bucket' => $this->configuration['bucket'], 'Key' => $this->addBaseFolder($identifier)]);
         $this->flushMetaInfoCache($identifier);
         $this->resetRequestCache();
-        return !$this->s3Client->doesObjectExist($this->configuration['bucket'], $identifier);
+        return !$this->s3Client->doesObjectExist($this->configuration['bucket'], $this->addBaseFolder($identifier));
     }
 
     /**
@@ -1394,7 +1440,7 @@ protected function createObject($identifier, $body = '', $overrideArgs = [])
         $this->normalizeIdentifier($identifier);
         $args = [
             'Bucket' => $this->configuration['bucket'],
-            'Key' => $identifier,
+            'Key' => $this->addBaseFolder($identifier),
             'Body' => $body
         ];
         $this->s3Client->putObject(array_merge_recursive($args, $overrideArgs));
@@ -1453,7 +1499,7 @@ protected function getStreamWrapperPath($file)
             throw new \RuntimeException('Type "' . gettype($file) . '" is not supported.', 1325191178);
         }
         $this->normalizeIdentifier($identifier);
-        return $basePath . $identifier;
+        return $basePath . $this->addBaseFolder($identifier);
     }
 
     /**
@@ -1517,22 +1563,39 @@ protected function getListObjects($identifier, $overrideArgs = [])
     {
         $args = [
             'Bucket' => $this->configuration['bucket'] ?? '',
-            'Prefix' => $identifier,
+            'Prefix' => $this->addBaseFolder($identifier),
         ];
         $result = $this->getCachedResponse('listObjectsV2', array_merge_recursive($args, $overrideArgs));
         // Cache the given meta info
         $metaInfoDownloadAdapter = GeneralUtility::makeInstance(MetaInfoDownloadAdapter::class);
 
         // with many files we come to the recursion which lessens the home of a cache hit, so we do not create the cache here
         if (isset($result['Contents']) && is_array($result['Contents'])) {
-            foreach ($result['Contents'] as $content) {
-                $fileIdentifier = $content['Key'];
+            $baseFolderSelfKey = null;
+            foreach ($result['Contents'] as $key => &$content) {
+                $content['Key'] = $this->removeBaseFolder($content['Key']);
+                if ($content['Key'] === '') {
+                    $baseFolderSelfKey = $key;
+                    continue;
+                }
+                $fileIdentifier = $identifier . $content['Key'];
                 $this->normalizeIdentifier($fileIdentifier);
                 $cacheIdentifier = md5($fileIdentifier);
                 if (!$this->metaInfoCache->has($cacheIdentifier)) {
                     $this->metaInfoCache->set($cacheIdentifier, $metaInfoDownloadAdapter->getMetaInfoFromResponse($this, $fileIdentifier, $content));
                 }
             }
+            unset($content);
+            if ($baseFolderSelfKey !== null) {
+                unset($result['Contents'][$baseFolderSelfKey]);
+            }
+        }
+
+        if (isset($result['CommonPrefixes'])) {
+            foreach ($result['CommonPrefixes'] as &$prefix) {
+                $prefix['Prefix'] = $this->removeBaseFolder($prefix['Prefix']);
+            }
+            unset($prefix);
         }
 
         if (isset($overrideArgs['MaxKeys']) && $overrideArgs['MaxKeys'] <= 1000) {
@@ -1586,8 +1649,8 @@ protected function copyObject($identifier, $targetIdentifier)
     {
         $this->s3Client->copyObject([
             'Bucket' => $this->configuration['bucket'],
-            'CopySource' => $this->configuration['bucket'] . '/' . $identifier,
-            'Key' => $targetIdentifier,
+            'CopySource' => $this->configuration['bucket'] . '/' . $this->addBaseFolder($identifier),
+            'Key' => $this->addBaseFolder($targetIdentifier),
             'CacheControl' => $this->getCacheControl($targetIdentifier)
         ]);
         $this->flushMetaInfoCache($targetIdentifier);

diff --git a/Configuration/FlexForm/AmazonS3DriverFlexForm.xml b/Configuration/FlexForm/AmazonS3DriverFlexForm.xml
@@ -187,6 +187,16 @@
 					</config>
 				</TCEforms>
 			</publicBaseUrl>
+			<baseFolder>
+				<TCEforms>
+					<label>LLL:EXT:aus_driver_amazon_s3/Resources/Private/Language/locallang_flexform.xlf:driverConfiguration.baseFolder</label>
+					<description>LLL:EXT:aus_driver_amazon_s3/Resources/Private/Language/locallang_flexform.xlf:driverConfiguration.baseFolder-description</description>
+					<config>
+						<type>input</type>
+						<size>30</size>
+					</config>
+				</TCEforms>
+			</baseFolder>
 			<cacheHeaderDuration>
 				<TCEforms>
 					<label>LLL:EXT:aus_driver_amazon_s3/Resources/Private/Language/locallang_flexform.xlf:driverConfiguration.cacheHeaderDuration</label>

diff --git a/Resources/Private/Language/locallang_flexform.xlf b/Resources/Private/Language/locallang_flexform.xlf
@@ -33,6 +33,12 @@
 			<trans-unit id="driverConfiguration.publicBaseUrl-description">
 				<source>Without protocol - "Protocol" setting is used here.</source>
 			</trans-unit>
+			<trans-unit id="driverConfiguration.baseFolder">
+				<source>Base folder</source>
+			</trans-unit>
+			<trans-unit id="driverConfiguration.baseFolder-description">
+				<source>Root folder to use as base for all file operations. Other files and folders outside this base folder are not visible.</source>
+			</trans-unit>
 			<trans-unit id="driverConfiguration.cacheHeaderDuration">
 				<source>Cache header: max age (in seconds, optional)</source>
 			</trans-unit>