Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bugfix/redis cache backend #143

Merged
merged 6 commits into from
Sep 30, 2024
Merged

Bugfix/redis cache backend #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fb5adb0
[BUGFIX] to allow redis to work
weakbit Jan 4, 2024
15fcf80
Extended the readme to clearify caching mechanism
weakbit Jan 4, 2024
4e2b314
Merge branch 'master' into bugfix/redis-cache-backend
weakbit Jun 14, 2024
e39f3a5
Bugfix, after changing files, the request cache has to flush to get a…
weakbit Jun 14, 2024
3166b15
Code cleanup
weakbit Jun 14, 2024
434da66
Merge branch 'master' into bugfix/redis-cache-backend
weakbit Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 31 additions & 26 deletions Classes/Driver/AmazonS3Driver.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,6 @@ public function initialize()
$this->initializeBaseUrl()
->initializeSettings()
->initializeClient();
$this->resetRequestCache();
// Test connection if we are in the edit view of this storage
if (
$this->compatibilityService->isBackend()
Expand Down Expand Up @@ -1230,32 +1229,34 @@ protected function prefixExists(string $identifier): bool
* @param string $identifier
* @return array|null Returns an array with the meta info or "null"
*/
protected function getMetaInfo($identifier)
protected function getMetaInfo($identifier): ?array
{
$this->normalizeIdentifier($identifier);
$cacheIdentifier = md5($identifier);
if (!$this->metaInfoCache->has($cacheIdentifier)) {
try {
$metadata = $this->s3Client->headObject([
'Bucket' => $this->configuration['bucket'],
'Key' => $identifier
])->toArray();
$metaInfoDownloadAdapter = GeneralUtility::makeInstance(MetaInfoDownloadAdapter::class);
$metaInfo = $metaInfoDownloadAdapter->getMetaInfoFromResponse($this, $identifier, $metadata);
$this->metaInfoCache->set($cacheIdentifier, $metaInfo);
return $metaInfo;
} catch (\Exception $exc) {
// Ignore file not found errors
if (!$exc->getPrevious() || $exc->getPrevious()->getCode() !== 404) {
/** @var \TYPO3\CMS\Core\Log\Logger $logger */
$logger = GeneralUtility::makeInstance(LogManager::class)->getLogger(__CLASS__);
$logger->log(LogLevel::WARNING, $exc->getMessage(), $exc->getTrace());
}
$this->metaInfoCache->remove($cacheIdentifier);
return null;
$metaInfo = $this->metaInfoCache->has($cacheIdentifier) ? $this->metaInfoCache->get($cacheIdentifier) : false;
if ($metaInfo) {
return $metaInfo;
}

try {
$metadata = $this->s3Client->headObject([
'Bucket' => $this->configuration['bucket'],
'Key' => $identifier
])->toArray();
$metaInfoDownloadAdapter = GeneralUtility::makeInstance(MetaInfoDownloadAdapter::class);
$metaInfo = $metaInfoDownloadAdapter->getMetaInfoFromResponse($this, $identifier, $metadata);
$this->metaInfoCache->set($cacheIdentifier, $metaInfo);
return $metaInfo;
} catch (\Exception $exc) {
// Ignore file not found errors
if (!$exc->getPrevious() || $exc->getPrevious()->getCode() !== 404) {
/** @var \TYPO3\CMS\Core\Log\Logger $logger */
$logger = GeneralUtility::makeInstance(LogManager::class)->getLogger(__CLASS__);
$logger->log(LogLevel::WARNING, $exc->getMessage(), $exc->getTrace());
}
$this->metaInfoCache->remove($cacheIdentifier);
return null;
}
return $this->metaInfoCache->get($cacheIdentifier);
}

/**
Expand All @@ -1268,7 +1269,10 @@ protected function getCachedResponse($function, $parameter)
$cacheIdentifier = md5($function) . '-' . md5(serialize($parameter));

if ($this->requestCache->has($cacheIdentifier)) {
return $this->requestCache->get($cacheIdentifier);
$response = $this->requestCache->get($cacheIdentifier);
if (false !== $response) {
return $response;
}
}

$result = $this->s3Client->$function($parameter)->toArray();
Expand All @@ -1282,13 +1286,14 @@ protected function getCachedResponse($function, $parameter)
* @param $identifier
* @return void
*/
protected function flushMetaInfoCache($identifier)
protected function flushMetaInfoCache($identifier): void
{
$this->normalizeIdentifier($identifier);
$cacheIdentifier = md5($identifier);
if ($this->metaInfoCache->has($cacheIdentifier)) {
$this->metaInfoCache->flush($cacheIdentifier);
$this->metaInfoCache->remove($cacheIdentifier);
}
$this->requestCache->flush();
}

/**
Expand Down Expand Up @@ -1529,7 +1534,7 @@ protected function getListObjects($identifier, $overrideArgs = [])
$fileIdentifier = $content['Key'];
$this->normalizeIdentifier($fileIdentifier);
$cacheIdentifier = md5($fileIdentifier);
if (!$this->metaInfoCache->has($cacheIdentifier)) {
if (!$this->metaInfoCache->has($cacheIdentifier) || !$this->metaInfoCache->get($cacheIdentifier)) {
$this->metaInfoCache->set($cacheIdentifier, $metaInfoDownloadAdapter->getMetaInfoFromResponse($this, $fileIdentifier, $content));
}
}
Expand Down
47 changes: 28 additions & 19 deletions Classes/Service/MetaDataUpdateService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
use AUS\AusDriverAmazonS3\Driver\AmazonS3Driver;
use AUS\AusDriverAmazonS3\Index\Extractor;
use TYPO3\CMS\Core\Resource\AbstractFile;
use TYPO3\CMS\Core\Resource\Exception\InvalidUidException;
use TYPO3\CMS\Core\Resource\Index\MetaDataRepository;
use TYPO3\CMS\Core\Resource\ResourceFactory;
use TYPO3\CMS\Core\Resource\ResourceStorage;
Expand All @@ -29,36 +30,44 @@
*/
class MetaDataUpdateService implements SingletonInterface
{
/**
* @throws InvalidUidException
*/
public function updateMetadata(array $fileProperties): void
{
if ($fileProperties['type'] === AbstractFile::FILETYPE_IMAGE) {
$storage = $this->getStorage((int) $fileProperties['storage']);
if ($fileProperties['type'] !== AbstractFile::FILETYPE_IMAGE) {
return;
}

$storage = $this->getStorage((int)$fileProperties['storage']);

// only process on our driver type where data was missing
if ($storage->getDriverType() !== AmazonS3Driver::DRIVER_TYPE) {
return;
}
// only process on our driver type where data was missing
if ($storage->getDriverType() !== AmazonS3Driver::DRIVER_TYPE) {
return;
}

$file = $storage->getFile($fileProperties['identifier']);
$imageDimensions = $this->getExtractor()->getImageDimensionsOfRemoteFile($file);
$file = $storage->getFile($fileProperties['identifier']);
$imageDimensions = $this->getExtractor()->getImageDimensionsOfRemoteFile($file);

$metaDataRepository = $this->getMetaDataRepository();
$metaData = $metaDataRepository->findByFileUid($fileProperties['uid']);
$metaDataRepository = $this->getMetaDataRepository();
$metaData = $metaDataRepository->findByFileUid($fileProperties['uid']);

$create = count($metaData) === 0;
$metaData['width'] = $imageDimensions[0];
$metaData['height'] = $imageDimensions[1];
if ($create) {
$metaDataRepository->createMetaDataRecord($fileProperties['uid'], $metaData);
} else {
$metaDataRepository->update($fileProperties['uid'], $metaData);
}
$create = count($metaData) === 0;
$metaData['width'] = $imageDimensions[0];
$metaData['height'] = $imageDimensions[1];
if ($create) {
$metaDataRepository->createMetaDataRecord($fileProperties['uid'], $metaData);
} else {
/** @noinspection PhpInternalEntityUsedInspection */
$metaDataRepository->update($fileProperties['uid'], $metaData);
}
}

protected function getStorage(int $uid): ResourceStorage
{
return GeneralUtility::makeInstance(ResourceFactory::class)->getStorageObject($uid);
$resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
assert($resourceFactory instanceof ResourceFactory);
return $resourceFactory->getStorageObject($uid);
}

protected function getExtractor(): Extractor
Expand Down
24 changes: 24 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,30 @@ By default, these caches are transient. However, if you choose to configure a pe

Detailed instructions on how to customize these cache backends can be found in the [TYPO3 CachingFramework Configuration Guide](https://docs.typo3.org/m/typo3/reference-coreapi/main/en-us/ApiOverview/CachingFramework/Configuration/Index.html). Remember, thorough testing is essential when modifying cache backends.

Example with simple file backend; all changes through TYPO3
```php
[
'frontend' => \TYPO3\CMS\Core\Cache\Frontend\VariableFrontend::class,
'backend' => \TYPO3\CMS\Core\Cache\Backend\SimpleFileBackend::class,
'groups' => [
'pages'
],
];
```
Example with redis
```php
[
'frontend' => \TYPO3\CMS\Core\Cache\Frontend\VariableFrontend::class,
'backend' => \TYPO3\CMS\Core\Cache\Backend\RedisBackend::class,
'options' => [
'defaultLifetime' => 0, // infinite
'database' => 0,
'hostname' => 'redis',
'port' => 6379,
],
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no groups = pages ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The example shows a possibility to keep this cache also the page cache is cleared. That might make sense in many cases

];
```

## Extend Extension

### Initialize S3 Client
Expand Down
Loading