From 42605ca3ac39f791437568ff6ae71dfe5ed8a9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hling=2C=20Maximilian?= Date: Tue, 14 May 2024 14:56:22 +0200 Subject: [PATCH 1/8] Prioritize newer files when tagging unscanned files --- lib/Db/DbFileMapper.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Db/DbFileMapper.php b/lib/Db/DbFileMapper.php index f2b1863b..4aea9a76 100644 --- a/lib/Db/DbFileMapper.php +++ b/lib/Db/DbFileMapper.php @@ -36,6 +36,7 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit): array ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%'))) ->andWhere($qb->expr()->lte('f.size', $qb->createNamedParameter(VerdictService::MAX_FILE_SIZE))) ->andWhere($qb->expr()->like('f.path', $qb->createNamedParameter('files/%'))) + ->orderBy('f.fileid', 'DESC') ->setMaxResults($limit); $fileIds = []; From 98a41ffaf7b9b91b88b4fea871d5633c609754c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hling=2C=20Maximilian?= Date: Tue, 14 May 2024 15:31:40 +0200 Subject: [PATCH 2/8] Increase amount of files tagged per background job --- lib/BackgroundJobs/TagUnscannedJob.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/BackgroundJobs/TagUnscannedJob.php b/lib/BackgroundJobs/TagUnscannedJob.php index 6eff4ae9..4ffd7694 100644 --- a/lib/BackgroundJobs/TagUnscannedJob.php +++ b/lib/BackgroundJobs/TagUnscannedJob.php @@ -47,7 +47,7 @@ protected function run($argument): void $excludedTagIds = [$unscannedTag->getId(), $maliciousTag->getId(), $cleanTag->getId(), $pupTag->getId()]; - $fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 1000); + $fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 10000); if (count($fileIds) == 0) { return; From 7e522ca2069a095c502cdc251c47367c16de9059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hling=2C=20Maximilian?= Date: Tue, 14 May 2024 15:37:31 +0200 Subject: [PATCH 3/8] Add yet another docker compose file (more is better) --- docker-compose.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..fb00daeb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,48 @@ +version: "3.8" +services: + db: + image: mariadb + volumes: + - db_data:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: unsafe + MYSQL_USER: mysql + MYSQL_PASSWORD: unsafe + MYSQL_DATABASE: nextcloud + restart: unless-stopped + + redis: + image: redis:alpine + restart: unless-stopped + + nextcloud: + image: nextcloud:stable + volumes: + - nextcloud_data:/var/www/html + environment: + MYSQL_USER: mysql + MYSQL_PASSWORD: unsafe + MYSQL_DATABASE: nextcloud + MYSQL_HOST: db + REDIS_HOST: redis + ports: + - "80:80" + - "443:443" + restart: unless-stopped + depends_on: + - db + - redis + + cron: + image: nextcloud:stable + volumes: + - nextcloud_data:/var/www/html + entrypoint: /cron.sh + restart: unless-stopped + depends_on: + - db + - redis + +volumes: + db_data: {} + nextcloud_data: {} From d170341b7b2ecb0b23844a3229d20a19c38f878e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hling=2C=20Maximilian?= Date: Tue, 14 May 2024 16:11:11 +0200 Subject: [PATCH 4/8] Add some more helpful commands to README --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 924c4253..19372532 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,9 @@ folder "nextcloud-docker-dev" and running ```docker compose up nextcloud proxy`` ### Useful commands -To trigger cronjobs manually you can use the following command: -```bash -docker exec --user www-data {nextcloud_container} php /var/www/html/cron.php -``` +| Description | Command | +|---------------------------|--------------------------------------------------------------------------------------| +| Trigger cronjobs manually | `docker exec --user www-data {nextcloud_container} php /var/www/html/cron.php` | +| Upgrade Nextcloud via CLI | `docker exec --user www-data {nextcloud_container} php occ upgrade` | +| Watch logs | `docker exec --user www-data {nextcloud_container} php occ log:watch` | +| Set log level to debug | `docker exec --user www-data {nextcloud_container} php occ log:manage --level DEBUG` | From ab09b5893ad4701431222eb4935049f96b4cf1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hling=2C=20Maximilian?= Date: Tue, 14 May 2024 16:16:08 +0200 Subject: [PATCH 5/8] Reduce interval of unscanned tag scanner --- lib/BackgroundJobs/TagUnscannedJob.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/BackgroundJobs/TagUnscannedJob.php b/lib/BackgroundJobs/TagUnscannedJob.php index 4ffd7694..42310139 100644 --- a/lib/BackgroundJobs/TagUnscannedJob.php +++ b/lib/BackgroundJobs/TagUnscannedJob.php @@ -22,7 +22,7 @@ public function __construct(ITimeFactory $time, IConfig $appConfig, TagService $ $this->appConfig = $appConfig; $this->tagService = $tagService; - $this->setInterval(5 * 60); + $this->setInterval(60); $this->setAllowParallelRuns(false); $this->setTimeSensitivity(self::TIME_SENSITIVE); } From 969ee0d09925c7830ccc19db7e6328ae69e4c787 Mon Sep 17 00:00:00 2001 From: PT-ATA No One Date: Tue, 21 May 2024 11:41:00 +0200 Subject: [PATCH 6/8] Add logging for start and end of background jobs --- lib/BackgroundJobs/ScanJob.php | 6 ++++-- lib/BackgroundJobs/TagUnscannedJob.php | 13 ++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/BackgroundJobs/ScanJob.php b/lib/BackgroundJobs/ScanJob.php index c1a31747..08df5d7a 100644 --- a/lib/BackgroundJobs/ScanJob.php +++ b/lib/BackgroundJobs/ScanJob.php @@ -28,7 +28,7 @@ public function __construct(LoggerInterface $logger, ITimeFactory $time, TagServ $this->scanService = $scanService; $this->appConfig = $appConfig; - $this->setInterval(5 * 60); + $this->setInterval(60); $this->setAllowParallelRuns(false); $this->setTimeSensitivity(self::TIME_SENSITIVE); } @@ -74,7 +74,7 @@ protected function run($argument): void } } - $this->logger->debug("Scanning " . count($fileIds) . " files: " . implode(", ", $fileIds)); + $this->logger->debug("Scanning files"); foreach ($fileIds as $fileId) { try { @@ -83,5 +83,7 @@ protected function run($argument): void $this->logger->error("Failed to scan file with id " . $fileId . ": " . $e->getMessage()); } } + + $this->logger->debug("Scanned " . count($fileIds) . " files"); } } diff --git a/lib/BackgroundJobs/TagUnscannedJob.php b/lib/BackgroundJobs/TagUnscannedJob.php index 42310139..7add2239 100644 --- a/lib/BackgroundJobs/TagUnscannedJob.php +++ b/lib/BackgroundJobs/TagUnscannedJob.php @@ -7,6 +7,7 @@ use OCP\AppFramework\Utility\ITimeFactory; use OCP\DB\Exception; use OCP\IConfig; +use Psr\Log\LoggerInterface; class TagUnscannedJob extends TimedJob { @@ -14,13 +15,15 @@ class TagUnscannedJob extends TimedJob private TagService $tagService; private IConfig $appConfig; + private LoggerInterface $logger; - public function __construct(ITimeFactory $time, IConfig $appConfig, TagService $tagService) + public function __construct(ITimeFactory $time, IConfig $appConfig, TagService $tagService, LoggerInterface $logger) { parent::__construct($time); $this->appConfig = $appConfig; $this->tagService = $tagService; + $this->logger = $logger; $this->setInterval(60); $this->setAllowParallelRuns(false); @@ -40,6 +43,8 @@ protected function run($argument): void return; } + $this->logger->debug("Tagging unscanned files"); + $unscannedTag = $this->tagService->getTag(TagService::UNSCANNED); $maliciousTag = $this->tagService->getTag(TagService::MALICIOUS); $pupTag = $this->tagService->getTag(TagService::PUP); @@ -49,15 +54,13 @@ protected function run($argument): void $fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 10000); - if (count($fileIds) == 0) { - return; - } - foreach ($fileIds as $fileId) { if ($this->tagService->hasCleanMaliciousOrPupTag($fileId)) { continue; } $this->tagService->setTag($fileId, TagService::UNSCANNED); } + + $this->logger->debug("Tagged " . count($fileIds) . " unscanned files"); } } From 172bad449e98266008adb1b37437a16a1dc3f807 Mon Sep 17 00:00:00 2001 From: PT-ATA No One Date: Tue, 21 May 2024 14:31:24 +0200 Subject: [PATCH 7/8] Prioritize newer files during background scanning --- lib/Db/DbFileMapper.php | 31 +++++++++++++++++++++++++++++++ lib/Service/TagService.php | 30 +++++++++++++++--------------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/lib/Db/DbFileMapper.php b/lib/Db/DbFileMapper.php index 4aea9a76..dec809a7 100644 --- a/lib/Db/DbFileMapper.php +++ b/lib/Db/DbFileMapper.php @@ -47,6 +47,37 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit): array return $fileIds; } + /** + * Get file ids that have at least one of the given tags + * @param array $includedTagIds + * @param int $limit + * @return array of file ids + * @throws Exception if the database platform is not supported + */ + public function getFileIdsWithTags(array $includedTagIds, int $limit): array + { + $qb = $this->db->getQueryBuilder(); + $qb->automaticTablePrefix(true); + + $qb->select('f.fileid') + ->from($this->getTableName(), 'f') + ->leftJoin('f', 'systemtag_object_mapping', 'o', $qb->expr()->eq('f.fileid', $qb->createFunction($this->getPlatformSpecificCast()))) + ->leftJoin('f', 'mimetypes', 'm', $qb->expr()->eq('f.mimetype', 'm.id')) + ->where($qb->expr()->in('o.systemtagid', $qb->createNamedParameter($includedTagIds, IQueryBuilder::PARAM_INT_ARRAY))) + ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%'))) + ->andWhere($qb->expr()->lte('f.size', $qb->createNamedParameter(VerdictService::MAX_FILE_SIZE))) + ->andWhere($qb->expr()->like('f.path', $qb->createNamedParameter('files/%'))) + ->orderBy('f.fileid', 'DESC') + ->setMaxResults($limit); + + $fileIds = []; + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + $fileIds[] = $row['fileid']; + } + return $fileIds; + } + /** * Create a platform-specific cast function * @return string the database platform-specific cast function diff --git a/lib/Service/TagService.php b/lib/Service/TagService.php index 2b9a0022..07c47807 100644 --- a/lib/Service/TagService.php +++ b/lib/Service/TagService.php @@ -110,17 +110,17 @@ public function hasUnscannedTag(int $fileId): bool /** * @param string $tagName * @param int $limit Count of object ids you want to get - * @param string $offset The last object id you already received * @return array + * @throws Exception if the database platform is not supported */ - public function getFileIdsWithTag(string $tagName, int $limit, string $offset): array + public function getFileIdsWithTag(string $tagName, int $limit): array { try { $tag = $this->getTag($tagName, false); } catch (TagNotFoundException) { return []; } - return $this->tagMapper->getObjectIdsForTags([$tag->getId()], 'files', $limit, $offset); + return $this->dbFileMapper->getFileIdsWithTags([$tag->getId()], $limit); } /** @@ -138,24 +138,24 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit): array * Get file ids that have any of the given tags * @param array $tagIds The tags to get the file ids for * @param int $limit The count of file ids you want to get - * @param ISystemTag|null $priorTagId Tag id to prioritize over the others + * @param ISystemTag|null $priorityTagId Tag id to prioritize over the others * @return array * @throws TagNotFoundException if a tag does not exist + * @throws Exception If the database platform is not supported */ - public function getRandomTaggedFileIds(array $tagIds, int $limit, ?ISystemTag $priorTagId = null): array + public function getRandomTaggedFileIds(array $tagIds, int $limit, ?ISystemTag $priorityTagId = null): array { - if ($priorTagId === null) { - $objectIds = $this->tagMapper->getObjectIdsForTags($tagIds, 'files'); - shuffle($objectIds); - return array_slice($objectIds, 0, $limit); + $objectIdsPriority = []; + if ($priorityTagId !== null) { + $objectIdsPriority = $this->dbFileMapper->getFileIdsWithTags([$priorityTagId->getId()], $limit); + shuffle($objectIdsPriority); } - $objectIdsPrior = $this->tagMapper->getObjectIdsForTags([$priorTagId->getId()], 'files', $limit, 0); - if (count($objectIdsPrior) >= $limit) { - return $objectIdsPrior; + if (count($objectIdsPriority) < $limit) { + $objectIds = $this->dbFileMapper->getFileIdsWithTags($tagIds, $limit - count($objectIdsPriority)); + shuffle($objectIds); + return array_merge($objectIdsPriority, $objectIds); } - $objectIds = $this->tagMapper->getObjectIdsForTags($tagIds, 'files'); - shuffle($objectIds); - return array_merge($objectIdsPrior, array_slice($objectIds, 0, $limit - count($objectIdsPrior))); + return $objectIdsPriority; } /** From 6629edca8a7430221cb793fd3b7969ab22498e9f Mon Sep 17 00:00:00 2001 From: PT-ATA No One Date: Wed, 22 May 2024 09:53:42 +0200 Subject: [PATCH 8/8] take version from tag and write it to the appinfo --- .github/workflows/{makefile.yml => release-app.yml} | 6 ++++++ .gitignore | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) rename .github/workflows/{makefile.yml => release-app.yml} (70%) diff --git a/.github/workflows/makefile.yml b/.github/workflows/release-app.yml similarity index 70% rename from .github/workflows/makefile.yml rename to .github/workflows/release-app.yml index 6768b285..fdeb9ce6 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/release-app.yml @@ -23,6 +23,12 @@ jobs: - name: Build run: make build + - name: replace version + if: startsWith(github.ref, 'refs/tags/') + run: | + RELEASE_VERSION=${GITHUB_REF#refs/tags/} + sed -i "s/version = '0\.0\.0'/version = '$RELEASE_VERSION'/g" ./appinfo/info.xml + - name: Create artifact run: make appstore diff --git a/.gitignore b/.gitignore index 93d5a1ba..cfbc04f7 100644 --- a/.gitignore +++ b/.gitignore @@ -41,9 +41,6 @@ js/*hot-update.* # the fuelphp document /docs/ -# Github specific files -.github/ - # you may install these packages with `oil package`. # http://fuelphp.com/docs/packages/oil/package.html # /fuel/packages/auth/