diff --git a/.github/workflows/makefile.yml b/.github/workflows/release-app.yml similarity index 70% rename from .github/workflows/makefile.yml rename to .github/workflows/release-app.yml index 6768b285..fdeb9ce6 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/release-app.yml @@ -23,6 +23,12 @@ jobs: - name: Build run: make build + - name: replace version + if: startsWith(github.ref, 'refs/tags/') + run: | + RELEASE_VERSION=${GITHUB_REF#refs/tags/} + sed -i "s/version = '0\.0\.0'/version = '$RELEASE_VERSION'/g" ./appinfo/info.xml + - name: Create artifact run: make appstore diff --git a/.gitignore b/.gitignore index 93d5a1ba..cfbc04f7 100644 --- a/.gitignore +++ b/.gitignore @@ -41,9 +41,6 @@ js/*hot-update.* # the fuelphp document /docs/ -# Github specific files -.github/ - # you may install these packages with `oil package`. # http://fuelphp.com/docs/packages/oil/package.html # /fuel/packages/auth/ diff --git a/README.md b/README.md index 924c4253..19372532 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,9 @@ folder "nextcloud-docker-dev" and running ```docker compose up nextcloud proxy`` ### Useful commands -To trigger cronjobs manually you can use the following command: -```bash -docker exec --user www-data {nextcloud_container} php /var/www/html/cron.php -``` +| Description | Command | +|---------------------------|--------------------------------------------------------------------------------------| +| Trigger cronjobs manually | `docker exec --user www-data {nextcloud_container} php /var/www/html/cron.php` | +| Upgrade Nextcloud via CLI | `docker exec --user www-data {nextcloud_container} php occ upgrade` | +| Watch logs | `docker exec --user www-data {nextcloud_container} php occ log:watch` | +| Set log level to debug | `docker exec --user www-data {nextcloud_container} php occ log:manage --level DEBUG` | diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..fb00daeb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,48 @@ +version: "3.8" +services: + db: + image: mariadb + volumes: + - db_data:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: unsafe + MYSQL_USER: mysql + MYSQL_PASSWORD: unsafe + MYSQL_DATABASE: nextcloud + restart: unless-stopped + + redis: + image: redis:alpine + restart: unless-stopped + + nextcloud: + image: nextcloud:stable + volumes: + - nextcloud_data:/var/www/html + environment: + MYSQL_USER: mysql + MYSQL_PASSWORD: unsafe + MYSQL_DATABASE: nextcloud + MYSQL_HOST: db + REDIS_HOST: redis + ports: + - "80:80" + - "443:443" + restart: unless-stopped + depends_on: + - db + - redis + + cron: + image: nextcloud:stable + volumes: + - nextcloud_data:/var/www/html + entrypoint: /cron.sh + restart: unless-stopped + depends_on: + - db + - redis + +volumes: + db_data: {} + nextcloud_data: {} diff --git a/lib/BackgroundJobs/ScanJob.php b/lib/BackgroundJobs/ScanJob.php index c1a31747..08df5d7a 100644 --- a/lib/BackgroundJobs/ScanJob.php +++ b/lib/BackgroundJobs/ScanJob.php @@ -28,7 +28,7 @@ public function __construct(LoggerInterface $logger, ITimeFactory $time, TagServ $this->scanService = $scanService; $this->appConfig = $appConfig; - $this->setInterval(5 * 60); + $this->setInterval(60); $this->setAllowParallelRuns(false); $this->setTimeSensitivity(self::TIME_SENSITIVE); } @@ -74,7 +74,7 @@ protected function run($argument): void } } - $this->logger->debug("Scanning " . count($fileIds) . " files: " . implode(", ", $fileIds)); + $this->logger->debug("Scanning files"); foreach ($fileIds as $fileId) { try { @@ -83,5 +83,7 @@ protected function run($argument): void $this->logger->error("Failed to scan file with id " . $fileId . ": " . $e->getMessage()); } } + + $this->logger->debug("Scanned " . count($fileIds) . " files"); } } diff --git a/lib/BackgroundJobs/TagUnscannedJob.php b/lib/BackgroundJobs/TagUnscannedJob.php index 6eff4ae9..7add2239 100644 --- a/lib/BackgroundJobs/TagUnscannedJob.php +++ b/lib/BackgroundJobs/TagUnscannedJob.php @@ -7,6 +7,7 @@ use OCP\AppFramework\Utility\ITimeFactory; use OCP\DB\Exception; use OCP\IConfig; +use Psr\Log\LoggerInterface; class TagUnscannedJob extends TimedJob { @@ -14,15 +15,17 @@ class TagUnscannedJob extends TimedJob private TagService $tagService; private IConfig $appConfig; + private LoggerInterface $logger; - public function __construct(ITimeFactory $time, IConfig $appConfig, TagService $tagService) + public function __construct(ITimeFactory $time, IConfig $appConfig, TagService $tagService, LoggerInterface $logger) { parent::__construct($time); $this->appConfig = $appConfig; $this->tagService = $tagService; + $this->logger = $logger; - $this->setInterval(5 * 60); + $this->setInterval(60); $this->setAllowParallelRuns(false); $this->setTimeSensitivity(self::TIME_SENSITIVE); } @@ -40,6 +43,8 @@ protected function run($argument): void return; } + $this->logger->debug("Tagging unscanned files"); + $unscannedTag = $this->tagService->getTag(TagService::UNSCANNED); $maliciousTag = $this->tagService->getTag(TagService::MALICIOUS); $pupTag = $this->tagService->getTag(TagService::PUP); @@ -47,11 +52,7 @@ protected function run($argument): void $excludedTagIds = [$unscannedTag->getId(), $maliciousTag->getId(), $cleanTag->getId(), $pupTag->getId()]; - $fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 1000); - - if (count($fileIds) == 0) { - return; - } + $fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 10000); foreach ($fileIds as $fileId) { if ($this->tagService->hasCleanMaliciousOrPupTag($fileId)) { @@ -59,5 +60,7 @@ protected function run($argument): void } $this->tagService->setTag($fileId, TagService::UNSCANNED); } + + $this->logger->debug("Tagged " . count($fileIds) . " unscanned files"); } } diff --git a/lib/Db/DbFileMapper.php b/lib/Db/DbFileMapper.php index f2b1863b..dec809a7 100644 --- a/lib/Db/DbFileMapper.php +++ b/lib/Db/DbFileMapper.php @@ -36,6 +36,38 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit): array ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%'))) ->andWhere($qb->expr()->lte('f.size', $qb->createNamedParameter(VerdictService::MAX_FILE_SIZE))) ->andWhere($qb->expr()->like('f.path', $qb->createNamedParameter('files/%'))) + ->orderBy('f.fileid', 'DESC') + ->setMaxResults($limit); + + $fileIds = []; + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + $fileIds[] = $row['fileid']; + } + return $fileIds; + } + + /** + * Get file ids that have at least one of the given tags + * @param array $includedTagIds + * @param int $limit + * @return array of file ids + * @throws Exception if the database platform is not supported + */ + public function getFileIdsWithTags(array $includedTagIds, int $limit): array + { + $qb = $this->db->getQueryBuilder(); + $qb->automaticTablePrefix(true); + + $qb->select('f.fileid') + ->from($this->getTableName(), 'f') + ->leftJoin('f', 'systemtag_object_mapping', 'o', $qb->expr()->eq('f.fileid', $qb->createFunction($this->getPlatformSpecificCast()))) + ->leftJoin('f', 'mimetypes', 'm', $qb->expr()->eq('f.mimetype', 'm.id')) + ->where($qb->expr()->in('o.systemtagid', $qb->createNamedParameter($includedTagIds, IQueryBuilder::PARAM_INT_ARRAY))) + ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%'))) + ->andWhere($qb->expr()->lte('f.size', $qb->createNamedParameter(VerdictService::MAX_FILE_SIZE))) + ->andWhere($qb->expr()->like('f.path', $qb->createNamedParameter('files/%'))) + ->orderBy('f.fileid', 'DESC') ->setMaxResults($limit); $fileIds = []; diff --git a/lib/Service/TagService.php b/lib/Service/TagService.php index 2b9a0022..07c47807 100644 --- a/lib/Service/TagService.php +++ b/lib/Service/TagService.php @@ -110,17 +110,17 @@ public function hasUnscannedTag(int $fileId): bool /** * @param string $tagName * @param int $limit Count of object ids you want to get - * @param string $offset The last object id you already received * @return array + * @throws Exception if the database platform is not supported */ - public function getFileIdsWithTag(string $tagName, int $limit, string $offset): array + public function getFileIdsWithTag(string $tagName, int $limit): array { try { $tag = $this->getTag($tagName, false); } catch (TagNotFoundException) { return []; } - return $this->tagMapper->getObjectIdsForTags([$tag->getId()], 'files', $limit, $offset); + return $this->dbFileMapper->getFileIdsWithTags([$tag->getId()], $limit); } /** @@ -138,24 +138,24 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit): array * Get file ids that have any of the given tags * @param array $tagIds The tags to get the file ids for * @param int $limit The count of file ids you want to get - * @param ISystemTag|null $priorTagId Tag id to prioritize over the others + * @param ISystemTag|null $priorityTagId Tag id to prioritize over the others * @return array * @throws TagNotFoundException if a tag does not exist + * @throws Exception If the database platform is not supported */ - public function getRandomTaggedFileIds(array $tagIds, int $limit, ?ISystemTag $priorTagId = null): array + public function getRandomTaggedFileIds(array $tagIds, int $limit, ?ISystemTag $priorityTagId = null): array { - if ($priorTagId === null) { - $objectIds = $this->tagMapper->getObjectIdsForTags($tagIds, 'files'); - shuffle($objectIds); - return array_slice($objectIds, 0, $limit); + $objectIdsPriority = []; + if ($priorityTagId !== null) { + $objectIdsPriority = $this->dbFileMapper->getFileIdsWithTags([$priorityTagId->getId()], $limit); + shuffle($objectIdsPriority); } - $objectIdsPrior = $this->tagMapper->getObjectIdsForTags([$priorTagId->getId()], 'files', $limit, 0); - if (count($objectIdsPrior) >= $limit) { - return $objectIdsPrior; + if (count($objectIdsPriority) < $limit) { + $objectIds = $this->dbFileMapper->getFileIdsWithTags($tagIds, $limit - count($objectIdsPriority)); + shuffle($objectIds); + return array_merge($objectIdsPriority, $objectIds); } - $objectIds = $this->tagMapper->getObjectIdsForTags($tagIds, 'files'); - shuffle($objectIds); - return array_merge($objectIdsPrior, array_slice($objectIds, 0, $limit - count($objectIdsPrior))); + return $objectIdsPriority; } /**