Merge pull request #27 from GDATASoftwareAG/scanning-improvements

Scanning improvements
GDATASoftwareAG · May 22, 2024 · 520a0b9 · 520a0b9
2 parents 0f3c429 + 6629edc
commit 520a0b9
Show file tree

Hide file tree

Showing 8 changed files with 121 additions and 31 deletions.
diff --git a/.github/workflows/makefile.yml → .github/workflows/release-app.yml b/.github/workflows/makefile.yml → .github/workflows/release-app.yml
@@ -23,6 +23,12 @@ jobs:
     - name: Build 
       run: make build
 
+    - name: replace version
+      if: startsWith(github.ref, 'refs/tags/')
+      run: |
+        RELEASE_VERSION=${GITHUB_REF#refs/tags/}
+        sed -i "s/version = '0\.0\.0'/version = '$RELEASE_VERSION'/g" ./appinfo/info.xml
+
     - name: Create artifact 
       run: make appstore
 

diff --git a/.gitignore b/.gitignore
@@ -41,9 +41,6 @@ js/*hot-update.*
 # the fuelphp document
 /docs/
 
-# Github specific files
-.github/
-
 # you may install these packages with `oil package`.
 # http://fuelphp.com/docs/packages/oil/package.html
 # /fuel/packages/auth/

diff --git a/README.md b/README.md
@@ -56,7 +56,9 @@ folder "nextcloud-docker-dev" and running ```docker compose up nextcloud proxy``
 
 ### Useful commands
 
-To trigger cronjobs manually you can use the following command:
-```bash
-docker exec --user www-data {nextcloud_container} php /var/www/html/cron.php
-```
+| Description               | Command                                                                              |
+|---------------------------|--------------------------------------------------------------------------------------|
+| Trigger cronjobs manually | `docker exec --user www-data {nextcloud_container} php /var/www/html/cron.php`       |
+| Upgrade Nextcloud via CLI | `docker exec --user www-data {nextcloud_container} php occ upgrade`                  |
+| Watch logs                | `docker exec --user www-data {nextcloud_container} php occ log:watch`                |
+| Set log level to debug    | `docker exec --user www-data {nextcloud_container} php occ log:manage --level DEBUG` |
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,48 @@
+version: "3.8"
+services:
+  db:
+    image: mariadb
+    volumes:
+      - db_data:/var/lib/mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: unsafe
+      MYSQL_USER: mysql
+      MYSQL_PASSWORD: unsafe
+      MYSQL_DATABASE: nextcloud
+    restart: unless-stopped
+
+  redis:
+    image: redis:alpine
+    restart: unless-stopped
+
+  nextcloud:
+    image: nextcloud:stable
+    volumes:
+      - nextcloud_data:/var/www/html
+    environment:
+      MYSQL_USER: mysql
+      MYSQL_PASSWORD: unsafe
+      MYSQL_DATABASE: nextcloud
+      MYSQL_HOST: db
+      REDIS_HOST: redis
+    ports:
+     - "80:80"
+     - "443:443"
+    restart: unless-stopped
+    depends_on:
+      - db
+      - redis
+
+  cron:
+    image: nextcloud:stable
+    volumes:
+      - nextcloud_data:/var/www/html
+    entrypoint: /cron.sh
+    restart: unless-stopped
+    depends_on:
+      - db
+      - redis
+
+volumes:
+  db_data: {}
+  nextcloud_data: {}
diff --git a/lib/BackgroundJobs/ScanJob.php b/lib/BackgroundJobs/ScanJob.php
@@ -28,7 +28,7 @@ public function __construct(LoggerInterface $logger, ITimeFactory $time, TagServ
         $this->scanService = $scanService;
         $this->appConfig = $appConfig;
 
-        $this->setInterval(5 * 60);
+        $this->setInterval(60);
         $this->setAllowParallelRuns(false);
         $this->setTimeSensitivity(self::TIME_SENSITIVE);
     }
@@ -74,7 +74,7 @@ protected function run($argument): void
             }
         }
 
-        $this->logger->debug("Scanning " . count($fileIds) . " files: " . implode(", ", $fileIds));
+        $this->logger->debug("Scanning files");
 
         foreach ($fileIds as $fileId) {
             try {
@@ -83,5 +83,7 @@ protected function run($argument): void
                 $this->logger->error("Failed to scan file with id " . $fileId . ": " . $e->getMessage());
             }
         }
+
+        $this->logger->debug("Scanned " . count($fileIds) . " files");
     }
 }
diff --git a/lib/BackgroundJobs/TagUnscannedJob.php b/lib/BackgroundJobs/TagUnscannedJob.php
@@ -7,22 +7,25 @@
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\DB\Exception;
 use OCP\IConfig;
+use Psr\Log\LoggerInterface;
 
 class TagUnscannedJob extends TimedJob
 {
     private const APP_ID = "gdatavaas";
 
     private TagService $tagService;
     private IConfig $appConfig;
+    private LoggerInterface $logger;
 
-    public function __construct(ITimeFactory $time, IConfig $appConfig, TagService $tagService)
+    public function __construct(ITimeFactory $time, IConfig $appConfig, TagService $tagService, LoggerInterface $logger)
     {
         parent::__construct($time);
 
         $this->appConfig = $appConfig;
         $this->tagService = $tagService;
+        $this->logger = $logger;
 
-        $this->setInterval(5 * 60);
+        $this->setInterval(60);
         $this->setAllowParallelRuns(false);
         $this->setTimeSensitivity(self::TIME_SENSITIVE);
     }
@@ -40,24 +43,24 @@ protected function run($argument): void
             return;
         }
 
+        $this->logger->debug("Tagging unscanned files");
+
         $unscannedTag = $this->tagService->getTag(TagService::UNSCANNED);
         $maliciousTag = $this->tagService->getTag(TagService::MALICIOUS);
         $pupTag = $this->tagService->getTag(TagService::PUP);
         $cleanTag = $this->tagService->getTag(TagService::CLEAN);
 
         $excludedTagIds = [$unscannedTag->getId(), $maliciousTag->getId(), $cleanTag->getId(), $pupTag->getId()];
 
-        $fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 1000);
-
-        if (count($fileIds) == 0) {
-            return;
-        }
+        $fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 10000);
 
         foreach ($fileIds as $fileId) {
             if ($this->tagService->hasCleanMaliciousOrPupTag($fileId)) {
                 continue;
             }
             $this->tagService->setTag($fileId, TagService::UNSCANNED);
         }
+
+        $this->logger->debug("Tagged " . count($fileIds) . " unscanned files");
     }
 }
diff --git a/lib/Db/DbFileMapper.php b/lib/Db/DbFileMapper.php
@@ -36,6 +36,38 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit): array
             ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%')))
             ->andWhere($qb->expr()->lte('f.size', $qb->createNamedParameter(VerdictService::MAX_FILE_SIZE)))
             ->andWhere($qb->expr()->like('f.path', $qb->createNamedParameter('files/%')))
+            ->orderBy('f.fileid', 'DESC')
+            ->setMaxResults($limit);
+
+        $fileIds = [];
+        $result = $qb->executeQuery();
+        while ($row = $result->fetch()) {
+            $fileIds[] = $row['fileid'];
+        }
+        return $fileIds;
+    }
+
+    /**
+     * Get file ids that have at least one of the given tags
+     * @param array $includedTagIds
+     * @param int $limit
+     * @return array of file ids
+     * @throws Exception if the database platform is not supported
+     */
+    public function getFileIdsWithTags(array $includedTagIds, int $limit): array
+    {
+        $qb = $this->db->getQueryBuilder();
+        $qb->automaticTablePrefix(true);
+
+        $qb->select('f.fileid')
+            ->from($this->getTableName(), 'f')
+            ->leftJoin('f', 'systemtag_object_mapping', 'o', $qb->expr()->eq('f.fileid', $qb->createFunction($this->getPlatformSpecificCast())))
+            ->leftJoin('f', 'mimetypes', 'm', $qb->expr()->eq('f.mimetype', 'm.id'))
+            ->where($qb->expr()->in('o.systemtagid', $qb->createNamedParameter($includedTagIds, IQueryBuilder::PARAM_INT_ARRAY)))
+            ->andWhere($qb->expr()->notLike('m.mimetype', $qb->createNamedParameter('%unix-directory%')))
+            ->andWhere($qb->expr()->lte('f.size', $qb->createNamedParameter(VerdictService::MAX_FILE_SIZE)))
+            ->andWhere($qb->expr()->like('f.path', $qb->createNamedParameter('files/%')))
+            ->orderBy('f.fileid', 'DESC')
             ->setMaxResults($limit);
 
         $fileIds = [];

diff --git a/lib/Service/TagService.php b/lib/Service/TagService.php
@@ -110,17 +110,17 @@ public function hasUnscannedTag(int $fileId): bool
     /**
      * @param string $tagName
      * @param int $limit Count of object ids you want to get
-     * @param string $offset The last object id you already received
      * @return array
+     * @throws Exception if the database platform is not supported
      */
-    public function getFileIdsWithTag(string $tagName, int $limit, string $offset): array
+    public function getFileIdsWithTag(string $tagName, int $limit): array
     {
         try {
             $tag = $this->getTag($tagName, false);
         } catch (TagNotFoundException) {
             return [];
         }
-        return $this->tagMapper->getObjectIdsForTags([$tag->getId()], 'files', $limit, $offset);
+        return $this->dbFileMapper->getFileIdsWithTags([$tag->getId()], $limit);
     }
 
     /**
@@ -138,24 +138,24 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit): array
      * Get file ids that have any of the given tags
      * @param array $tagIds The tags to get the file ids for
      * @param int $limit The count of file ids you want to get
-     * @param ISystemTag|null $priorTagId Tag id to prioritize over the others
+     * @param ISystemTag|null $priorityTagId Tag id to prioritize over the others
      * @return array
      * @throws TagNotFoundException if a tag does not exist
+     * @throws Exception If the database platform is not supported
      */
-    public function getRandomTaggedFileIds(array $tagIds, int $limit, ?ISystemTag $priorTagId = null): array
+    public function getRandomTaggedFileIds(array $tagIds, int $limit, ?ISystemTag $priorityTagId = null): array
     {
-        if ($priorTagId === null) {
-            $objectIds = $this->tagMapper->getObjectIdsForTags($tagIds, 'files');
-            shuffle($objectIds);
-            return array_slice($objectIds, 0, $limit);
+        $objectIdsPriority = [];
+        if ($priorityTagId !== null) {
+            $objectIdsPriority = $this->dbFileMapper->getFileIdsWithTags([$priorityTagId->getId()], $limit);
+            shuffle($objectIdsPriority);
         }
-        $objectIdsPrior = $this->tagMapper->getObjectIdsForTags([$priorTagId->getId()], 'files', $limit, 0);
-        if (count($objectIdsPrior) >= $limit) {
-            return $objectIdsPrior;
+        if (count($objectIdsPriority) < $limit) {
+            $objectIds = $this->dbFileMapper->getFileIdsWithTags($tagIds, $limit - count($objectIdsPriority));
+            shuffle($objectIds);
+            return array_merge($objectIdsPriority, $objectIds);
         }
-        $objectIds = $this->tagMapper->getObjectIdsForTags($tagIds, 'files');
-        shuffle($objectIds);
-        return array_merge($objectIdsPrior, array_slice($objectIds, 0, $limit - count($objectIdsPrior)));
+        return $objectIdsPriority;
     }
 
     /**