Merge pull request #20 from gardenlinux/show-complete-entries

Provide pre-fabricate CVE entries

src/glvd/cli/combine_all.py

@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import logging
+
+import asyncio
+from sqlalchemy import (
+    select,
+    text,
+)
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from ..database import Base, AllCve
+
+
+logger = logging.getLogger(__name__)
+
+
+class CombineDeb:
+    stmt_combine_new = (
+        text('''
+        SELECT
+                nvd_cve.cve_id
+                , nvd_cve.data
+                , array_to_json(
+                    array_agg(deb_cve.data_cpe_match)
+                ) AS data_cpe_matches
+            FROM
+                nvd_cve
+                INNER JOIN deb_cve USING (cve_id)
+            GROUP BY
+                nvd_cve.cve_id
+        ''')
+    )
+
+    async def combine_update(
+        self,
+        session: AsyncSession,
+        new_entries: dict[str, AllCve],
+    ) -> None:
+        stmt = select(AllCve)
+
+        async for r in await session.stream(stmt):
+            entry = r[0]
+
+            new_entry = new_entries.pop(entry.cve_id, None)
+            if not new_entry:
+                await session.delete(entry)
+                continue
+
+            # Update object in place. Only real changes will be commited
+            entry.merge(new_entry)
+
+    async def combine_insert(
+        self,
+        session: AsyncSession,
+        new_entries: dict[str, AllCve],
+    ) -> None:
+        for entry in new_entries.values():
+            session.add(entry)
+
+    async def combine(
+        self,
+        session: AsyncSession,
+    ) -> None:
+        new_entries: dict[str, AllCve] = {}
+
+        async for r in await session.stream(self.stmt_combine_new):
+            cve_id, data, data_cpe_matches = r
+
+            data.setdefault('configurations', []).append({
+                'nodes': [{
+                    'cpeMatch': data_cpe_matches,
+                    'negate': False,
+                    'operator': 'OR',
+                }],
+            })
+
+            new_entries[cve_id] = AllCve(
+                cve_id=cve_id,
+                data=data,
+            )
+
+        await self.combine_update(session, new_entries)
+        await self.combine_insert(session, new_entries)
+
+    async def __call__(
+        self,
+        engine: AsyncEngine,
+    ) -> None:
+        async with engine.begin() as conn:
+            await conn.run_sync(Base.metadata.create_all)
+
+        async with async_sessionmaker(engine)() as session:
+            await self.combine(session)
+            await session.commit()
+
+
+if __name__ == '__main__':
+    import argparse
+    logging.basicConfig(level=logging.DEBUG)
+    parser = argparse.ArgumentParser()
+    args = parser.parse_args()
+    engine = create_async_engine(
+        "postgresql+asyncpg:///",
+        echo=True,
+    )
+    main = CombineDeb()
+    asyncio.run(main(engine))

src/glvd/database/__init__.py

@@ -118,3 +118,14 @@ def merge(self, other: Self) -> None:
         self.deb_version_fixed = other.deb_version_fixed
         self.debsec_vulnerable = other.debsec_vulnerable
         self.data_cpe_match = other.data_cpe_match
+
+
+class AllCve(Base):
+    __tablename__ = 'all_cve'
+
+    cve_id: Mapped[str] = mapped_column(primary_key=True)
+    last_mod: Mapped[datetime] = mapped_column(init=False, server_default=func.now(), onupdate=func.now())
+    data: Mapped[Any]
+
+    def merge(self, other: Self) -> None:
+        self.data = other.data

src/glvd/web/nvd.py

@@ -11,20 +11,13 @@
 bp = Blueprint('nvd', __name__)
 
 
-# XXX: Can we replace that with a view, which combines data and data_configurations in the database?
 stmt_cve_deb_cpe_version = (
     text('''
         SELECT
-                nvd_cve.data,
-                array_to_json(
-                    array_remove(
-                        array_agg(deb_cve.data_cpe_match),
-                        NULL
-                    )
-                ) AS data_cpe_matches
+                all_cve.data
             FROM
-                nvd_cve
-                LEFT OUTER JOIN deb_cve USING (cve_id)
+                all_cve
+                INNER JOIN deb_cve USING (cve_id)
                 INNER JOIN dist_cpe ON (deb_cve.dist_id = dist_cpe.id)
             WHERE
                 dist_cpe.cpe_vendor = :cpe_vendor AND
@@ -35,8 +28,8 @@
                     deb_cve.deb_version_fixed > :deb_version OR
                     deb_cve.deb_version_fixed IS NULL
                 )
-            GROUP BY
-                nvd_cve.cve_id
+            ORDER BY
+                all_cve.cve_id
     ''')
     .bindparams(
         bindparam('cpe_vendor'),
@@ -50,25 +43,19 @@
 stmt_cve_deb_cpe_vulnerable = (
     text('''
         SELECT
-                nvd_cve.data,
-                array_to_json(
-                    array_remove(
-                        array_agg(deb_cve.data_cpe_match),
-                        NULL
-                    )
-                ) AS data_cpe_matches
+                all_cve.data
             FROM
-                nvd_cve
-                LEFT OUTER JOIN deb_cve USING (cve_id)
+                all_cve
+                INNER JOIN deb_cve USING (cve_id)
                 INNER JOIN dist_cpe ON (deb_cve.dist_id = dist_cpe.id)
             WHERE
                 dist_cpe.cpe_vendor = :cpe_vendor AND
                 dist_cpe.cpe_product = :cpe_product AND
                 dist_cpe.cpe_version LIKE :cpe_version AND
                 deb_cve.deb_source LIKE :deb_source AND
                 deb_cve.debsec_vulnerable = TRUE
-            GROUP BY
-                nvd_cve.cve_id
+            ORDER BY
+                all_cve.cve_id
     ''')
     .bindparams(
         bindparam('cpe_vendor'),
@@ -81,20 +68,13 @@
 stmt_cve_deb_cve_id = (
     text('''
         SELECT
-                nvd_cve.data,
-                array_to_json(
-                    array_remove(
-                        array_agg(deb_cve.data_cpe_match),
-                        NULL
-                    )
-                ) AS data_cpe_matches
+                all_cve.data
             FROM
-                nvd_cve
-                LEFT OUTER JOIN deb_cve USING (cve_id)
+                all_cve
             WHERE
                 cve_id = :cve_id
             GROUP BY
-                nvd_cve.cve_id
+                all_cve.cve_id
     ''')
     .bindparams(
         bindparam('cve_id'),
@@ -129,17 +109,8 @@ async def nvd_cve_deb():
     async with current_app.db_begin() as conn:
         results = []
         async for r in await conn.stream(stmt):
-            data, data_cpe_matches = r
-            if data_cpe_matches:
-                data.setdefault('configurations', []).append({
-                    'nodes': [{
-                        'cpeMatch': data_cpe_matches,
-                        'negate': False,
-                        'operator': 'OR',
-                    }],
-                })
             results.append({
-                'cve': data,
+                'cve': r[0],
             })
 
         return {

tests/web/test_nvd_cve.py

@@ -2,18 +2,15 @@
 
 import pytest
 
-from datetime import datetime
-
-from glvd.database import NvdCve
+from glvd.database import AllCve
 
 
 class TestNvdCve:
     @pytest.fixture(autouse=True, scope='class')
     async def setup_example(self, db_session_class):
         for i in range(2):
-            db_session_class.add(NvdCve(
+            db_session_class.add(AllCve(
                 cve_id=f'TEST-{i}',
-                last_mod=datetime.fromisoformat('2019-04-01T00:00:00'),
                 data={
                     'id': f'TEST-{i}',
                 },