Merge branch 'develop' into lineage/skeleton

.dockerignore

@@ -0,0 +1,14 @@
+.serverless
+.build
+.github
+.gitignore
+dist
+postgres-data
+es-data
+matomo-data
+matomo-db-data
+nvd-dump
+minio-data
+**/node_modules
+**/.cache
+./docs/node_modules

.github/CODEOWNERS

@@ -3,7 +3,9 @@
 # These owners will be the default owners for everything in the
 # repo. Unless a later match takes precedence, these owners will be
 # requested for review when someone opens a pull request.
-* @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj
+
+
+* @aloftus23 @cduhn17 @Matthew-Grayson @nickviola @rapidray12 @schmelz21
 
 # These folks own any files in the .github directory at the root of
 # the repository and any of its subdirectories.

.github/codeql.yml

@@ -0,0 +1,4 @@
+---
+query-filters:
+  - exclude:
+      id: js/unused-local-variable

.github/dependabot.yml

@@ -1,10 +1,5 @@
 ---
-
-# Any ignore directives should be uncommented in downstream projects to disable
-# Dependabot updates for the given dependency. Downstream projects will get
-# these updates when the pull request(s) in the appropriate skeleton are merged
-# and Lineage processes these changes.
-
+version: 2
 updates:
   - directory: /
     # ignore:
@@ -22,14 +17,50 @@ updates:
     package-ecosystem: github-actions
     schedule:
       interval: weekly
-
-  - directory: /
-    package-ecosystem: pip
-    schedule:
-      interval: weekly
-
   - directory: /
     package-ecosystem: terraform
     schedule:
       interval: weekly
-version: 2
+  - package-ecosystem: 'npm'
+    directory: '/'
+    schedule:
+      interval: 'weekly'
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch","version-update:semver-minor"]
+  - package-ecosystem: "npm"
+    directory: "/frontend"
+    schedule:
+      interval: "weekly"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch","version-update:semver-minor"]
+  - package-ecosystem: "npm"
+    directory: "/backend"
+    schedule:
+      interval: "weekly"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch","version-update:semver-minor"]
+  - package-ecosystem: "pip"
+    directory: "/backend/worker"
+    schedule:
+      interval: "weekly"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch","version-update:semver-minor"]
+  - package-ecosystem: 'docker'
+    directory: '/'
+    schedule:
+      interval: 'weekly'
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch","version-update:semver-minor"]
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'weekly'
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch","version-update:semver-minor"]
+

.github/workflows/backend.yml

@@ -0,0 +1,241 @@
+---
+name: Backend Pipeline
+
+on:
+  push:
+    branches:
+      - develop
+      - production
+    paths:
+      - 'backend/**'
+      - '.github/workflows/backend.yml'
+  pull_request:
+    branches:
+      - develop
+      - production
+    paths:
+      - 'backend/**'
+      - '.github/workflows/backend.yml'
+
+defaults:
+  run:
+    working-directory: ./backend
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - name: Restore npm cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: npm ci
+      - name: Lint
+        run: npm run lint
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - name: Restore npm cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: npm ci
+      - name: Run site locally
+        run: |
+          cp dev.env.example .env
+          docker-compose up -d db backend es
+          npm install -g wait-port
+          wait-port -t 3000 5432 9200 9300
+        working-directory: ./
+      - name: Sync database
+        run: npm run syncdb
+        working-directory: ./backend
+      - name: Test
+        run: npm run test -- --collectCoverage --silent
+      - name: Package
+        run: npx sls package
+        env:
+          SLS_DEBUG: '*'
+  test_worker:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - name: Restore npm cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: npm ci
+      - name: Build
+        run: npx webpack --config webpack.worker.config.js
+      - name: Run db locally
+        run: |
+          cp dev.env.example .env
+          docker-compose up -d db
+          npm install -g wait-port
+          wait-port -t 3000 5432
+        working-directory: ./
+      - name: Test
+        run: node dist/worker.bundle.js
+        env:
+          CROSSFEED_COMMAND_OPTIONS: '{"scanName": "test"}'
+          DB_USERNAME: crossfeed
+          DB_PASSWORD: password
+  test_python:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/[email protected]
+        with:
+          python-version: '3.10'
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            pip-
+      - run: pip install -r worker/requirements.txt
+      - run: pytest
+  build_worker:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - name: Restore npm cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: npm ci
+      - name: Build worker container
+        run: npm run build-worker
+        working-directory: ./backend
+  deploy_staging:
+    needs: [build_worker, lint, test, test_worker, test_python]
+    runs-on: ubuntu-latest
+    environment: staging
+    concurrency: '1'
+    if: github.event_name == 'push' && github.ref == 'refs/heads/develop'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - name: Restore npm cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Ensure domain exists
+        run: npx sls create_domain --stage=staging
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SLS_DEBUG: '*'
+
+      - name: Deploy backend
+        run: npx sls deploy --stage=staging
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SLS_DEBUG: '*'
+
+      - name: Deploy worker
+        run: npm run deploy-worker-staging
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncdb
+        run: aws lambda invoke --function-name crossfeed-staging-syncdb --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+  deploy_prod:
+    needs: [build_worker, lint, test, test_python]
+    runs-on: ubuntu-latest
+    environment: production
+    concurrency: '1'
+    if: github.event_name == 'push' && github.ref == 'refs/heads/production'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - name: Restore npm cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Ensure domain exists
+        run: npx sls create_domain --stage=prod
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SLS_DEBUG: '*'
+
+      - name: Deploy backend
+        run: npx sls deploy --stage=prod
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SLS_DEBUG: '*'
+
+      - name: Deploy worker
+        run: npm run deploy-worker-prod
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncdb
+        run: aws lambda invoke --function-name crossfeed-prod-syncdb --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/codeql.yml

@@ -0,0 +1,43 @@
+---
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "develop", "production" ]
+  pull_request:
+    branches: [ "develop" ]
+  schedule:
+    - cron: "23 17 * * 6"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ javascript ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          config-file: ./.github/codeql.yml
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"