Merge pull request #12 from magda-io/built-with-v3-sdk

Upgrade to Magda minion SDK v3
magda-io · Apr 17, 2024 · ecafafa · ecafafa
2 parents b87a9af + c767eb0
commit ecafafa
Show file tree

Hide file tree

Showing 24 changed files with 2,360 additions and 1,442 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -11,10 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js 14
+      - name: Use Node.js 18
         uses: actions/setup-node@v1
         with:
-          node-version: 14
+          node-version: 18
 
       - name: Login to GitHub Container Registry
         run: |
@@ -31,7 +31,7 @@ jobs:
       - name: Check Helm Chart Document
         run: |
           code=0
-          docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.5.0 -t ./README.md.gotmpl -o ../../README.md || code=$?;
+          docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.13.1 -t ./README.md.gotmpl -o ../../README.md || code=$?;
           if [ "$code" != "0" ]; then 
             echo "Failed to run helm-docs!";
             exit 1;
@@ -41,7 +41,7 @@ jobs:
           git ls-files -m | grep -i readme.md || code=$?;
           if [ "$code" == "0" ]; then
             echo -e "Some of helm chart docs are required to be updated using the [helm-docs](https://github.com/norwoodj/helm-docs) tool. \n
-            Please run helm-docs (v1.5.0) at project root, review & commit docs changes and push a new commit.";
+            Please run helm-docs (v1.13.1) at project root, review & commit docs changes and push a new commit.";
             exit 1;
           else 
             echo -e "helm docs check passed. helm docs update is not required.";

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,10 +14,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Use Node.js 14
+      - name: Use Node.js 18
         uses: actions/setup-node@v1
         with:
-          node-version: 14
+          node-version: 18
 
       - name: Login to GitHub Container Registry
         run: |
@@ -34,7 +34,7 @@ jobs:
       - name: Check Helm Chart Document
         run: |
           code=0
-          docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.5.0 -t ./README.md.gotmpl -o ../../README.md || code=$?;
+          docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.13.1 -t ./README.md.gotmpl -o ../../README.md || code=$?;
           if [ "$code" != "0" ]; then 
             echo "Failed to run helm-docs!";
             exit 1;
@@ -44,7 +44,7 @@ jobs:
           git ls-files -m | grep -i readme.md || code=$?;
           if [ "$code" == "0" ]; then
             echo -e "Some of helm chart docs are required to be updated using the [helm-docs](https://github.com/norwoodj/helm-docs) tool. \n
-            Please run helm-docs (v1.5.0) at project root, review & commit docs changes and push a new commit.";
+            Please run helm-docs (v1.13.1) at project root, review & commit docs changes and push a new commit.";
             exit 1;
           else 
             echo -e "helm docs check passed. helm docs update is not required.";

diff --git a/.gitignore b/.gitignore
@@ -51,4 +51,5 @@ magda-tenant-api/lib/
 #typescript
 tsconfig.tsbuildinfo
 
-deploy/*/charts
+deploy/*/charts
+coverage/
diff --git a/.mocharc.json b/.mocharc.json
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,3 +1,9 @@
+# 3.0.0
+
+-   Upgrade nodejs to version 18
+-   Upgrade to Magda minion SDK v3
+-   Fixed support for checking internal storage resource links
+
 # 2.0.0
 
 -   Upgrade nodejs to version 14

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:14-alpine
+FROM node:18-alpine
 
 RUN mkdir -p /usr/src/app
 COPY . /usr/src/app

diff --git a/README.md b/README.md
@@ -16,37 +16,39 @@ It's recommended to deploy minions with as [dependencies](https://helm.sh/docs/t
 
 ```yaml
 dependencies:
-    - name: magda-minion-broken-link
-      version: "2.0.0"
-      repository: "oci://ghcr.io/magda-io/charts"
+  - name: magda-minion-broken-link
+    version: "2.0.0"
+    repository: "oci://ghcr.io/magda-io/charts"
 ```
 
 ## Requirements
 
 Kubernetes: `>= 1.21.0`
 
-| Repository                    | Name         | Version |
-| ----------------------------- | ------------ | ------- |
-| oci://ghcr.io/magda-io/charts | magda-common | 2.1.1   |
+| Repository | Name | Version |
+|------------|------|---------|
+| oci://ghcr.io/magda-io/charts | magda-common | 2.1.1 |
 
 ## Values
 
-| Key                          | Type   | Default                                  | Description |
-| ---------------------------- | ------ | ---------------------------------------- | ----------- |
-| cronJobImage.name            | string | `"alpine"`                               |             |
-| cronJobImage.pullPolicy      | string | `"IfNotPresent"`                         |             |
-| cronJobImage.pullSecrets     | bool   | `false`                                  |             |
-| cronJobImage.repository      | string | `"docker.io"`                            |             |
-| cronJobImage.tag             | string | `"latest"`                               |             |
-| defaultAdminUserId           | string | `"00000000-0000-4000-8000-000000000000"` |             |
-| defaultImage.imagePullSecret | bool   | `false`                                  |             |
-| defaultImage.pullPolicy      | string | `"IfNotPresent"`                         |             |
-| defaultImage.repository      | string | `"ghcr.io/magda-io"`                     |             |
-| global.image                 | object | `{}`                                     |             |
-| global.minions.image         | object | `{}`                                     |             |
-| global.rollingUpdate         | object | `{}`                                     |             |
-| image.name                   | string | `"magda-minion-broken-link"`             |             |
-| resources.limits.cpu         | string | `"200m"`                                 |             |
-| resources.requests.cpu       | string | `"50m"`                                  |             |
-| resources.requests.memory    | string | `"40Mi"`                                 |             |
-| schedule                     | string | `"0 0 14,28 * *"`                        |             |
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| cronJobImage.name | string | `"alpine"` |  |
+| cronJobImage.pullPolicy | string | `"IfNotPresent"` |  |
+| cronJobImage.pullSecrets | bool | `false` |  |
+| cronJobImage.repository | string | `"docker.io"` |  |
+| cronJobImage.tag | string | `"latest"` |  |
+| datasetBucketName | string | `""` | The name of the storage bucket where all dataset files are stored.  Should match storage API config. By default, it will use the value of `global.defaultDatasetBucket` (defined in `magda-core` chart) unless you specify a different value here. |
+| defaultAdminUserId | string | `"00000000-0000-4000-8000-000000000000"` |  |
+| defaultImage.imagePullSecret | bool | `false` |  |
+| defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
+| defaultImage.repository | string | `"ghcr.io/magda-io"` |  |
+| global.image | object | `{}` |  |
+| global.minions.image | object | `{}` |  |
+| global.rollingUpdate | object | `{}` |  |
+| image.name | string | `"magda-minion-broken-link"` |  |
+| resources.limits.cpu | string | `"200m"` |  |
+| resources.requests.cpu | string | `"50m"` |  |
+| resources.requests.memory | string | `"40Mi"` |  |
+| schedule | string | `"0 0 14,28 * *"` |  |
+| storageApiBaseUrl | string | `"http://storage-api/v0"` | The base URL of the storage API to use when generating access URLs for MAGDA internal stored resources. |
diff --git a/deploy/magda-minion-broken-link/Chart.yaml b/deploy/magda-minion-broken-link/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: magda-minion-broken-link
 description: A Helm chart for Magda Broken Link Minion
-version: "2.0.0"
+version: "3.0.0"
 kubeVersion: ">= 1.21.0"
 home: "https://github.com/magda-io/magda-minion-broken-link"
 sources: [ "https://github.com/magda-io/magda-minion-broken-link" ]

diff --git a/deploy/magda-minion-broken-link/templates/deployment.yaml b/deploy/magda-minion-broken-link/templates/deployment.yaml
@@ -48,6 +48,10 @@ spec:
           value: {{ .Values.global.defaultAdminUserId | default .Values.defaultAdminUserId }}
         - name: INTERNAL_URL
           value: "http://minion-broken-link"
+        - name: STORAGE_API_BASE_URL
+          value: {{ .Values.storageApiBaseUrl | quote }}
+        - name: DATASET_BUCKET_NAME
+          value: {{ .Values.datasetBucketName | default .Values.global.defaultDatasetBucket | quote }}
 {{- if .Values.domainWaitTimeConfig }}
         - name: DOMAIN_WAIT_TIME_CONFIG
           value: '{{ toJson .Values.domainWaitTimeConfig | indent 2 }}'

diff --git a/deploy/magda-minion-broken-link/values.yaml b/deploy/magda-minion-broken-link/values.yaml
@@ -32,6 +32,14 @@ schedule: "0 0 14,28 * *"
 
 defaultAdminUserId: "00000000-0000-4000-8000-000000000000"
 
+# -- The base URL of the storage API to use when generating access URLs for MAGDA internal stored resources.
+storageApiBaseUrl: "http://storage-api/v0"
+
+# -- The name of the storage bucket where all dataset files are stored. 
+# Should match storage API config.
+# By default, it will use the value of `global.defaultDatasetBucket` (defined in `magda-core` chart) unless you specify a different value here.
+datasetBucketName: ""
+
 # Setup Domain Wait Time
 # domainWaitTimeConfig:
 #   data.csiro.au: 5

diff --git a/package.json b/package.json
@@ -1,7 +1,8 @@
 {
   "name": "@magda/minion-broken-link",
   "description": "MAGDA Broken Link Minion",
-  "version": "2.0.0",
+  "version": "3.0.0",
+  "type": "module",
   "scripts": {
     "prebuild": "rimraf dist tsconfig.tsbuildinfo",
     "build": "yarn run compile",
@@ -11,10 +12,10 @@
     "dev": "run-typescript-in-nodemon src/index.ts",
     "docker-build-local": "create-docker-context-for-node-component --build --push --tag auto --local",
     "docker-build-prod": "create-docker-context-for-node-component --build --push --tag auto",
-    "test": "mocha",
+    "test": "c8 mocha",
     "helm-lint": "helm lint deploy/magda-minion-broken-link -f deploy/test-deploy.yaml",
     "retag-and-push": "retag-and-push",
-    "helm-docs": "helm-docs -t ./README.md.gotmpl -o ../../README.md",
+    "helm-docs": "docker run --rm -v \"$(pwd):/helm-docs\" -u $(id -u) jnorwood/helm-docs:v1.13.1 -t ./README.md.gotmpl -o ../../README.md",
     "update-all-charts": "helm dep up ./deploy/magda-minion-broken-link",
     "add-all-chart-version-changes": "git ls-files -m | grep Chart.yaml | xargs git add && git ls-files -m | grep Chart.lock | xargs git add",
     "add-all-helm-docs-changes": "yarn helm-docs && git ls-files -m | grep -i readme.md | xargs git add",
@@ -23,39 +24,59 @@
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@magda/arbitraries": "^2.1.1",
+    "@magda/arbitraries": "^3.0.2-alpha.1",
     "@magda/ci-utils": "^1.0.5",
-    "@magda/docker-utils": "^2.1.1",
+    "@magda/docker-utils": "^3.0.2-alpha.1",
     "@types/chai": "^4.3.3",
     "@types/ftp": "^0.3.29",
     "@types/lodash": "^4.14.185",
     "@types/lru-cache": "4.0.0",
     "@types/mocha": "^9.1.1",
     "@types/nock": "^11.1.0",
+    "@types/node": "^18.19.31",
     "@types/read-pkg-up": "^3.0.1",
     "@types/request": "^2.48.1",
     "@types/sinon": "^7.5.1",
     "@types/urijs": "^1.19.19",
     "ajv": "^6.10.2",
-    "chai": "^4.2.0",
+    "c8": "^9.0.0",
+    "chai": "^5.0.0-rc.0",
     "husky": "^3.1.0",
     "jsverify": "^0.8.2",
-    "mocha": "^10.0.0",
-    "nock": "^13.2.9",
-    "prettier": "^1.19.1",
-    "pretty-quick": "^2.0.1",
+    "mocha": "^10.2.0",
+    "nock": "^13.4.0",
+    "prettier": "^3.2.5",
+    "pretty-quick": "^4.0.0",
     "rimraf": "^3.0.0",
     "sinon": "^8.1.1",
-    "ts-node": "^10.9.1",
-    "typescript": "^4.2.4"
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  },
+  "mocha": {
+    "import": "tsx/esm",
+    "spec": [
+      "src/test/**/*.spec.ts"
+    ]
+  },
+  "c8": {
+    "all": true,
+    "clean": true,
+    "src": [
+      "./src"
+    ],
+    "exclude": [
+      "src/test/**"
+    ]
   },
   "dependencies": {
-    "@magda/minion-sdk": "^2.1.1",
-    "@magda/registry-aspects": "^2.1.1",
-    "@magda/utils": "^2.1.1",
+    "@magda/minion-sdk": "^3.0.2-alpha.1",
+    "@magda/registry-aspects": "^3.0.2-alpha.1",
+    "@magda/utils": "^3.0.2-alpha.1",
     "ftp": "^0.3.10",
     "lodash": "^4.17.4",
     "lru-cache": "4.0.2",
+    "read-pkg-up": "^3.0.0",
+    "request": "^2.88.2",
     "urijs": "^1.19.11"
   },
   "config": {
@@ -77,5 +98,8 @@
     "hooks": {
       "pre-commit": "pretty-quick --staged"
     }
+  },
+  "engines": {
+    "node": ">=18.19.0"
   }
 }
diff --git a/prettier.config.js → prettier.config.mjs b/prettier.config.js → prettier.config.mjs
@@ -1,4 +1,4 @@
-module.exports = {
+const config = {
     tabWidth: 4,
     singleQuote: false,
     printWidth: 80,
@@ -25,3 +25,5 @@ module.exports = {
         }
     ]
 };
+
+export default config;