Upgrade Taskgraph and cleanup firefox-translations-training (#341)

* Delete in-tree docker_image and fetch transforms * Set default task-expires-after to 90 days * Use default task-expires-after to set expires-after * Use Taskgraph 7.1.0
mozilla · Jan 8, 2024 · 03a5ff8 · 03a5ff8
1 parent 5214364
commit 03a5ff8
Show file tree

Hide file tree

Showing 39 changed files with 263 additions and 818 deletions.
diff --git a/taskcluster/config.yml b/taskcluster/config.yml
@@ -2,6 +2,7 @@
 trust-domain: "translations"
 task-priority: low
 task-deadline-after: 60 days
+task-expires-after: 90 days
 treeherder:
     group-names:
         "I": "Docker images"

diff --git a/taskcluster/kinds/all-pr/kind.yml b/taskcluster/kinds/all-pr/kind.yml
@@ -34,7 +34,6 @@ tasks:
                 - attributes
 
         run-on-tasks-for: ["github-pull-request"]
-        expires-after: "90 days"
         worker-type: succeed
 
         from-deps:

diff --git a/taskcluster/kinds/bicleaner/kind.yml b/taskcluster/kinds/bicleaner/kind.yml
@@ -48,8 +48,6 @@ task-defaults:
         substitution-fields:
             - run.command
 
-    expires-after: "90 days"
-
     worker:
         max-run-time: 86400
         env:

diff --git a/taskcluster/kinds/cefilter/kind.yml b/taskcluster/kinds/cefilter/kind.yml
@@ -40,7 +40,6 @@ tasks:
                 - fetches
                 - attributes
         worker-type: b-cpu-largedisk
-        expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}
             max-run-time: 86400

diff --git a/taskcluster/kinds/clean-corpus/kind.yml b/taskcluster/kinds/clean-corpus/kind.yml
@@ -54,7 +54,6 @@ tasks:
                 - fetches
                 - worker.env
                 - run.command
-        expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}
             max-run-time: 86400

diff --git a/taskcluster/kinds/clean-mono/kind.yml b/taskcluster/kinds/clean-mono/kind.yml
@@ -54,7 +54,6 @@ task-defaults:
             - dependencies
             - fetches
             - run.command
-    expires-after: "90 days"
     worker:
         docker-image: {"in-tree": "train"}
         max-run-time: 86400

diff --git a/taskcluster/kinds/collect-corpus/kind.yml b/taskcluster/kinds/collect-corpus/kind.yml
@@ -54,7 +54,6 @@ tasks:
                     - artifact: corpus.{trg_locale}.zst
 
         worker-type: b-cpu-largedisk
-        expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}
             max-run-time: 86400

diff --git a/taskcluster/kinds/collect-mono-src/kind.yml b/taskcluster/kinds/collect-mono-src/kind.yml
@@ -35,7 +35,6 @@ task-defaults:
             - fetches
             - attributes
     worker-type: b-cpu-largedisk
-    expires-after: "90 days"
     worker:
         docker-image: {"in-tree": "train"}
         max-run-time: 86400

diff --git a/taskcluster/kinds/collect-mono-trg/kind.yml b/taskcluster/kinds/collect-mono-trg/kind.yml
@@ -35,7 +35,6 @@ task-defaults:
             - fetches
             - attributes
     worker-type: b-cpu-largedisk
-    expires-after: "90 days"
     worker:
         docker-image: {"in-tree": "train"}
         max-run-time: 86400

diff --git a/taskcluster/kinds/dataset/kind.yml b/taskcluster/kinds/dataset/kind.yml
@@ -28,7 +28,6 @@ task-defaults:
             - run.command
     task-context:
         substitution-fields: []
-    expires-after: "90 days"
     worker:
         docker-image: {in-tree: toolchain-build}
         max-run-time: 86400

diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml
@@ -2,7 +2,7 @@
 loader: taskgraph.loader.transform:loader
 
 transforms:
-    - translations_taskgraph.transforms.docker_image:transforms
+    - taskgraph.transforms.docker_image:transforms
     - taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/kinds/evaluate-quantized/kind.yml b/taskcluster/kinds/evaluate-quantized/kind.yml
@@ -53,7 +53,6 @@ task-defaults:
             pipeline_setup: >-
 
     worker-type: b-gpu
-    expires-after: "90 days"
     worker:
         artifacts:
             - name: public/build

diff --git a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml
@@ -44,7 +44,6 @@ task-defaults:
             src_locale: training_config.experiment.src
             trg_locale: training_config.experiment.trg
     worker-type: b-gpu
-    expires-after: "90 days"
     worker:
         artifacts:
             - name: public/build

diff --git a/taskcluster/kinds/evaluate/kind.yml b/taskcluster/kinds/evaluate/kind.yml
@@ -49,7 +49,6 @@ task-defaults:
             trg_locale: training_config.experiment.trg
             split_chunks: training_config.experiment.teacher-ensemble
     worker-type: b-gpu
-    expires-after: "90 days"
     worker:
         artifacts:
             - name: public/build

diff --git a/taskcluster/kinds/export/kind.yml b/taskcluster/kinds/export/kind.yml
@@ -40,7 +40,6 @@ tasks:
                 - dependencies
                 - worker.env
                 - attributes
-        expires-after: "90 days"
         worker-type: b-cpu
         worker:
             docker-image: {"in-tree": "train"}

diff --git a/taskcluster/kinds/extract-best/kind.yml b/taskcluster/kinds/extract-best/kind.yml
@@ -58,7 +58,6 @@ tasks:
                 - run.command
 
         worker-type: b-cpu-largedisk
-        expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}
             max-run-time: 86400

diff --git a/taskcluster/kinds/fetch/kind.yml b/taskcluster/kinds/fetch/kind.yml
@@ -5,13 +5,12 @@
 loader: taskgraph.loader.transform:loader
 
 transforms:
-    - translations_taskgraph.transforms.fetch:transforms
+    - taskgraph.transforms.fetch:transforms
     - taskgraph.transforms.run:transforms
     - taskgraph.transforms.task:transforms
 
 task-defaults:
     docker-image: {in-tree: base}
-    expires-after: "90 days"
 
 tasks-from:
     - bicleaner.yml

diff --git a/taskcluster/kinds/finetune-student/kind.yml b/taskcluster/kinds/finetune-student/kind.yml
@@ -55,7 +55,6 @@ tasks:
         run-on-tasks-for: []
 
         worker-type: b-largegpu-xlargedisk
-        expires-after: "90 days"
         worker:
             max-run-time: 2592000
             env:

diff --git a/taskcluster/kinds/merge-corpus/kind.yml b/taskcluster/kinds/merge-corpus/kind.yml
@@ -52,7 +52,6 @@ task-defaults:
             - "{dataset_sanitized}.{src_locale}.zst"
             - "{dataset_sanitized}.{trg_locale}.zst"
     worker-type: b-cpu-largedisk
-    expires-after: "90 days"
     worker:
         docker-image: {"in-tree": "train"}
         max-run-time: 86400

diff --git a/taskcluster/kinds/merge-devset/kind.yml b/taskcluster/kinds/merge-devset/kind.yml
@@ -52,7 +52,6 @@ task-defaults:
             - "{dataset_sanitized}.{src_locale}.zst"
             - "{dataset_sanitized}.{trg_locale}.zst"
     worker-type: b-cpu
-    expires-after: "90 days"
     worker:
         docker-image: {"in-tree": "train"}
         max-run-time: 86400

diff --git a/taskcluster/kinds/merge-mono/kind.yml b/taskcluster/kinds/merge-mono/kind.yml
@@ -47,7 +47,6 @@ task-defaults:
             - "{dataset_sanitized}.{locale}.zst"
 
     worker-type: b-cpu-largedisk
-    expires-after: "90 days"
     worker:
         docker-image: { "in-tree": "train" }
         max-run-time: 86400

diff --git a/taskcluster/kinds/merge-translated/kind.yml b/taskcluster/kinds/merge-translated/kind.yml
@@ -39,7 +39,6 @@ task-defaults:
             - run.command
             - attributes
     worker-type: b-cpu-largedisk
-    expires-after: "90 days"
     worker:
         docker-image: {"in-tree": "train"}
         max-run-time: 86400

diff --git a/taskcluster/kinds/quantize/kind.yml b/taskcluster/kinds/quantize/kind.yml
@@ -46,7 +46,6 @@ tasks:
                 - run.command
                 - attributes
         worker-type: b-cpu
-        expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}
             max-run-time: 86400

diff --git a/taskcluster/kinds/score/kind.yml b/taskcluster/kinds/score/kind.yml
@@ -45,7 +45,6 @@ tasks:
                 - run.command
 
         worker-type: b-largegpu
-        expires-after: "90 days"
         worker:
             max-run-time: 2592000
             env:

diff --git a/taskcluster/kinds/split-corpus/kind.yml b/taskcluster/kinds/split-corpus/kind.yml
@@ -41,7 +41,6 @@ tasks:
                 - attributes
                 - run.command
         worker-type: b-cpu-largedisk
-        expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}
             max-run-time: 86400

diff --git a/taskcluster/kinds/split-mono-src/kind.yml b/taskcluster/kinds/split-mono-src/kind.yml
@@ -39,7 +39,6 @@ task-defaults:
             - run.command
 
     worker-type: b-cpu-largedisk
-    expires-after: "90 days"
     worker:
         docker-image: { "in-tree": "train" }
         max-run-time: 86400

diff --git a/taskcluster/kinds/split-mono-trg/kind.yml b/taskcluster/kinds/split-mono-trg/kind.yml
@@ -39,7 +39,6 @@ task-defaults:
             - run.command
 
     worker-type: b-cpu-largedisk
-    expires-after: "90 days"
     worker:
         docker-image: { "in-tree": "train" }
         max-run-time: 86400

diff --git a/taskcluster/kinds/tests/kind.yml b/taskcluster/kinds/tests/kind.yml
@@ -25,7 +25,6 @@ tasks:
   snakemake-dry-run:
     # Ensure that the snakemake workflow is still executing correctly, even though
     # taskcluster is the preferred execution environment.
-    expires-after: "90 days"
     worker-type: b-cpu
     worker:
       max-run-time: 3600
@@ -58,7 +57,6 @@ tasks:
 
   black:
     # Run python's black formatter, which formats python files.
-    expires-after: "90 days"
     worker-type: b-cpu
     worker:
       max-run-time: 3600
@@ -74,7 +72,6 @@ tasks:
 
   lint:
     # Run ruff, a python linter.
-    expires-after: "90 days"
     worker-type: b-cpu
     worker:
       max-run-time: 3600
@@ -90,7 +87,6 @@ tasks:
 
   test:
     # Run unit tests
-    expires-after: "90 days"
     worker-type: b-cpu
     worker:
       max-run-time: 3600
@@ -111,7 +107,6 @@ tasks:
         - Makefile
 
   taskgraph-definition:
-      expires-after: "90 days"
       worker-type: b-cpu
       worker:
           docker-image: {in-tree: base}
@@ -127,7 +122,6 @@ tasks:
               make validate-taskgraph
 
   taskgraph-diff:
-      expires-after: "90 days"
       worker-type: b-cpu
       worker:
           docker-image: {in-tree: base}

diff --git a/taskcluster/kinds/toolchain/kind.yml b/taskcluster/kinds/toolchain/kind.yml
@@ -14,7 +14,6 @@ transforms:
 
 task-defaults:
     worker-type: b-cpu
-    expires-after: "90 days"
     worker:
         docker-image: {"in-tree": "toolchain-build"}
         max-run-time: 3600

diff --git a/taskcluster/kinds/train-backwards/kind.yml b/taskcluster/kinds/train-backwards/kind.yml
@@ -53,7 +53,6 @@ tasks:
                 - attributes
                 - run.command
         worker-type: b-largegpu-largedisk
-        expires-after: "90 days"
         worker:
             max-run-time: 2592000
             env:

diff --git a/taskcluster/kinds/train-student/kind.yml b/taskcluster/kinds/train-student/kind.yml
@@ -49,7 +49,6 @@ tasks:
                 from-parameters:
                     marian_args: training_config.marian-args.training-student
         worker-type: b-largegpu-xlargedisk
-        expires-after: "90 days"
         worker:
             max-run-time: 2592000
             env:

diff --git a/taskcluster/kinds/train-teacher/kind.yml b/taskcluster/kinds/train-teacher/kind.yml
@@ -71,7 +71,6 @@ tasks:
                     teacher-ensemble: training_config.experiment.teacher-ensemble
                     pretrained_teacher: training_config.experiment.pretrained-models.train-teacher
         worker-type: b-largegpu-xlargedisk
-        expires-after: "90 days"
         worker:
             max-run-time: 2592000
             env:

diff --git a/taskcluster/kinds/train-vocab/kind.yml b/taskcluster/kinds/train-vocab/kind.yml
@@ -44,7 +44,6 @@ tasks:
                 - attributes
                 - run.command
         worker-type: b-cpu
-        expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}
             max-run-time: 86400

diff --git a/taskcluster/kinds/translate-corpus/kind.yml b/taskcluster/kinds/translate-corpus/kind.yml
@@ -90,7 +90,6 @@ tasks:
         run-on-tasks-for: []
 
         worker-type: b-largegpu-xlargedisk
-        expires-after: "90 days"
         worker:
             max-run-time: 2592000
             artifacts:

diff --git a/taskcluster/kinds/translate-mono-src/kind.yml b/taskcluster/kinds/translate-mono-src/kind.yml
@@ -43,7 +43,6 @@ task-defaults:
             - attributes
 
     worker-type: b-largegpu
-    expires-after: "90 days"
     worker:
         max-run-time: 2592000
         artifacts:

diff --git a/taskcluster/kinds/translate-mono-trg/kind.yml b/taskcluster/kinds/translate-mono-trg/kind.yml
@@ -50,7 +50,6 @@ task-defaults:
         from-parameters: training_config.marian-args.decoding-backward
 
     worker-type: b-largegpu
-    expires-after: "90 days"
     worker:
         max-run-time: 2592000
         artifacts: