Add dvc tracking for true north pipelines

open-innovations · Jun 26, 2024 · a50389d · a50389d
1 parent 3324d84
commit a50389d
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 1 deletion.
diff --git a/...ines/truenorth/analyse_members_list.ipynb → ...nes/true-north/analyse_members_list.ipynb b/...ines/truenorth/analyse_members_list.ipynb → ...nes/true-north/analyse_members_list.ipynb
diff --git a/pipelines/truenorth/clean_members_list.ipynb → ...lines/true-north/clean_members_list.ipynb b/pipelines/truenorth/clean_members_list.ipynb → ...lines/true-north/clean_members_list.ipynb
diff --git a/pipelines/true-north/dvc.lock b/pipelines/true-north/dvc.lock
@@ -20,3 +20,31 @@ stages:
       hash: md5
       md5: 4382c75c5d97e014925b742d1616f2e0
       size: 49895
+  analyse:
+    cmd: papermill --no-progress-bar --no-report-mode analyse_members_list.ipynb ../../output/analyse_members_list.ipynb
+    deps:
+    - path: ../../working/true-north/true_north_clean.csv
+      hash: md5
+      md5: 4382c75c5d97e014925b742d1616f2e0
+      size: 49895
+    - path: analyse_members_list.ipynb
+      hash: md5
+      md5: a7d601e7ac5dad7cbe2af9e314379cb3
+      size: 48822
+    outs:
+    - path: ../../src/overview/membership/_data/cumsum.csv
+      hash: md5
+      md5: 279d9b43f0e02df2dfa2239c920fecd2
+      size: 532
+    - path: ../../src/overview/membership/_data/sector_word_counts.csv
+      hash: md5
+      md5: 6c8ace26a5f3c54414211f99c38c7e78
+      size: 205
+    - path: ../../src/overview/membership/_data/top_locations.csv
+      hash: md5
+      md5: f4b101ad1a782fb74a0622d735c7e00c
+      size: 95
+    - path: ../../src/overview/membership/_data/true_north_members_list.csv
+      hash: md5
+      md5: 444df14148705233e1c6ebc06d9454a0
+      size: 750
diff --git a/pipelines/true-north/dvc.yaml b/pipelines/true-north/dvc.yaml
@@ -0,0 +1,22 @@
+vars:
+  - TOP: ../..
+stages: 
+  clean:
+    cmd: papermill --no-progress-bar --no-report-mode clean_members_list.ipynb ${TOP}/output/clean_members_list.ipynb
+    deps: 
+      - clean_members_list.ipynb
+      - ${TOP}/working/true-north/True North June data (updated).xlsx
+      - ${TOP}/working/true-north/true north_additional_members_list_19_06_24.xlsx
+    outs:
+      - ${TOP}/working/true-north/true_north_clean.csv
+  analyse:
+    cmd: papermill --no-progress-bar --no-report-mode analyse_members_list.ipynb ${TOP}/output/analyse_members_list.ipynb
+    deps: 
+      - analyse_members_list.ipynb
+      - ${TOP}/working/true-north/true_north_clean.csv
+    outs:
+      - ${TOP}/src/overview/membership/_data/cumsum.csv
+      - ${TOP}/src/overview/membership/_data/sector_word_counts.csv
+      - ${TOP}/src/overview/membership/_data/top_locations.csv
+      - ${TOP}/src/overview/membership/_data/true_north_members_list.csv
+
diff --git a/pipelines/truenorth/northern_stars.ipynb → pipelines/true-north/northern_stars.ipynb b/pipelines/truenorth/northern_stars.ipynb → pipelines/true-north/northern_stars.ipynb
diff --git a/src/overview/membership/_data/.gitignore b/src/overview/membership/_data/.gitignore
@@ -0,0 +1,4 @@
+/cumsum.csv
+/sector_word_counts.csv
+/top_locations.csv
+/true_north_members_list.csv
diff --git a/src/overview/membership/index.vto b/src/overview/membership/index.vto
@@ -2,7 +2,7 @@
 title: Membership
 section: overview
 nicesection: True North
-updated: 2024-06-24 11:50:07
+updated: 2024-06-26 14:54:08
 ---
 
 <details>