Truncated main workflow (#638)

* Add options to truncate sequences from main workflow

* Update gitignore

* Fill missing lineage/clade values before DB push

.gitignore

@@ -98,6 +98,8 @@ data_flu/**
 data_gisaid_flu/**
 data_gisaid_rsv/**
 data_genbank_rsv/**
+data_6month
+data_6month/**
 
 # Ignore server passwords
 server/htpasswd
@@ -108,7 +110,8 @@ daily_update.sh
 daily_update_new.sh
 daily_update_rsv.sh
 daily_update_sars2_genbank.sh
-daily_update_sars2_gisaid.sh
+daily_update_sars2_gisaid_full.sh
+daily_update_sars2_gisaid_6month.sh
 update_gisaid.sh
 update_genbank.sh
 filter_list.txt
@@ -143,4 +146,4 @@ data_flu_small/**
 workflow_main/notebooks/**
 
 # SnapGene - temp files
-static_data/flu/alignments/.sglock/**
+static_data/flu/alignments/.sglock/**

config/config_flu_genbank.yaml

@@ -36,6 +36,20 @@ chunk_size: 10000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1", "2", "3", "4", "5", "6", "7", "8"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_flu_gisaid.yaml

@@ -32,6 +32,20 @@ chunk_size: 10000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1", "2", "3", "4", "5", "6", "7", "8"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_flu_gisaid_dev.yaml

@@ -32,6 +32,20 @@ chunk_size: 10000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1", "2", "3", "4", "5", "6", "7", "8"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_rsv_custom.yaml

@@ -31,6 +31,20 @@ chunk_size: 100000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_rsv_genbank.yaml

@@ -36,6 +36,20 @@ chunk_size: 100000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_sars2_alpha.yaml

@@ -31,6 +31,20 @@ chunk_size: 100000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_sars2_custom.yaml

@@ -31,6 +31,20 @@ chunk_size: 100000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_sars2_genbank.yaml

@@ -35,6 +35,20 @@ chunk_size: 100000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_sars2_genbank_dev.yaml

@@ -36,6 +36,20 @@ chunk_size: 100000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1"]
 
 # Insertions or deletions with more than this difference in bases between the

config/config_sars2_gisaid.yaml

@@ -31,6 +31,20 @@ chunk_size: 100000
 #       ANALYSIS
 # --------------------
 
+# Don't process sequences prior to this date
+# Leave empty to ignore
+start_date_cutoff:
+# Don't process sequences after this date
+# Leave empty to ignore
+end_date_cutoff:
+
+# Don't process sequences after X days ago
+# Leave empty to ignore
+start_date_cutoff_days_ago:
+# Don't process sequences prior to X days ago
+# Leave empty to ignore
+end_date_cutoff_days_ago:
+
 segments: ["1"]
 
 # Insertions or deletions with more than this difference in bases between the