-
Notifications
You must be signed in to change notification settings - Fork 11
/
1_fetch.yml
124 lines (108 loc) · 3.49 KB
/
1_fetch.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
target_default: 1_fetch
packages:
- dataRetrieval
- dplyr
- scipiper
- sf
- tidyr
- yaml
- aws.s3
- aws.signature
file_extensions:
- ind
sources:
- 1_fetch/src/map_utils.R
- 1_fetch/src/fetch_dv_sites.R
#- 1_fetch/src/fetch_flood_stage.R
- 1_fetch/src/fetch_inst_stage.R
- 1_fetch/src/fetch_site_locations.R
- 1_fetch/src/fetch_site_stats.R
- 1_fetch/src/fetch_dv_data.R
- 1_fetch/src/fetch_dv_historic.R
- 1_fetch/src/fetch_states.R
targets:
1_fetch:
depends:
- 1_fetch/out/sites.rds.ind
- 1_fetch/out/sites_stage.rds.ind
- 1_fetch/out/site_locations.rds.ind
- 1_fetch/out/site_stats.rds.ind
- 1_fetch/out/dv_data.rds.ind
- 1_fetch/out/dv_data_fixed_gh.rds.ind
- 1_fetch/out/dv_historic.rds
- 1_fetch/out/pre_state_boundaries_census.zip
# -- read config --
viz_config:
command: yaml.load_file('viz_config.yml')
vizDates:
command: viz_config[[I('vizDates')]]
projection:
command: viz_config[[I('projection')]]
request_limit:
command: viz_config[[I('request_limit')]]
request_limit_historic:
command: viz_config[[I('request_limit_historic')]]
request_limit_inst:
command: viz_config[[I('request_limit_inst')]]
percentiles_all:
command: viz_config[[I('percentiles')]]
percentiles:
command: percentiles_all[[I('calc')]]
# -- view polygon --
view_config:
command: viz_config[I(c('bbox', 'projection', 'width', 'height'))]
view_polygon:
command: as_view_polygon(view_config)
# -- get sites --
1_fetch/out/sites.rds.ind:
command: fetch_dv_sites(
ind_file = target_name,
dates = vizDates)
1_fetch/out/sites.rds:
command: gd_get('1_fetch/out/sites.rds.ind')
# -- get sites --
#1_fetch/out/sites_stage.rds.ind:
# command: fetch_flood_stage(
# ind_file = target_name,
# sites_ind = '1_fetch/out/sites.rds.ind')
1_fetch/out/sites_stage.rds:
command: gd_get('1_fetch/out/sites_stage.rds.ind')
# -- get site locations --
1_fetch/out/site_locations.rds.ind:
command: fetch_site_locations(
ind_file = target_name,
sites_ind = '1_fetch/out/sites.rds.ind')
1_fetch/out/site_locations.rds:
command: gd_get('1_fetch/out/site_locations.rds.ind')
# -- get dv historic data from `national-flow-observations` pipeline --
# Setup known info
s3_bucket:
command: c(I("ds-pipeline-national-flow-observations"))
1_fetch/out/dv_historic.rds:
command: save_object(
file = target_name,
object = I("20_data_munge/out/daily_flow.rds"),
bucket = s3_bucket)
# -- get dv data --
1_fetch/out/dv_data.rds.ind:
command: fetch_dv_data(
ind_file = target_name,
sites_ind = '1_fetch/out/sites.rds.ind',
dates = vizDates,
request_limit = request_limit)
1_fetch/out/dv_data.rds:
command: gd_get('1_fetch/out/dv_data.rds.ind')
# -- calculate dv stage for gages missing it using instantaneous gage height (gh) --
1_fetch/out/dv_data_fixed_gh.rds.ind:
command: fetch_inst_stage(
ind_file = target_name,
dv_data_ind = '1_fetch/out/dv_data.rds.ind',
stage_data_ind = '1_fetch/out/sites_stage.rds.ind',
request_limit = request_limit_inst)
1_fetch/out/dv_data_fixed_gh.rds:
command: gd_get('1_fetch/out/dv_data_fixed_gh.rds.ind')
# -- get state boundaries --
1_fetch/out/pre_state_boundaries_census.zip.ind:
command: fetch_states(target_name)
1_fetch/out/pre_state_boundaries_census.zip:
command: gd_get('1_fetch/out/pre_state_boundaries_census.zip.ind')