From aec5532f5afe0dbcea0eafe2c4c6839e7ef9e691 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 24 Jun 2024 10:23:10 +0200 Subject: [PATCH 01/13] add zaproxy --- .github/workflows/zaproxy.yml | 40 ++++++++++ ...K171200_C_EDZW_20230117120502_51362175.txt | 72 ++++++++++++++++++ ...K171800_C_EDZW_20230117180502_51662689.txt | 75 +++++++++++++++++++ tests/data3/SICG20FCBB_202308.txt | 37 +++++++++ .../WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv | 20 +++++ ...S_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv | 5 ++ ...IGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv | 5 ++ tests/data3/gts_headers_mapping.csv | 8 ++ .../cd-surface-weather-observations.yml | 69 +++++++++++++++++ .../mw-surface-weather-observations.yml | 68 +++++++++++++++++ tests/data3/metadata/station/station_list.csv | 20 +++++ 11 files changed, 419 insertions(+) create mode 100644 .github/workflows/zaproxy.yml create mode 100644 tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt create mode 100644 tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt create mode 100644 tests/data3/SICG20FCBB_202308.txt create mode 100644 tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv create mode 100644 tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv create mode 100644 tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv create mode 100644 tests/data3/gts_headers_mapping.csv create mode 100644 tests/data3/metadata/discovery/cd-surface-weather-observations.yml create mode 100644 tests/data3/metadata/discovery/mw-surface-weather-observations.yml create mode 100644 tests/data3/metadata/station/station_list.csv diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml new file mode 100644 index 000000000..91f0ee977 --- /dev/null +++ b/.github/workflows/zaproxy.yml @@ -0,0 +1,40 @@ +name: Run ZAP Baseline Scan ⚙️ + +on: [ push, pull_request ] + +jobs: + main: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: build wis2box + run: | + python3 wis2box-ctl.py build + python3 wis2box-ctl.py update + - name: start containers using tests/test.env ⚙️ + run: | + cp tests/test.env wis2box.env + python3 wis2box-ctl.py start + python3 wis2box-ctl.py status -a + sleep 30 + - name: populate stations from CSV 📡 + run: | + python3 wis2box-ctl.py execute wis2box metadata station publish-collection + - name: add Malawi synop data to the system 🇲🇼 + env: + TOPIC_HIERARCHY: mw-mw_met_centre.data.core.weather.surface-based-observations.synop + CHANNEL: origin/a/wis2/mw-mw_met_centre/data/core/weather/surface-based-observations/synop + TERRITORY: MWI + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/mw-surface-weather-observations.yml + DISCOVERY_METADATA_ID: urn:wmo:md:mw-mw_met_centre:surface-weather-observations + run: | + python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA + python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL + python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA + sleep 10 + - name: ZAP baseline Scan + uses: zaproxy/action-baseline@v0.12.0 + with: + target: 'http://localhost' + #rules_file_name: '.zap/rules.tsv' \ No newline at end of file diff --git a/tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt b/tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt new file mode 100644 index 000000000..286a4972d --- /dev/null +++ b/tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt @@ -0,0 +1,72 @@ +SMRO01 YRBK 171200 +AAXX 17121 +15015 01597 71702 10057 20036 39390 42628 50004 60021 78082 87300 333 +4/000 +55304 0//// 20643 3//// 69977 91003 91108= +15020 02597 61303 10104 20040 39783 49976 58007 60001 83570 333 4/000 +55308 +0//// 21085 3//// 60007 91005 91106= +15090 02597 72003 10109 20036 39904 49994 56010 60001 83571 333 4/000 +55301 +00256 20589 30542 60007 91005 91107= +15108 01/92 92514 11028 21028 37901 48315 50001 69941 77174 333 48013 +55300 +0//// 20270 3//// 69947 91020 911// 92727 92913 96047= +15120 02598 61904 10106 20016 39496 49978 50003 60001 83360 333 55309 +00649 +21413 30243 60007 91007 91107= +15150 02597 61604 10091 20036 39782 49991 57013 60001 86800 333 4/000 +55308 +0//// 20986 3//// 60007 91006 91109= +15170 01597 81701 10050 20031 39238 42652 53007 69961 76162 885// 333 +55300 +0//// 20206 3//// 69937 91003 91104= +15200 02598 51707 10111 20062 39840 49980 58006 60001 81140 333 4/000 +55310 +0//// 21428 3//// 60007 91010 91111= +15230 01598 70101 10089 20054 39714 40002 58007 60011 70262 87500 333 +55303 +00347 20712 30640 60007 91002 91102= +15260 05599 21502 10088 20029 39467 49998 50004 60001 7000/ 82800 333 +55309 +0//// 21214 3//// 60007 91003 91104 95000= +15280 01/90 92518 11047 21054 37334 47872 51004 60001 74143 333 49070 +55300 +0//// 20000 3//// 60007 91026 911// 92956= +15292 01597 61505 10074 20048 39713 40004 58010 60021 70182 86300 333 +55302 +0//// 20637 3//// 60007 91008 91108 95090= +15310 02998 71903 10111 20066 39942 40026 55008 60001 82038 333 4/000 +55304 +00448 20862 30571 60007 91007 91110= +15335 02597 51705 10124 20085 30028 40035 57013 60001 83530 333 55309 +0//// +21651 3//// 60007 91008 91110= +15346 01596 73401 10059 20047 39746 40035 58003 60021 76162 85570 333 +4/000 +55300 0//// 20415 3//// 69947 91002 91103= +15350 02997 32204 10103 20057 39906 40025 57020 60001 80002 333 4/000 +55308 +0//// 20913 3//// 60007 91006 91106= +15360 02997 61910 10098 20077 30030 40046 57009 60001 86070 222// +06070 20503 +333 55303 ///// 20758 3//// 60007 91011 91112 92447= +15410 01597 80302 10066 20045 39934 40030 57013 60031 70262 8457/ 333 +4/000 +55301 0//// 20600 3//// 60017 91004 91106= +15420 01597 62101 10127 20087 39912 40021 58011 69911 70262 83570 333 +55305 +0//// 20850 3//// 60007 91003 91103= +15450 01596 70203 10078 20054 39792 40025 50001 69911 71086 84530 333 +55306 +00537 21202 30592 69907 91004 91105= +15460 01997 72106 10175 20069 30002 40026 57014 69911 70282 87070 333 +4/000 +55310 0//// 21460 3//// 60007 91011 91111= +15470 02997 51902 10141 20082 39897 40019 58008 60001 85050 333 55310 +0//// +21469 3//// 60007 91003 91103= +15480 05997 42104 10146 20092 30023 40040 57015 60001 7000/ 82031 +222// 06078 +2//// 333 55309 00621 21275 30948 60007 91010 91112 92427= + diff --git a/tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt b/tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt new file mode 100644 index 000000000..4f6048c3a --- /dev/null +++ b/tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt @@ -0,0 +1,75 @@ +SMRO01 YRBK 171800 +AAXX 17181 +15015 01598 82700 10039 20026 39397 42633 58006 60022 70282 8657/ 333 +10066 +20029 31010 4/000 55300 0//// 20000 3//// 69917 91002 91104= +15020 02997 01503 10071 20018 39811 40008 51023 60002 333 10113 20024 +31012 +4/000 55300 0//// 20000 3//// 60007 91006 91106= +15090 02997 12101 10084 20048 39928 40020 51019 60002 81030 333 10122 +20059 +3/010 4/000 55300 10143 20000 30000 60007 91004 91104= +15108 01299 22710 11024 21040 37925 48339 52022 69972 73674 81540 333 +11021 +21036 3//// 48014 55300 0//// 20000 3//// 60007 91016 911// 92727 +92813 92911= +15120 02997 71303 10079 20027 39497 49984 58006 60002 87070 333 10117 +20049 +31016 55300 10080 20000 30000 60007 91006 91106= +15150 02997 01902 10058 20046 39811 40024 51021 60002 333 10117 20028 +31013 +4/000 55300 0//// 20000 3//// 60007 91004 91104= +15170 01998 23100 10001 21001 39242 42656 51002 69982 70361 82040 333 +10074 +21016 31005 55300 0//// 20000 3//// 60007 91001 91102= +15200 01598 81007 10092 20066 39784 49924 58038 69912 78081 883// 333 +10117 +20060 31014 55300 0//// 20000 3//// 69917 91011 91111= +15230 02997 71702 10086 20038 39690 49978 58010 60012 87070 333 10122 +20027 +31014 4/000 55300 10110 20000 30000 60007 91004 91105= +15260 05598 30802 10088 20031 39460 49990 58005 60002 7000/ 83500 333 +10109 +20016 31009 55300 0//// 20000 3//// 60007 91004 91105= +15280 01/90 92012 11041 21047 37349 47889 50004 60022 74143 333 11041 +21059 +3//// 49075 55300 0//// 20000 3//// 60007 91020 911// 92818 92946= +15292 02597 81505 10078 20052 39664 49953 58027 60022 883// 333 10112 +20045 +31014 55300 0//// 20000 3//// 60007 91010 91112= +15310 02997 11801 10090 20072 39953 40039 52011 60002 80001 333 10132 +20065 +31015 4/000 55300 10160 20000 30000 60007 91002 91105= +15335 02997 21804 10096 20079 30040 40047 52008 60002 82030 333 10134 +20072 +31016 4/000 55300 0//// 20000 3//// 60007 91006 91107= +15346 01597 83603 10064 20048 39737 40024 58005 60022 70262 8557/ 333 +10079 +20034 32010 4/000 55300 0//// 20000 3//// 60007 91005 91105= +15350 02997 01001 10058 20044 39916 40037 50006 60002 333 10121 20009 +31008 +4/000 55300 0//// 20000 3//// 60007 91003 91103= +15360 02998 01908 10092 20084 30042 40059 53009 60002 222// 06068 +20402 333 +10100 20087 3//// 55300 ///// 20000 3//// 60007 91009 91112 92438= +15410 02597 80302 10072 20047 39909 40004 58013 60032 8457/ 333 10080 +20035 +31009 4/000 55300 0//// 20000 3//// 60007 91004 91106= +15420 02997 60701 10057 20056 39919 40030 52002 69912 80006 333 10157 +20028 +31013 55300 0//// 20000 3//// 60007 91002 91102= +15450 02597 70706 10075 20060 39768 40001 58013 69912 83570 333 10093 +20038 +31012 55300 10119 20000 30000 60007 91009 91109= +15460 02997 41702 10126 20072 30011 40035 50003 69912 80008 333 10180 +20074 +31017 4/000 55300 0//// 20000 3//// 60007 91006 91106= +15470 02597 60503 10078 20064 39890 40015 58006 60002 82370 333 10143 +20037 +31016 55300 0//// 20000 3//// 60007 91003 91104= +15480 05997 41901 10114 20089 30035 40052 51009 60002 7000/ 80005 +222// 06074 +2//// 333 10150 20084 31015 4/000 55300 10188 20000 30000 60007 91004 +91106 +92427= + diff --git a/tests/data3/SICG20FCBB_202308.txt b/tests/data3/SICG20FCBB_202308.txt new file mode 100644 index 000000000..8de45d968 --- /dev/null +++ b/tests/data3/SICG20FCBB_202308.txt @@ -0,0 +1,37 @@ +SICG20 FCBB 030900 +AAXX 03094 +64400 42460 71004 10285 20245 30113 40133 8493/ + 333 59005 83813 81930 87363 94966 95836= +AAXX 03093 +64401 42460 60000 10266 20225 39760 40129 85230 + 333 59001 85816 83360 94919 95839= +64402 41460 82604 10226 20217 39553 40128 76164 8592/ + 333 59008 84610 81926 88460 94942 95832= +64403 42460 53602 10262 20237 39/// 40/// 85230 + 333 5//// 85816 83364 94919 95839= +64405 42460 62604 10253 20222 39530 40/// 82170 + 333 59007 82812 86360= +64406 42460 71802 10260 20228 39/// 40/// 8417/ + 333 5//// 84813 87362 94919 95839= +AAXX 03094 +64450 42460 71402 10245 20215 39780 40132 8387/ + 333 59006 81816 83623 87463 94908 95838= +AAXX 03093 +64452 42460 71402 10285 20228 39/// 40/// 8327/ + 333 5//// 83817 87362 94913 95833= +64453 42960 53204 10235 20206 39/// 48/// 83032 + 333 5//// 83360 85076= +64454 NIL= +64456 42460 52204 10264 20229 39/// 40/// 84230 + 333 5//// 84810 94919 95839= +64458 42460 60000 10257 20211 39744 40138 86280 + 333 59022 86816 84362 94919 95879= +64459 42460 72202 10255 20218 39782 40/// 8357/ + 333 59011 83613 87360= +64460 NIL= +64462 41106 81402 10205 20198 74644 886// + 333 88703== +64463 NIL= +AAXX 03094 +64461 42460 52804 10264 20232 39766 40136 83430 + 333 59006 83613 83360= \ No newline at end of file diff --git a/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv b/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv new file mode 100644 index 000000000..408a434da --- /dev/null +++ b/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv @@ -0,0 +1,20 @@ +"TOA5","Namitambo","CR300","4720","CR310-CELL200.Std.08.01","CPU:CR310_Malawi_V1R6_06072021_T3.CR300","13058","SYNOP" +"TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24" +"TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2" +"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp" +"2021-07-07 16:55:00",0,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,14,55,-15.84,35.27,806,807,93284.05,102745.5,"NAN",4,1.5,288.7,286.8,88,-1,0,-24,0,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,3.077,187.9,2.975,"None",7.16,-1,0.02758914,-24,0.02758914 +"2021-07-07 17:55:00",1,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,15,55,-15.84,35.27,806,807,93327.5,102793.3,"NAN",4,1.5,288.1,286.5,90.4,-1,0.225,-24,0.225,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,2.658,190.9,2.22,"None",4.87,-1,0.01137306,-24,0.03896223 +"2021-07-07 18:55:00",2,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,16,55,-15.84,35.27,806,807,93385.84,102857.6,"NAN",4,1.5,287.9,286.9,93.5,-1,0.433,-24,0.658,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.639,199.1,1.708,"None",3.71,-1,0,-24,0.03896223 +"2021-07-07 19:55:00",3,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,17,55,-15.84,35.27,806,807,93446.33,102924.2,162.2813,2,1.5,287.5,286.6,94,-1,0.513,-24,1.171,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.838,193.7,1.961,"None",4.29,-1,0,-24,0.03896223 +"2021-07-07 20:55:00",4,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,18,55,-15.84,35.27,806,807,93461.93,102941.4,134.4297,2,1.5,287.6,286.7,94,-1,0.518,-24,1.689,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.638,197.1,1.012,"None",2.72,-1,0,-24,0.03896223 +"2021-07-07 21:55:00",5,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,19,55,-15.84,35.27,806,807,93464.36,102944.1,78.51563,2,1.5,287.7,286.2,91,-1,0.462,-24,2.15,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.432,201.1,1.241,"None",3.25,-1,0,-24,0.03896223 +"2021-07-07 22:55:00",6,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,20,55,-15.84,35.27,806,807,93432.13,102908.6,-14.20313,7,1.5,287.6,286.2,91.5,-1,0.446,-24,2.597,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.555,195.9,1.971,"None",4.61,-1,0,-24,0.03896223 +"2021-07-07 23:55:00",7,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,21,55,-15.84,35.27,806,807,93422.84,102898.4,-39.09375,7,1.5,287.4,286.1,91.9,-1,0.286,-24,2.882,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.552,200.2,1.194,"None",3.24,-1,0,-24,0.03896223 +"2021-07-08 00:55:00",8,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,22,55,-15.84,35.27,806,807,93388.43,102860.5,-75.92969,7,1.5,287.3,286,91.7,-1,0.514,-24,3.396,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.493,194.7,1.571,"None",3.18,-1,0,-24,0.03896223 +"2021-07-08 01:55:00",9,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,23,55,-15.84,35.27,806,807,93347.19,102815.1,-84.9375,7,1.5,286.7,285.6,93.1,-1,0.433,-24,3.828,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.364,195.7,1.334,"None",2.75,-1,0,-24,0.03896223 +"2021-07-08 02:55:00",10,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,0,55,-15.84,35.27,806,807,93310.4,102774.6,-112.4375,7,1.5,286.8,285.7,93.4,-1,0.425,-24,4.254,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.059,205.9,1.232,"None",3.84,-1,0,-24,0.03896223 +"2021-07-08 03:55:00",11,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,1,55,-15.84,35.27,806,807,93307.27,102771.1,-81.15625,7,1.5,286.5,284.8,89.1,-1,0.466,-24,4.72,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.044,236.2,1.536,"None",3.88,-1,0,-24,0.03896223 +"2021-07-08 04:55:00",12,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,2,55,-15.84,35.27,806,807,93339.49,102806.6,-7.695313,7,1.5,286.8,284.2,84.4,-1,0.513,-24,5.233,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.331,251.6,1.173,"None",3.17,-1,0,-24,0.03896223 +"2021-07-08 05:55:00",13,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,3,55,-15.84,35.27,806,807,93404.13,102877.8,93.73438,2,1.5,286.1,284.8,91.8,-1,0.507,-24,5.741,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,0.964,212,0.86,"None",2.58,-1,0,-24,0.03896223 +"2021-07-08 06:55:00",14,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,4,55,-15.84,35.27,806,807,93466.27,102946.3,158.9922,2,1.5,286.5,284.6,88.1,-1,0.071,-24,5.812,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,0.899,229.2,0.909,"None",1.98,-1,"NAN",-24,"NAN" +"2021-07-08 07:55:00",15,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,5,55,-15.84,35.27,806,807,93531.97,103018.5,192.4766,2,1.5,289.5,285.2,75.59,-1,0.453,-24,6.265,1.5,-1,0,-24,0,289.6,269.4,2,"0",2,-10,1.202,219.2,1.161,"None",3.47,-1,0.8832501,-24,"NAN" diff --git a/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv b/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv new file mode 100644 index 000000000..ed30d1d75 --- /dev/null +++ b/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv @@ -0,0 +1,5 @@ +"TOA5","Namitambo","CR300","4720","CR310-CELL200.Std.08.01","CPU:CR310_Malawi_V1R6_06072021_T3.CR300","13058","SYNOP" +"TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24" +"TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2" +"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp" +"2021-11-18 11:55:00",414,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,11,18,9,55,-15.84,35.27,806,807,92434.89,101809.4,-195.9453,7,1.5,307.9,287.7,29.88,-1,24.93,-24,102.5,1.5,-1,0,-24,0,308.1,300.8,2,"0",2,-10,2.084,67.12,2.198,"None",5.72,-1,3715672,-24,1.335E+07 diff --git a/tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv b/tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv new file mode 100644 index 000000000..8ccad38db --- /dev/null +++ b/tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv @@ -0,0 +1,5 @@ +"TOA5","Namitambo","CR300","4720","CR310-CELL200.Std.08.01","CPU:CR310_Malawi_V1R6_06072021_T3.CR300","13058","SYNOP" +"TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24" +"TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2" +"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp" +"2021-11-18 11:55:00",2328,6,"0-454-2-AWSTOLEZA","Toleza Farm","0",2021,11,18,9,55,-14.95,34.96,764,765,93868.82,102865.8,-229.2422,7,1.5,308.5,288.8,30.92,-1,0.29,-24,1.446,1.5,-1,0,-24,0,310.1,303.8,2,"0",2,-10,1.487,50.36,1.08,"None",2.55,-1,3.075586,-24,11.8543 diff --git a/tests/data3/gts_headers_mapping.csv b/tests/data3/gts_headers_mapping.csv new file mode 100644 index 000000000..e2a240054 --- /dev/null +++ b/tests/data3/gts_headers_mapping.csv @@ -0,0 +1,8 @@ +string_in_filepath,ttaaii,cccc +ISIN21DAMM,ISIN21,DAMM +ISMD01LIIB,ISMD01,LIBB +ISMD02LIIB,ISMD02,LIBB +SMRO01YRBK,SMRO01,YRBK +IOBH01LFPW,IOBH01,LFPW +ISSA05LFPW,ISSA05,LFPW +SICG20FCBB,SICG20,FCBB \ No newline at end of file diff --git a/tests/data3/metadata/discovery/cd-surface-weather-observations.yml b/tests/data3/metadata/discovery/cd-surface-weather-observations.yml new file mode 100644 index 000000000..76dcda288 --- /dev/null +++ b/tests/data3/metadata/discovery/cd-surface-weather-observations.yml @@ -0,0 +1,69 @@ +wis2box: + retention: P180D + topic_hierarchy: cd-brazza_met_centre/data/core/weather/surface-based-observations/synop + country: cog + centre_id: cd-brazza_met_centre + data_mappings: + plugins: + txt: + - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR + notify: true + file-pattern: '^.*_(\d{4})(\d{2}).*\.txt$' + bufr4: + - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON + buckets: + - ${WIS2BOX_STORAGE_PUBLIC} + file-pattern: '^.*\.bufr4$' + +mcf: + version: 1.0 + +metadata: + identifier: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations + hierarchylevel: dataset + +identification: + title: Surface weather observations from Republic of Congo + abstract: Surface weather observations from Republic of Congo + dates: + creation: 2023-03-26 + keywords: + default: + keywords: + - surface weather + - temperature + - observations + wmo: + keywords: + - weather + keywords_type: theme + vocabulary: + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://codes.wmo.int/topic-hierarchy/earth-system-discipline + extents: + spatial: + - bbox: [11.0937728207,-5.03798674888,18.4530652198,3.72819651938] + crs: 4326 + temporal: + - begin: 2023-03-26 + end: null + resolution: P1H + url: https://example.org/malawi-surface-weather-observations + wmo_data_policy: core + +contact: + host: + organization: Congo National Meteorological Service (DMN) + url: https://www.dirmet.cg + individualname: Firstname Lastname + positionname: Position Name + phone: null + fax: null + address: P.O. Box 208 + city: Brazzaville + administrativearea: null + postalcode: null + country: Republic of Congo + email: you@example.com + hoursofservice: 0700h - 1500h UTC + contactinstructions: email diff --git a/tests/data3/metadata/discovery/mw-surface-weather-observations.yml b/tests/data3/metadata/discovery/mw-surface-weather-observations.yml new file mode 100644 index 000000000..bcb14c489 --- /dev/null +++ b/tests/data3/metadata/discovery/mw-surface-weather-observations.yml @@ -0,0 +1,68 @@ +wis2box: + retention: P30D + topic_hierarchy: mw-mw_met_centre.data.core.weather.surface-based-observations.synop + country: mwi + centre_id: mw-mw_met_centre + data_mappings: + plugins: + csv: + - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR + template: CampbellAfrica-v1-template + notify: true + file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.csv$' + bufr4: + - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON + file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$' + +mcf: + version: 1.0 + +metadata: + identifier: urn:wmo:md:mw-mw_met_centre:surface-weather-observations + hierarchylevel: dataset + +identification: + title: Surface weather observations from Malawi + abstract: Surface weather observations from Malawi + dates: + creation: 2021-11-29 + keywords: + default: + keywords: + - surface weather + - temperature + - observations + wmo: + keywords: + - weather + keywords_type: theme + vocabulary: + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline + extents: + spatial: + - bbox: [32.6881653175,-16.8012997372,35.7719047381,-9.23059905359] + crs: 4326 + temporal: + - begin: 2021-11-29 + end: null + resolution: P1H + url: https://example.org/malawi-surface-weather-observations + wmo_data_policy: core + +contact: + host: + organization: Department of Climate Change and Meteorologial Services (DCCMS) + url: https://www.metmalawi.gov.mw + individualname: Firstname Lastname + positionname: Position Name + phone: "+2651822014" + fax: "+2651822215" + address: P.O. Box 1808 + city: Blantyre + administrativearea: Blantyre District + postalcode: M3H 5T4 + country: Malawi + email: you@example.org + hoursofservice: 0700h - 1500h UTC + contactinstructions: email diff --git a/tests/data3/metadata/station/station_list.csv b/tests/data3/metadata/station/station_list.csv new file mode 100644 index 000000000..f4e3f38b8 --- /dev/null +++ b/tests/data3/metadata/station/station_list.csv @@ -0,0 +1,20 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +NAMITAMBO,0-454-2-AWSNAMITAMBO,AWSNAMITAMBO,landFixed,-15.84052,35.27428,806,,MWI,africa +TOLEZA,0-454-2-AWSTOLEZA,AWSTOLEZA,landFixed,-14.948,34.955,764,,MWI,africa +POINTE-NOIRE,0-20000-0-64400,64400,landFixed,-4.8045,11.8817,18.00,19.20,COD,africa +DOLISIE,0-20000-0-64401,64401,landFixed,-4.2119,12.6680,330.00,331.25,COD,africa +MOUYONDZI,0-20000-0-64402,64402,landFixed,-4.0000,13.9500,509.00,510.20,COD,africa +MAKABANA,0-20000-0-64403,64403,landFixed,-3.4833,12.6166,160.00,162.20,COD,africa +SIBITI,0-20000-0-64405,64405,landFixed,-3.6845,13.3698,530.00,531.40,COD,africa +BRAZZAVILLE /MAYA-MAYA,0-20000-0-64450,64450,landFixed,-4.2636,15.2429,314.27,315.6,COD,africa +M'POUYA,0-20000-0-64452,64452,landFixed,-2.6166,16.2166,311.00,312.30,COD,africa +DJAMBALA,0-20000-0-64453,64453,landFixed,-2.5333,14.7666,789.00,790.20,COD,africa +GAMBOMA,0-20000-0-64454,64454,landFixed,-1.9000,15.8500,476.00,477.20,COD,africa +MAKOUA,0-20000-0-64456,64456,landFixed,-0.0166,15.6500,394.00,396.20,COD,africa +OUESSO,0-20000-0-64458,64458,landFixed,1.6125,16.0402,352.00,353.10,COD,africa +IMPFONDO,0-20000-0-64459,64459,landFixed,1.5928,18.0540,324.00,325.20,COD,africa +SOUANKE,0-20000-0-64460,64460,landFixed,2.0666,14.0333,549.00,550.20,COD,africa +KELLE,0-20000-0-64462,64462,landFixed,0.0829,14.5329,408.00,409.30,COD,africa +EWO,0-20000-0-64463,64463,landFixed,-0.8541,14.8034,479.00,480.20,COD,africa +N'KAYI,0-20000-0-64406,64406,landFixed,-4.2218,13.2856,165.00,166.40,COD,africa +OLLOMBO,0-20000-0-64461,64461,landFixed,-1.2257,15.9166,327.05,328.45,COD,africa \ No newline at end of file From 8d1e2d297c2c4180d7ecae6057418f404995a1d3 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 24 Jun 2024 10:27:05 +0200 Subject: [PATCH 02/13] build and start rolled into one step --- .github/workflows/zaproxy.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index 91f0ee977..36eaee3f3 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -8,16 +8,14 @@ jobs: steps: - uses: actions/checkout@v2 - - name: build wis2box - run: | - python3 wis2box-ctl.py build - python3 wis2box-ctl.py update - - name: start containers using tests/test.env ⚙️ + - name: build and start containers using tests/test.env ⚙️ run: | cp tests/test.env wis2box.env + python3 wis2box-ctl.py build python3 wis2box-ctl.py start python3 wis2box-ctl.py status -a sleep 30 + python3 wis2box-ctl.py status -a - name: populate stations from CSV 📡 run: | python3 wis2box-ctl.py execute wis2box metadata station publish-collection From 2b358364fd12fe228e227adc6dd1dc1b1a069fc3 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 24 Jun 2024 14:33:09 +0200 Subject: [PATCH 03/13] add rules.tsv --- .github/workflows/zaproxy.yml | 4 ++-- .zap/rules.tsv | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 .zap/rules.tsv diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index 36eaee3f3..74d0a5629 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -1,6 +1,6 @@ name: Run ZAP Baseline Scan ⚙️ -on: [ push, pull_request ] +on: [ push ] jobs: main: @@ -35,4 +35,4 @@ jobs: uses: zaproxy/action-baseline@v0.12.0 with: target: 'http://localhost' - #rules_file_name: '.zap/rules.tsv' \ No newline at end of file + rules_file_name: '.zap/rules.tsv' \ No newline at end of file diff --git a/.zap/rules.tsv b/.zap/rules.tsv new file mode 100644 index 000000000..56741ad20 --- /dev/null +++ b/.zap/rules.tsv @@ -0,0 +1,19 @@ +10202 IGNORE Absence of Anti-CSRF Tokens Medium +10038 IGNORE Content Security Policy (CSP) Header Not Set Medium +10098 IGNORE Cross-Domain Misconfiguration Medium +10020 IGNORE Missing Anti-clickjacking Header Medium +90003 IGNORE Sub Resource Integrity Attribute Missing Medium +90022 IGNORE Application Error Disclosure Medium +10054 IGNORE Cookie with SameSite Attribute None Low +10017 IGNORE Cross-Domain JavaScript Source File Inclusion Low +10023 IGNORE Information Disclosure - Debug Error Messages Low +10063 IGNORE Permissions Policy Header Not Set Low +10037 IGNORE "Server Leaks Information via ""X-Powered-By"" HTTP Response Header Field(s)" Low +10096 IGNORE Timestamp Disclosure - Unix Low +10021 IGNORE X-Content-Type-Options Header Missing Low +10027 IGNORE Information Disclosure - Suspicious Comments Informational +90033 IGNORE Loosely Scoped Cookie Informational +10109 IGNORE Modern Web Application Informational +10049 IGNORE Non-Storable Content Informational +10112 IGNORE Session Management Response Identified Informational +10049 IGNORE Storable and Cacheable Content Informational From 2d47ee58f5302a5fad841af77a4820401d89491a Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 24 Jun 2024 16:41:01 +0200 Subject: [PATCH 04/13] don't fail on warnings, ignore other low risk --- .github/workflows/zaproxy.yml | 4 +++- .zap/rules.tsv | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index 74d0a5629..63be53b24 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -35,4 +35,6 @@ jobs: uses: zaproxy/action-baseline@v0.12.0 with: target: 'http://localhost' - rules_file_name: '.zap/rules.tsv' \ No newline at end of file + rules_file_name: '.zap/rules.tsv' + cmd_options: '-I' + allow_issue_writing: 'false' \ No newline at end of file diff --git a/.zap/rules.tsv b/.zap/rules.tsv index 56741ad20..2e58035e8 100644 --- a/.zap/rules.tsv +++ b/.zap/rules.tsv @@ -17,3 +17,6 @@ 10049 IGNORE Non-Storable Content Informational 10112 IGNORE Session Management Response Identified Informational 10049 IGNORE Storable and Cacheable Content Informational +10009 IGNORE In Page Banner Information Leak Low +10036 IGNORE "Server Leaks Version Information via ""Server"" HTTP Response Header Field" Low +10110 IGNORE Dangerous JS Functions Low From 995d979d10f7c983040e46afdc7d7da923e0a006 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 24 Jun 2024 17:00:17 +0200 Subject: [PATCH 05/13] remove rules.tsv --- .zap/rules.tsv | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .zap/rules.tsv diff --git a/.zap/rules.tsv b/.zap/rules.tsv deleted file mode 100644 index 2e58035e8..000000000 --- a/.zap/rules.tsv +++ /dev/null @@ -1,22 +0,0 @@ -10202 IGNORE Absence of Anti-CSRF Tokens Medium -10038 IGNORE Content Security Policy (CSP) Header Not Set Medium -10098 IGNORE Cross-Domain Misconfiguration Medium -10020 IGNORE Missing Anti-clickjacking Header Medium -90003 IGNORE Sub Resource Integrity Attribute Missing Medium -90022 IGNORE Application Error Disclosure Medium -10054 IGNORE Cookie with SameSite Attribute None Low -10017 IGNORE Cross-Domain JavaScript Source File Inclusion Low -10023 IGNORE Information Disclosure - Debug Error Messages Low -10063 IGNORE Permissions Policy Header Not Set Low -10037 IGNORE "Server Leaks Information via ""X-Powered-By"" HTTP Response Header Field(s)" Low -10096 IGNORE Timestamp Disclosure - Unix Low -10021 IGNORE X-Content-Type-Options Header Missing Low -10027 IGNORE Information Disclosure - Suspicious Comments Informational -90033 IGNORE Loosely Scoped Cookie Informational -10109 IGNORE Modern Web Application Informational -10049 IGNORE Non-Storable Content Informational -10112 IGNORE Session Management Response Identified Informational -10049 IGNORE Storable and Cacheable Content Informational -10009 IGNORE In Page Banner Information Leak Low -10036 IGNORE "Server Leaks Version Information via ""Server"" HTTP Response Header Field" Low -10110 IGNORE Dangerous JS Functions Low From d111f1a2c01e123543c3192c673724e524490d9b Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 24 Jun 2024 17:16:19 +0200 Subject: [PATCH 06/13] put rules back -and- fail on warnings --- .github/workflows/zaproxy.yml | 1 - .zap/rules.tsv | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 .zap/rules.tsv diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index 63be53b24..d29660eb9 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -36,5 +36,4 @@ jobs: with: target: 'http://localhost' rules_file_name: '.zap/rules.tsv' - cmd_options: '-I' allow_issue_writing: 'false' \ No newline at end of file diff --git a/.zap/rules.tsv b/.zap/rules.tsv new file mode 100644 index 000000000..2e58035e8 --- /dev/null +++ b/.zap/rules.tsv @@ -0,0 +1,22 @@ +10202 IGNORE Absence of Anti-CSRF Tokens Medium +10038 IGNORE Content Security Policy (CSP) Header Not Set Medium +10098 IGNORE Cross-Domain Misconfiguration Medium +10020 IGNORE Missing Anti-clickjacking Header Medium +90003 IGNORE Sub Resource Integrity Attribute Missing Medium +90022 IGNORE Application Error Disclosure Medium +10054 IGNORE Cookie with SameSite Attribute None Low +10017 IGNORE Cross-Domain JavaScript Source File Inclusion Low +10023 IGNORE Information Disclosure - Debug Error Messages Low +10063 IGNORE Permissions Policy Header Not Set Low +10037 IGNORE "Server Leaks Information via ""X-Powered-By"" HTTP Response Header Field(s)" Low +10096 IGNORE Timestamp Disclosure - Unix Low +10021 IGNORE X-Content-Type-Options Header Missing Low +10027 IGNORE Information Disclosure - Suspicious Comments Informational +90033 IGNORE Loosely Scoped Cookie Informational +10109 IGNORE Modern Web Application Informational +10049 IGNORE Non-Storable Content Informational +10112 IGNORE Session Management Response Identified Informational +10049 IGNORE Storable and Cacheable Content Informational +10009 IGNORE In Page Banner Information Leak Low +10036 IGNORE "Server Leaks Version Information via ""Server"" HTTP Response Header Field" Low +10110 IGNORE Dangerous JS Functions Low From 3d97f1639aa7c0abc2db430ffe977da024c9b50c Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 26 Jun 2024 14:47:54 +0200 Subject: [PATCH 07/13] ZAP baseline Scan on wis2box-webapp --- .github/workflows/zaproxy.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index d29660eb9..b64dabc11 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -31,9 +31,17 @@ jobs: python3 wis2box-ctl.py execute wis2box metadata station add-topic --territory-name $TERRITORY $CHANNEL python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA sleep 10 - - name: ZAP baseline Scan + - name: ZAP baseline Scan on UI 🕵️‍♂️ uses: zaproxy/action-baseline@v0.12.0 with: target: 'http://localhost' rules_file_name: '.zap/rules.tsv' + allow_issue_writing: 'false' + - name: ZAP baseline Scan on wis2box-webapp 🕵️‍♂️ + uses: zaproxy/action-baseline@v0.12.0 + env: + ZAP_AUTH_HEADER_VALUE: "Basic d2lzMmJveC11c2VyOndpczJib3h0ZXN0MTIzCg==" # wis2box-user:wis2boxtest123, testing only + with: + target: 'http://localhost/wis2box-webapp' + rules_file_name: '.zap/rules.tsv' allow_issue_writing: 'false' \ No newline at end of file From 85d825af9e8de36394c0991e986fed29a182db18 Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 26 Jun 2024 15:17:37 +0200 Subject: [PATCH 08/13] add header --- .github/workflows/zaproxy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index b64dabc11..fa0712735 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -40,7 +40,8 @@ jobs: - name: ZAP baseline Scan on wis2box-webapp 🕵️‍♂️ uses: zaproxy/action-baseline@v0.12.0 env: - ZAP_AUTH_HEADER_VALUE: "Basic d2lzMmJveC11c2VyOndpczJib3h0ZXN0MTIzCg==" # wis2box-user:wis2boxtest123, testing only + ZAP_AUTH_HEADER_VALUE: "Basic d2lzMmJveC11c2VyOndpczJib3h0ZXN0MTIzCg==" + ZAP_AUTH_HEADER: "Authorization" with: target: 'http://localhost/wis2box-webapp' rules_file_name: '.zap/rules.tsv' From dbbb97524bbd4ec801686d5980cc984798d6caaa Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 26 Jun 2024 15:38:18 +0200 Subject: [PATCH 09/13] fix auth --- .github/workflows/zaproxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index fa0712735..313bc2d9c 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -40,7 +40,7 @@ jobs: - name: ZAP baseline Scan on wis2box-webapp 🕵️‍♂️ uses: zaproxy/action-baseline@v0.12.0 env: - ZAP_AUTH_HEADER_VALUE: "Basic d2lzMmJveC11c2VyOndpczJib3h0ZXN0MTIzCg==" + ZAP_AUTH_HEADER_VALUE: "Basic d2lzMmJveC11c2VyOndpczJib3h0ZXN0MTIz" ZAP_AUTH_HEADER: "Authorization" with: target: 'http://localhost/wis2box-webapp' From b27a8287aa0a4b6d614fa21f904658f2c23ba7b1 Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 26 Jun 2024 15:54:51 +0200 Subject: [PATCH 10/13] check that action fails --- .github/workflows/zaproxy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml index 313bc2d9c..a04fb9652 100644 --- a/.github/workflows/zaproxy.yml +++ b/.github/workflows/zaproxy.yml @@ -37,6 +37,7 @@ jobs: target: 'http://localhost' rules_file_name: '.zap/rules.tsv' allow_issue_writing: 'false' + fail_action: 'true' - name: ZAP baseline Scan on wis2box-webapp 🕵️‍♂️ uses: zaproxy/action-baseline@v0.12.0 env: @@ -45,4 +46,5 @@ jobs: with: target: 'http://localhost/wis2box-webapp' rules_file_name: '.zap/rules.tsv' - allow_issue_writing: 'false' \ No newline at end of file + allow_issue_writing: 'false' + fail_action: 'true' \ No newline at end of file From 54e8f7f9be252bc32f44d80175f38a658379c4b9 Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 26 Jun 2024 17:15:02 +0200 Subject: [PATCH 11/13] remove accidental commit --- ...K171200_C_EDZW_20230117120502_51362175.txt | 72 ------------------ ...K171800_C_EDZW_20230117180502_51662689.txt | 75 ------------------- tests/data3/SICG20FCBB_202308.txt | 37 --------- .../WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv | 20 ----- ...S_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv | 5 -- ...IGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv | 5 -- tests/data3/gts_headers_mapping.csv | 8 -- .../cd-surface-weather-observations.yml | 69 ----------------- .../mw-surface-weather-observations.yml | 68 ----------------- tests/data3/metadata/station/station_list.csv | 20 ----- 10 files changed, 379 deletions(-) delete mode 100644 tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt delete mode 100644 tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt delete mode 100644 tests/data3/SICG20FCBB_202308.txt delete mode 100644 tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv delete mode 100644 tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv delete mode 100644 tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv delete mode 100644 tests/data3/gts_headers_mapping.csv delete mode 100644 tests/data3/metadata/discovery/cd-surface-weather-observations.yml delete mode 100644 tests/data3/metadata/discovery/mw-surface-weather-observations.yml delete mode 100644 tests/data3/metadata/station/station_list.csv diff --git a/tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt b/tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt deleted file mode 100644 index 286a4972d..000000000 --- a/tests/data3/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt +++ /dev/null @@ -1,72 +0,0 @@ -SMRO01 YRBK 171200 -AAXX 17121 -15015 01597 71702 10057 20036 39390 42628 50004 60021 78082 87300 333 -4/000 -55304 0//// 20643 3//// 69977 91003 91108= -15020 02597 61303 10104 20040 39783 49976 58007 60001 83570 333 4/000 -55308 -0//// 21085 3//// 60007 91005 91106= -15090 02597 72003 10109 20036 39904 49994 56010 60001 83571 333 4/000 -55301 -00256 20589 30542 60007 91005 91107= -15108 01/92 92514 11028 21028 37901 48315 50001 69941 77174 333 48013 -55300 -0//// 20270 3//// 69947 91020 911// 92727 92913 96047= -15120 02598 61904 10106 20016 39496 49978 50003 60001 83360 333 55309 -00649 -21413 30243 60007 91007 91107= -15150 02597 61604 10091 20036 39782 49991 57013 60001 86800 333 4/000 -55308 -0//// 20986 3//// 60007 91006 91109= -15170 01597 81701 10050 20031 39238 42652 53007 69961 76162 885// 333 -55300 -0//// 20206 3//// 69937 91003 91104= -15200 02598 51707 10111 20062 39840 49980 58006 60001 81140 333 4/000 -55310 -0//// 21428 3//// 60007 91010 91111= -15230 01598 70101 10089 20054 39714 40002 58007 60011 70262 87500 333 -55303 -00347 20712 30640 60007 91002 91102= -15260 05599 21502 10088 20029 39467 49998 50004 60001 7000/ 82800 333 -55309 -0//// 21214 3//// 60007 91003 91104 95000= -15280 01/90 92518 11047 21054 37334 47872 51004 60001 74143 333 49070 -55300 -0//// 20000 3//// 60007 91026 911// 92956= -15292 01597 61505 10074 20048 39713 40004 58010 60021 70182 86300 333 -55302 -0//// 20637 3//// 60007 91008 91108 95090= -15310 02998 71903 10111 20066 39942 40026 55008 60001 82038 333 4/000 -55304 -00448 20862 30571 60007 91007 91110= -15335 02597 51705 10124 20085 30028 40035 57013 60001 83530 333 55309 -0//// -21651 3//// 60007 91008 91110= -15346 01596 73401 10059 20047 39746 40035 58003 60021 76162 85570 333 -4/000 -55300 0//// 20415 3//// 69947 91002 91103= -15350 02997 32204 10103 20057 39906 40025 57020 60001 80002 333 4/000 -55308 -0//// 20913 3//// 60007 91006 91106= -15360 02997 61910 10098 20077 30030 40046 57009 60001 86070 222// -06070 20503 -333 55303 ///// 20758 3//// 60007 91011 91112 92447= -15410 01597 80302 10066 20045 39934 40030 57013 60031 70262 8457/ 333 -4/000 -55301 0//// 20600 3//// 60017 91004 91106= -15420 01597 62101 10127 20087 39912 40021 58011 69911 70262 83570 333 -55305 -0//// 20850 3//// 60007 91003 91103= -15450 01596 70203 10078 20054 39792 40025 50001 69911 71086 84530 333 -55306 -00537 21202 30592 69907 91004 91105= -15460 01997 72106 10175 20069 30002 40026 57014 69911 70282 87070 333 -4/000 -55310 0//// 21460 3//// 60007 91011 91111= -15470 02997 51902 10141 20082 39897 40019 58008 60001 85050 333 55310 -0//// -21469 3//// 60007 91003 91103= -15480 05997 42104 10146 20092 30023 40040 57015 60001 7000/ 82031 -222// 06078 -2//// 333 55309 00621 21275 30948 60007 91010 91112 92427= - diff --git a/tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt b/tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt deleted file mode 100644 index 4f6048c3a..000000000 --- a/tests/data3/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt +++ /dev/null @@ -1,75 +0,0 @@ -SMRO01 YRBK 171800 -AAXX 17181 -15015 01598 82700 10039 20026 39397 42633 58006 60022 70282 8657/ 333 -10066 -20029 31010 4/000 55300 0//// 20000 3//// 69917 91002 91104= -15020 02997 01503 10071 20018 39811 40008 51023 60002 333 10113 20024 -31012 -4/000 55300 0//// 20000 3//// 60007 91006 91106= -15090 02997 12101 10084 20048 39928 40020 51019 60002 81030 333 10122 -20059 -3/010 4/000 55300 10143 20000 30000 60007 91004 91104= -15108 01299 22710 11024 21040 37925 48339 52022 69972 73674 81540 333 -11021 -21036 3//// 48014 55300 0//// 20000 3//// 60007 91016 911// 92727 -92813 92911= -15120 02997 71303 10079 20027 39497 49984 58006 60002 87070 333 10117 -20049 -31016 55300 10080 20000 30000 60007 91006 91106= -15150 02997 01902 10058 20046 39811 40024 51021 60002 333 10117 20028 -31013 -4/000 55300 0//// 20000 3//// 60007 91004 91104= -15170 01998 23100 10001 21001 39242 42656 51002 69982 70361 82040 333 -10074 -21016 31005 55300 0//// 20000 3//// 60007 91001 91102= -15200 01598 81007 10092 20066 39784 49924 58038 69912 78081 883// 333 -10117 -20060 31014 55300 0//// 20000 3//// 69917 91011 91111= -15230 02997 71702 10086 20038 39690 49978 58010 60012 87070 333 10122 -20027 -31014 4/000 55300 10110 20000 30000 60007 91004 91105= -15260 05598 30802 10088 20031 39460 49990 58005 60002 7000/ 83500 333 -10109 -20016 31009 55300 0//// 20000 3//// 60007 91004 91105= -15280 01/90 92012 11041 21047 37349 47889 50004 60022 74143 333 11041 -21059 -3//// 49075 55300 0//// 20000 3//// 60007 91020 911// 92818 92946= -15292 02597 81505 10078 20052 39664 49953 58027 60022 883// 333 10112 -20045 -31014 55300 0//// 20000 3//// 60007 91010 91112= -15310 02997 11801 10090 20072 39953 40039 52011 60002 80001 333 10132 -20065 -31015 4/000 55300 10160 20000 30000 60007 91002 91105= -15335 02997 21804 10096 20079 30040 40047 52008 60002 82030 333 10134 -20072 -31016 4/000 55300 0//// 20000 3//// 60007 91006 91107= -15346 01597 83603 10064 20048 39737 40024 58005 60022 70262 8557/ 333 -10079 -20034 32010 4/000 55300 0//// 20000 3//// 60007 91005 91105= -15350 02997 01001 10058 20044 39916 40037 50006 60002 333 10121 20009 -31008 -4/000 55300 0//// 20000 3//// 60007 91003 91103= -15360 02998 01908 10092 20084 30042 40059 53009 60002 222// 06068 -20402 333 -10100 20087 3//// 55300 ///// 20000 3//// 60007 91009 91112 92438= -15410 02597 80302 10072 20047 39909 40004 58013 60032 8457/ 333 10080 -20035 -31009 4/000 55300 0//// 20000 3//// 60007 91004 91106= -15420 02997 60701 10057 20056 39919 40030 52002 69912 80006 333 10157 -20028 -31013 55300 0//// 20000 3//// 60007 91002 91102= -15450 02597 70706 10075 20060 39768 40001 58013 69912 83570 333 10093 -20038 -31012 55300 10119 20000 30000 60007 91009 91109= -15460 02997 41702 10126 20072 30011 40035 50003 69912 80008 333 10180 -20074 -31017 4/000 55300 0//// 20000 3//// 60007 91006 91106= -15470 02597 60503 10078 20064 39890 40015 58006 60002 82370 333 10143 -20037 -31016 55300 0//// 20000 3//// 60007 91003 91104= -15480 05997 41901 10114 20089 30035 40052 51009 60002 7000/ 80005 -222// 06074 -2//// 333 10150 20084 31015 4/000 55300 10188 20000 30000 60007 91004 -91106 -92427= - diff --git a/tests/data3/SICG20FCBB_202308.txt b/tests/data3/SICG20FCBB_202308.txt deleted file mode 100644 index 8de45d968..000000000 --- a/tests/data3/SICG20FCBB_202308.txt +++ /dev/null @@ -1,37 +0,0 @@ -SICG20 FCBB 030900 -AAXX 03094 -64400 42460 71004 10285 20245 30113 40133 8493/ - 333 59005 83813 81930 87363 94966 95836= -AAXX 03093 -64401 42460 60000 10266 20225 39760 40129 85230 - 333 59001 85816 83360 94919 95839= -64402 41460 82604 10226 20217 39553 40128 76164 8592/ - 333 59008 84610 81926 88460 94942 95832= -64403 42460 53602 10262 20237 39/// 40/// 85230 - 333 5//// 85816 83364 94919 95839= -64405 42460 62604 10253 20222 39530 40/// 82170 - 333 59007 82812 86360= -64406 42460 71802 10260 20228 39/// 40/// 8417/ - 333 5//// 84813 87362 94919 95839= -AAXX 03094 -64450 42460 71402 10245 20215 39780 40132 8387/ - 333 59006 81816 83623 87463 94908 95838= -AAXX 03093 -64452 42460 71402 10285 20228 39/// 40/// 8327/ - 333 5//// 83817 87362 94913 95833= -64453 42960 53204 10235 20206 39/// 48/// 83032 - 333 5//// 83360 85076= -64454 NIL= -64456 42460 52204 10264 20229 39/// 40/// 84230 - 333 5//// 84810 94919 95839= -64458 42460 60000 10257 20211 39744 40138 86280 - 333 59022 86816 84362 94919 95879= -64459 42460 72202 10255 20218 39782 40/// 8357/ - 333 59011 83613 87360= -64460 NIL= -64462 41106 81402 10205 20198 74644 886// - 333 88703== -64463 NIL= -AAXX 03094 -64461 42460 52804 10264 20232 39766 40136 83430 - 333 59006 83613 83360= \ No newline at end of file diff --git a/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv b/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv deleted file mode 100644 index 408a434da..000000000 --- a/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-07-07.csv +++ /dev/null @@ -1,20 +0,0 @@ -"TOA5","Namitambo","CR300","4720","CR310-CELL200.Std.08.01","CPU:CR310_Malawi_V1R6_06072021_T3.CR300","13058","SYNOP" -"TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24" -"TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2" -"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp" -"2021-07-07 16:55:00",0,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,14,55,-15.84,35.27,806,807,93284.05,102745.5,"NAN",4,1.5,288.7,286.8,88,-1,0,-24,0,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,3.077,187.9,2.975,"None",7.16,-1,0.02758914,-24,0.02758914 -"2021-07-07 17:55:00",1,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,15,55,-15.84,35.27,806,807,93327.5,102793.3,"NAN",4,1.5,288.1,286.5,90.4,-1,0.225,-24,0.225,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,2.658,190.9,2.22,"None",4.87,-1,0.01137306,-24,0.03896223 -"2021-07-07 18:55:00",2,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,16,55,-15.84,35.27,806,807,93385.84,102857.6,"NAN",4,1.5,287.9,286.9,93.5,-1,0.433,-24,0.658,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.639,199.1,1.708,"None",3.71,-1,0,-24,0.03896223 -"2021-07-07 19:55:00",3,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,17,55,-15.84,35.27,806,807,93446.33,102924.2,162.2813,2,1.5,287.5,286.6,94,-1,0.513,-24,1.171,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.838,193.7,1.961,"None",4.29,-1,0,-24,0.03896223 -"2021-07-07 20:55:00",4,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,18,55,-15.84,35.27,806,807,93461.93,102941.4,134.4297,2,1.5,287.6,286.7,94,-1,0.518,-24,1.689,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.638,197.1,1.012,"None",2.72,-1,0,-24,0.03896223 -"2021-07-07 21:55:00",5,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,19,55,-15.84,35.27,806,807,93464.36,102944.1,78.51563,2,1.5,287.7,286.2,91,-1,0.462,-24,2.15,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.432,201.1,1.241,"None",3.25,-1,0,-24,0.03896223 -"2021-07-07 22:55:00",6,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,20,55,-15.84,35.27,806,807,93432.13,102908.6,-14.20313,7,1.5,287.6,286.2,91.5,-1,0.446,-24,2.597,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.555,195.9,1.971,"None",4.61,-1,0,-24,0.03896223 -"2021-07-07 23:55:00",7,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,21,55,-15.84,35.27,806,807,93422.84,102898.4,-39.09375,7,1.5,287.4,286.1,91.9,-1,0.286,-24,2.882,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.552,200.2,1.194,"None",3.24,-1,0,-24,0.03896223 -"2021-07-08 00:55:00",8,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,22,55,-15.84,35.27,806,807,93388.43,102860.5,-75.92969,7,1.5,287.3,286,91.7,-1,0.514,-24,3.396,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.493,194.7,1.571,"None",3.18,-1,0,-24,0.03896223 -"2021-07-08 01:55:00",9,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,7,23,55,-15.84,35.27,806,807,93347.19,102815.1,-84.9375,7,1.5,286.7,285.6,93.1,-1,0.433,-24,3.828,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.364,195.7,1.334,"None",2.75,-1,0,-24,0.03896223 -"2021-07-08 02:55:00",10,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,0,55,-15.84,35.27,806,807,93310.4,102774.6,-112.4375,7,1.5,286.8,285.7,93.4,-1,0.425,-24,4.254,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.059,205.9,1.232,"None",3.84,-1,0,-24,0.03896223 -"2021-07-08 03:55:00",11,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,1,55,-15.84,35.27,806,807,93307.27,102771.1,-81.15625,7,1.5,286.5,284.8,89.1,-1,0.466,-24,4.72,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.044,236.2,1.536,"None",3.88,-1,0,-24,0.03896223 -"2021-07-08 04:55:00",12,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,2,55,-15.84,35.27,806,807,93339.49,102806.6,-7.695313,7,1.5,286.8,284.2,84.4,-1,0.513,-24,5.233,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,1.331,251.6,1.173,"None",3.17,-1,0,-24,0.03896223 -"2021-07-08 05:55:00",13,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,3,55,-15.84,35.27,806,807,93404.13,102877.8,93.73438,2,1.5,286.1,284.8,91.8,-1,0.507,-24,5.741,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,0.964,212,0.86,"None",2.58,-1,0,-24,0.03896223 -"2021-07-08 06:55:00",14,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,4,55,-15.84,35.27,806,807,93466.27,102946.3,158.9922,2,1.5,286.5,284.6,88.1,-1,0.071,-24,5.812,1.5,-1,0,-24,0,289.5,269.4,2,"0",2,-10,0.899,229.2,0.909,"None",1.98,-1,"NAN",-24,"NAN" -"2021-07-08 07:55:00",15,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,7,8,5,55,-15.84,35.27,806,807,93531.97,103018.5,192.4766,2,1.5,289.5,285.2,75.59,-1,0.453,-24,6.265,1.5,-1,0,-24,0,289.6,269.4,2,"0",2,-10,1.202,219.2,1.161,"None",3.47,-1,0.8832501,-24,"NAN" diff --git a/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv b/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv deleted file mode 100644 index ed30d1d75..000000000 --- a/tests/data3/WIGOS_0-454-2-AWSNAMITAMBO_2021-11-18T0955.csv +++ /dev/null @@ -1,5 +0,0 @@ -"TOA5","Namitambo","CR300","4720","CR310-CELL200.Std.08.01","CPU:CR310_Malawi_V1R6_06072021_T3.CR300","13058","SYNOP" -"TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24" -"TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2" -"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp" -"2021-11-18 11:55:00",414,6,"0-454-2-AWSNAMITAMBO","Namitambo","0",2021,11,18,9,55,-15.84,35.27,806,807,92434.89,101809.4,-195.9453,7,1.5,307.9,287.7,29.88,-1,24.93,-24,102.5,1.5,-1,0,-24,0,308.1,300.8,2,"0",2,-10,2.084,67.12,2.198,"None",5.72,-1,3715672,-24,1.335E+07 diff --git a/tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv b/tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv deleted file mode 100644 index 8ccad38db..000000000 --- a/tests/data3/WIGOS_0-454-2-AWSTOLEZA_2021-11-18T0955.csv +++ /dev/null @@ -1,5 +0,0 @@ -"TOA5","Namitambo","CR300","4720","CR310-CELL200.Std.08.01","CPU:CR310_Malawi_V1R6_06072021_T3.CR300","13058","SYNOP" -"TIMESTAMP","RECORD","WMO_Block","Station_ID","Station_Name","WMO_Station_Type","M_Year","M_Month","M_DayOfMonth","M_HourOfDay","M_Minutes","Latitude","Longitude","Elevation","BP_Elevation","BP","QNH","BP_Change","BP_Tendency","Temp_H","AirTempK","DewPointTempK","RH","Sun_hr","SunHrs","Sun_hr24","SunHrs24","Rain_H","Rain_hr","Rain_mm_Tot","Temp_hr24","Temp24T","AirTempMaxK","AirTempMinK","WSpeed_height","Wind_Type","Wind_Sig","Wind_T","WSpeed","WindDir","WSpeed10M_Avg","WindG_Sig","WindGust","Solar_hr","SlrJ","Solar_hr24","SlrJ24" -"TS","RN","","","","","","","","","","","","","m","","hPa","Pa","","m","K","K","%","","hours","","hours","m","","mm","","","K","K","","","","","meters/second","degrees","m/s","","m/s","","J/m^2","","J/m^2" -"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Tot","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","WVc","WVc","Smp","Smp","Smp","Smp","Tot","Smp","Smp" -"2021-11-18 11:55:00",2328,6,"0-454-2-AWSTOLEZA","Toleza Farm","0",2021,11,18,9,55,-14.95,34.96,764,765,93868.82,102865.8,-229.2422,7,1.5,308.5,288.8,30.92,-1,0.29,-24,1.446,1.5,-1,0,-24,0,310.1,303.8,2,"0",2,-10,1.487,50.36,1.08,"None",2.55,-1,3.075586,-24,11.8543 diff --git a/tests/data3/gts_headers_mapping.csv b/tests/data3/gts_headers_mapping.csv deleted file mode 100644 index e2a240054..000000000 --- a/tests/data3/gts_headers_mapping.csv +++ /dev/null @@ -1,8 +0,0 @@ -string_in_filepath,ttaaii,cccc -ISIN21DAMM,ISIN21,DAMM -ISMD01LIIB,ISMD01,LIBB -ISMD02LIIB,ISMD02,LIBB -SMRO01YRBK,SMRO01,YRBK -IOBH01LFPW,IOBH01,LFPW -ISSA05LFPW,ISSA05,LFPW -SICG20FCBB,SICG20,FCBB \ No newline at end of file diff --git a/tests/data3/metadata/discovery/cd-surface-weather-observations.yml b/tests/data3/metadata/discovery/cd-surface-weather-observations.yml deleted file mode 100644 index 76dcda288..000000000 --- a/tests/data3/metadata/discovery/cd-surface-weather-observations.yml +++ /dev/null @@ -1,69 +0,0 @@ -wis2box: - retention: P180D - topic_hierarchy: cd-brazza_met_centre/data/core/weather/surface-based-observations/synop - country: cog - centre_id: cd-brazza_met_centre - data_mappings: - plugins: - txt: - - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR - notify: true - file-pattern: '^.*_(\d{4})(\d{2}).*\.txt$' - bufr4: - - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON - buckets: - - ${WIS2BOX_STORAGE_PUBLIC} - file-pattern: '^.*\.bufr4$' - -mcf: - version: 1.0 - -metadata: - identifier: urn:wmo:md:cd-brazza_met_centre:surface-weather-observations - hierarchylevel: dataset - -identification: - title: Surface weather observations from Republic of Congo - abstract: Surface weather observations from Republic of Congo - dates: - creation: 2023-03-26 - keywords: - default: - keywords: - - surface weather - - temperature - - observations - wmo: - keywords: - - weather - keywords_type: theme - vocabulary: - name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. - url: https://codes.wmo.int/topic-hierarchy/earth-system-discipline - extents: - spatial: - - bbox: [11.0937728207,-5.03798674888,18.4530652198,3.72819651938] - crs: 4326 - temporal: - - begin: 2023-03-26 - end: null - resolution: P1H - url: https://example.org/malawi-surface-weather-observations - wmo_data_policy: core - -contact: - host: - organization: Congo National Meteorological Service (DMN) - url: https://www.dirmet.cg - individualname: Firstname Lastname - positionname: Position Name - phone: null - fax: null - address: P.O. Box 208 - city: Brazzaville - administrativearea: null - postalcode: null - country: Republic of Congo - email: you@example.com - hoursofservice: 0700h - 1500h UTC - contactinstructions: email diff --git a/tests/data3/metadata/discovery/mw-surface-weather-observations.yml b/tests/data3/metadata/discovery/mw-surface-weather-observations.yml deleted file mode 100644 index bcb14c489..000000000 --- a/tests/data3/metadata/discovery/mw-surface-weather-observations.yml +++ /dev/null @@ -1,68 +0,0 @@ -wis2box: - retention: P30D - topic_hierarchy: mw-mw_met_centre.data.core.weather.surface-based-observations.synop - country: mwi - centre_id: mw-mw_met_centre - data_mappings: - plugins: - csv: - - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR - template: CampbellAfrica-v1-template - notify: true - file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.csv$' - bufr4: - - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON - file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$' - -mcf: - version: 1.0 - -metadata: - identifier: urn:wmo:md:mw-mw_met_centre:surface-weather-observations - hierarchylevel: dataset - -identification: - title: Surface weather observations from Malawi - abstract: Surface weather observations from Malawi - dates: - creation: 2021-11-29 - keywords: - default: - keywords: - - surface weather - - temperature - - observations - wmo: - keywords: - - weather - keywords_type: theme - vocabulary: - name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. - url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline - extents: - spatial: - - bbox: [32.6881653175,-16.8012997372,35.7719047381,-9.23059905359] - crs: 4326 - temporal: - - begin: 2021-11-29 - end: null - resolution: P1H - url: https://example.org/malawi-surface-weather-observations - wmo_data_policy: core - -contact: - host: - organization: Department of Climate Change and Meteorologial Services (DCCMS) - url: https://www.metmalawi.gov.mw - individualname: Firstname Lastname - positionname: Position Name - phone: "+2651822014" - fax: "+2651822215" - address: P.O. Box 1808 - city: Blantyre - administrativearea: Blantyre District - postalcode: M3H 5T4 - country: Malawi - email: you@example.org - hoursofservice: 0700h - 1500h UTC - contactinstructions: email diff --git a/tests/data3/metadata/station/station_list.csv b/tests/data3/metadata/station/station_list.csv deleted file mode 100644 index f4e3f38b8..000000000 --- a/tests/data3/metadata/station/station_list.csv +++ /dev/null @@ -1,20 +0,0 @@ -station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region -NAMITAMBO,0-454-2-AWSNAMITAMBO,AWSNAMITAMBO,landFixed,-15.84052,35.27428,806,,MWI,africa -TOLEZA,0-454-2-AWSTOLEZA,AWSTOLEZA,landFixed,-14.948,34.955,764,,MWI,africa -POINTE-NOIRE,0-20000-0-64400,64400,landFixed,-4.8045,11.8817,18.00,19.20,COD,africa -DOLISIE,0-20000-0-64401,64401,landFixed,-4.2119,12.6680,330.00,331.25,COD,africa -MOUYONDZI,0-20000-0-64402,64402,landFixed,-4.0000,13.9500,509.00,510.20,COD,africa -MAKABANA,0-20000-0-64403,64403,landFixed,-3.4833,12.6166,160.00,162.20,COD,africa -SIBITI,0-20000-0-64405,64405,landFixed,-3.6845,13.3698,530.00,531.40,COD,africa -BRAZZAVILLE /MAYA-MAYA,0-20000-0-64450,64450,landFixed,-4.2636,15.2429,314.27,315.6,COD,africa -M'POUYA,0-20000-0-64452,64452,landFixed,-2.6166,16.2166,311.00,312.30,COD,africa -DJAMBALA,0-20000-0-64453,64453,landFixed,-2.5333,14.7666,789.00,790.20,COD,africa -GAMBOMA,0-20000-0-64454,64454,landFixed,-1.9000,15.8500,476.00,477.20,COD,africa -MAKOUA,0-20000-0-64456,64456,landFixed,-0.0166,15.6500,394.00,396.20,COD,africa -OUESSO,0-20000-0-64458,64458,landFixed,1.6125,16.0402,352.00,353.10,COD,africa -IMPFONDO,0-20000-0-64459,64459,landFixed,1.5928,18.0540,324.00,325.20,COD,africa -SOUANKE,0-20000-0-64460,64460,landFixed,2.0666,14.0333,549.00,550.20,COD,africa -KELLE,0-20000-0-64462,64462,landFixed,0.0829,14.5329,408.00,409.30,COD,africa -EWO,0-20000-0-64463,64463,landFixed,-0.8541,14.8034,479.00,480.20,COD,africa -N'KAYI,0-20000-0-64406,64406,landFixed,-4.2218,13.2856,165.00,166.40,COD,africa -OLLOMBO,0-20000-0-64461,64461,landFixed,-1.2257,15.9166,327.05,328.45,COD,africa \ No newline at end of file From 06433999c5c4e2030ed1a969a3a27b6a7ad8ccc1 Mon Sep 17 00:00:00 2001 From: Maaike Date: Wed, 26 Jun 2024 17:56:23 +0200 Subject: [PATCH 12/13] add IGNORE for medium risk on basic auth --- .zap/rules.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/.zap/rules.tsv b/.zap/rules.tsv index 2e58035e8..4a191ad7f 100644 --- a/.zap/rules.tsv +++ b/.zap/rules.tsv @@ -20,3 +20,4 @@ 10009 IGNORE In Page Banner Information Leak Low 10036 IGNORE "Server Leaks Version Information via ""Server"" HTTP Response Header Field" Low 10110 IGNORE Dangerous JS Functions Low +10105 IGNORE Authentication Credentials Captured Medium From cacad85575f47b5f9e18c26d9f0c53e4a71dcff8 Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 27 Jun 2024 08:44:52 +0200 Subject: [PATCH 13/13] avoid codes.wmo.int issue by adding _view=with_metadata --- wis2box-management/Dockerfile | 8 ++++---- wis2box-management/wis2box/metadata/station.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wis2box-management/Dockerfile b/wis2box-management/Dockerfile index 58176a7a8..7d435422e 100644 --- a/wis2box-management/Dockerfile +++ b/wis2box-management/Dockerfile @@ -51,10 +51,10 @@ RUN apt-get update -y && apt-get install -y ${DEBIAN_PACKAGES} \ && rm -rf /var/lib/apt/lists/* \ # cache WMDR codelists && mkdir -p /home/wis2box/wmdr-codelists \ - && curl -s "https://codes.wmo.int/wmdr/FacilityType?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/FacilityType.csv \ - && curl -s "https://codes.wmo.int/wmdr/InstrumentOperatingStatus?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/InstrumentOperatingStatus.csv \ - && curl -s "https://codes.wmo.int/wmdr/TerritoryName?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/TerritoryName.csv \ - && curl -s "https://codes.wmo.int/wmdr/WMORegion?_format=csv&status=valid" > /home/wis2box/wmdr-codelists/WMORegion.csv + && curl -s "https://codes.wmo.int/wmdr/FacilityType?_format=csv&_view=with_metadata&status=valid" > /home/wis2box/wmdr-codelists/FacilityType.csv \ + && curl -s "https://codes.wmo.int/wmdr/InstrumentOperatingStatus?_format=csv&_view=with_metadata&status=valid" > /home/wis2box/wmdr-codelists/InstrumentOperatingStatus.csv \ + && curl -s "https://codes.wmo.int/wmdr/TerritoryName?_format=csv&_view=with_metadata&status=valid" > /home/wis2box/wmdr-codelists/TerritoryName.csv \ + && curl -s "https://codes.wmo.int/wmdr/WMORegion?_format=csv&_view=with_metadata&status=valid" > /home/wis2box/wmdr-codelists/WMORegion.csv # copy the app diff --git a/wis2box-management/wis2box/metadata/station.py b/wis2box-management/wis2box/metadata/station.py index d1b729d78..dd7bf100e 100644 --- a/wis2box-management/wis2box/metadata/station.py +++ b/wis2box-management/wis2box/metadata/station.py @@ -78,7 +78,7 @@ def get_wmdr_codelists() -> dict: reader = csv.reader(fh) next(reader) for row in reader: - codelists[key].append(row[4]) + codelists[key].append(row[1]) return codelists