Nighttime zeros (#90) #minor

* make nighttimes zeros, even if pvlive is not giving any values * add csv * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * lint * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update docs * fix * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * PR comment --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
openclimatefix · Dec 6, 2024 · e78c7cc · e78c7cc
1 parent bfb88ca
commit e78c7cc
Show file tree

Hide file tree

Showing 7 changed files with 495 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -12,19 +12,29 @@ python gspconsumer/app.py
 ```
 
 The environmental variables are
-DB_URL: The natabase url you want to save the results to
-REGIME: Regime of which to pull, either 'in-day' or 'day-after'
-N_GSPS: The number of gsps you want to pull
-INCLUDE_NATIONAL: Option to load national data, or not
-UK_LONDON_HOUR: Optional to check UK London hour. This means can run this service at the same
+- DB_URL: The natabase url you want to save the results to
+- REGIME: Regime of which to pull, either 'in-day' or 'day-after'
+- N_GSPS: The number of gsps you want to pull
+- INCLUDE_NATIONAL: Option to load national data, or not
+- UK_LONDON_HOUR: Optional to check UK London hour. This means can run this service at the same
    UTC times, independently of the clock change.
-BACKFILL_HOURS: Optional, defaults to 2. The amount of hours of data that is backfilled.
+- BACKFILL_HOURS: Optional, defaults to 2. The amount of hours of data that is backfilled.
+- ELEVATION_LIMIT: Optional, defaults to 5. If no PVLive values are found, and sun elevation is below this, then the values are set to 0
 
 These options can also be enter like this:
 ```
 python gspconsumer/app.py --n-gsps=10
 ```
 
+## Tests
+
+To run tests use the following command
+```bash
+docker stop $(docker ps -a -q)
+docker-compose -f test-docker-compose.yml build
+docker-compose -f test-docker-compose.yml run gspconsumer
+```
+
 ## Contributors ✨
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):

diff --git a/gspconsumer/app.py b/gspconsumer/app.py
@@ -23,6 +23,7 @@
 import gspconsumer
 from gspconsumer.backup import make_gsp_yields_from_national
 from gspconsumer.gsps import filter_gsps_which_have_new_data, get_gsps
+from gspconsumer.nitghtime import make_night_time_zeros
 from gspconsumer.time import check_uk_london_hour
 
 logging.basicConfig(
@@ -181,6 +182,14 @@ def pull_data_and_save(
 
         logger.debug(f"Got {len(gsp_yield_df)} gsp yield for gsp id {gsp.gsp_id} before filtering")
 
+        if len(gsp_yield_df) == 0:
+            logger.warning(
+                f"Did not find any data for {gsp.gsp_id} for {start} to {end}. "
+                f"Will try adding some nighttime zeros"
+            )
+
+            gsp_yield_df = make_night_time_zeros(start, end, gsp, gsp_yield_df, regime)
+
         if len(gsp_yield_df) == 0:
             logger.warning(f"Did not find any data for {gsp.gsp_id} for {start} to {end}")
         else:

diff --git a/gspconsumer/data/uk_gsp_locations.csv b/gspconsumer/data/uk_gsp_locations.csv
@@ -0,0 +1,319 @@
+gsp_id,latitude,longitude
+0,54.08963881116971,-2.6122022827776883
+1,50.4397724226424,-3.7649400990870867
+2,56.31309981020173,-3.40104793395752
+3,51.4380654557521,-3.352160180214071
+4,51.53327168412609,-0.3157518812490342
+5,51.52942313656777,-0.24081634267910335
+6,57.755493801147615,-4.1785322260321305
+7,51.652073225559874,-2.890319162834425
+8,50.89355446330147,-4.11098446637198
+9,51.74266498490046,-0.8642803186998599
+10,56.566513966274705,-2.6678365936718658
+11,56.22803632360375,-4.960172753655607
+12,57.86151013368091,-6.923114214846543
+13,50.87224254012467,-2.7154607824588557
+14,55.43273455751717,-4.624794634236638
+15,55.891277293742505,-3.7033486164410157
+16,56.042394719893174,-3.75441821548991
+17,51.54759972064965,0.12254374911873507
+18,57.51208027106318,-4.56074158395304
+19,51.423966492325995,-0.02772223886373339
+20,51.3318892643049,-0.07712168315476395
+21,57.477901734795296,-2.922240308141663
+22,55.7585974911759,-2.1017569956578166
+23,52.28404838679972,-1.5293126278082902
+24,53.076807807307766,-0.22311467485776534
+25,53.400864212876336,-3.079410013762426
+26,52.218257137438115,-2.659810665988331
+27,55.129901721253624,-1.7185296612108043
+28,53.822927422774185,-2.6567640021220584
+29,57.159110592632416,-3.9118537107828377
+30,51.01789788234229,-0.26705292783343454
+31,56.01152050045698,-3.958975622602336
+32,50.932645888576474,-1.2399199218691819
+33,56.25945030724385,-3.9655265132553863
+34,51.865243659784134,0.5601665543062958
+35,55.86975691611967,-4.388407996798448
+36,53.96503431845062,-1.9466495036993317
+37,53.391432557807086,-2.1213104944140126
+38,52.216997412609025,1.0659884694103903
+39,56.81021841648707,-2.749571929394575
+40,51.66363639559655,-0.06240542367561253
+41,51.27091516105386,-1.0482775129242372
+42,57.30480825098837,-5.871650292244331
+43,58.086717568614205,-3.9091373884362395
+44,55.96263512455307,-3.442303196467965
+45,51.16068951709011,-2.8395765613083808
+46,56.44469070271509,-3.690054681046586
+47,52.25116099487113,0.23502126478236401
+48,52.74347622478786,-2.1288302985075886
+49,52.5644376647766,-1.9291229461414865
+50,55.61391229435623,-5.37974613160416
+51,54.93292247157611,-3.997156805570965
+52,53.66482502525799,-0.8546823570337597
+53,51.27739830888632,1.0683782124452061
+54,53.21819780280634,-2.905452310865807
+55,51.52166905448311,-3.144536709334452
+56,53.43233336888581,-2.378453167178157
+57,58.37842472288014,-4.852610102123524
+58,55.845571517497454,-4.175936468628338
+59,57.18718201390254,-4.912706999484707
+60,52.98119565951938,-2.1728469729121596
+61,55.199932748781904,-3.170685708427182
+62,56.4754216254694,-3.1009327767091555
+63,55.849103941610565,-4.263533973396236
+64,50.721962040482495,-2.4886533708148977
+65,51.37816990774408,-0.33184032326493434
+66,53.179824715270016,-1.3272940592791722
+67,51.51979907546201,-0.08973184968759648
+68,57.14097529662204,-2.085542213284961
+69,55.81567985403914,-4.1416759286807325
+70,55.861537642816444,-4.0182302662642
+71,55.909509427028986,-2.8404860886486776
+72,53.21630449039704,-3.3027420885142518
+73,56.66047829552779,-3.449016059367627
+74,52.407367813643226,-1.273231272299555
+75,51.75927411052895,-1.2843358105373595
+76,55.40378269458934,-4.228968799942811
+77,57.094756302891795,-2.238965615626567
+78,53.93211668027794,-0.49440245712902264
+79,55.7842152053763,-4.430505427711066
+80,55.94314028745244,-3.9601180625806838
+81,56.32731450286659,-2.993495296926359
+82,55.886680562762955,-3.30810736082433
+83,55.84289056926137,-4.203785614722218
+84,55.936040295581066,-4.715441036485857
+85,56.139939726111116,-3.7255561046232795
+86,55.961294716033485,-3.2089933679354834
+87,58.51417142110651,-4.089328755734948
+88,52.79017546658366,-1.6080826953586376
+89,53.77729736716154,-1.0012100670341677
+90,55.91091255171715,-3.5649787558896593
+91,55.96190090062895,-4.370424691307213
+92,58.30097119479877,-3.440292401098374
+93,56.463624974602105,-2.9813925014668823
+94,57.413880201547734,-6.389594935043644
+95,55.17087433226081,-3.797913294318771
+96,55.9685482623802,-2.5618944780754833
+97,56.08986997000184,-3.5331250574960285
+98,55.941503220299374,-5.106971441948878
+99,57.31095286885845,-2.1103537624357886
+100,51.49551330561556,-0.31573070724603086
+101,52.28965265373672,-0.21351426050408123
+102,55.69999612451459,-2.414739339281116
+103,52.048066463383655,-1.0182764434315115
+104,55.91571001123878,-4.124099513196762
+105,55.750061973437155,-4.173425113365193
+106,55.74555552855855,-4.178092502527667
+107,55.829874689635815,-4.45357310389885
+108,57.644535211722406,-3.2839633993038353
+109,53.65762747658316,-1.865284349997057
+110,51.70646105071326,-0.3451123596208883
+111,52.656634386500784,-1.2407123847360304
+112,55.90347541675805,-4.49432031505265
+113,50.81622914887224,-3.4586469826313535
+114,57.314963607773876,-4.898136383554622
+115,57.145885031196556,-4.656950811110966
+116,50.71291248228438,-1.332548878490778
+117,52.0898917880223,-1.8366671604791627
+118,53.66310267738869,-1.2987064825550447
+119,53.827862356021434,-1.3280378070764536
+120,57.20412755321525,-2.449621993760841
+121,56.894680724957574,-2.417686789887311
+122,53.31371284263113,-2.4641895137033107
+123,55.872139592415124,-4.282801988377205
+124,55.022561468353594,-2.2364507148785644
+125,57.6604554954919,-2.021033252797223
+126,53.12700249745209,-2.5900317579197347
+127,56.80986775054161,-5.423090655206695
+128,55.586394924152515,-2.9394449505267812
+129,55.74132119027129,-4.3045318162599955
+130,54.90727861857407,-4.939502230041431
+131,56.08915987739751,-3.2246559260420278
+132,56.22283233499512,-3.1931892312023056
+133,55.935376938954484,-3.224655393643302
+134,55.85370694900962,-4.3174337518122945
+135,52.46046605230226,-0.7633927168726975
+136,53.58508776148854,-0.20971295574010446
+137,55.980187161131084,-3.659022283021686
+138,57.78491796330342,-5.197091541800843
+139,60.36608868616882,-1.261051088790501
+140,51.569445589794476,-0.03680229489860089
+141,55.825849162769536,-4.27852895797351
+142,52.527599969534535,-1.687423045666542
+143,54.56526854204074,-2.88510508839719
+144,54.686468962693276,-1.312011497766392
+145,55.40895882218818,-2.6777483158795827
+146,54.81633473084184,-1.4060343985219468
+147,56.03133807747146,-4.763647232733751
+148,54.038921585917954,-2.7286259394684174
+149,55.72382798832993,-4.9044182242853624
+150,51.44070158349771,0.051868953363603365
+151,51.54790513827124,-3.0309032025134384
+152,50.31648734579069,-5.064246241572704
+153,56.03191099169258,-3.396685695443903
+154,57.350483409726415,-4.2541219399165735
+155,51.609702517575556,-2.378100403589691
+156,52.605136163335715,-2.540374776978396
+157,51.52530605432938,-0.0232766047354594
+158,51.543133915007466,-0.5663874209931994
+159,55.849976128661055,-4.626709555970372
+160,53.32302429182831,-1.5190268292608866
+161,55.730270680381864,-3.2390201723865304
+162,53.460450776946935,-0.38008867579268285
+163,53.56654698758173,-2.4219081215003713
+164,51.344644623718175,0.7931117817993524
+165,53.48220570181196,-2.858442640600742
+166,53.565323630587244,-2.805594668375912
+167,55.95801594170061,-4.254278649676919
+168,56.45944257484058,-4.363974563393436
+169,55.93562919122731,-4.410810669610842
+170,55.572444622936025,-4.477010500951957
+171,55.66062624416027,-4.516572312215519
+172,55.613043668869075,-4.6534228669832745
+173,56.70358005038768,-5.014194812627842
+174,51.378671915340995,0.46790716372421354
+175,53.78878855519252,-1.5916627693856975
+176,52.415529992197555,-1.9723175298335267
+177,54.136952659379006,-1.4576537614856915
+178,54.5158010331045,-0.9886887706958253
+179,58.04643420478772,-4.472134453127432
+180,51.43166168586822,-0.5028898475493283
+181,50.48185559544979,-4.300960273253076
+182,52.8372678064562,-3.1444338773422538
+183,56.22816423826592,-2.867633513529039
+184,55.851313833762894,-3.4743464431499445
+185,55.580405370381555,-3.711111072252207
+186,53.489155423348684,-2.962412597346384
+187,51.45828349101746,0.17366516282082894
+188,51.42927833817182,0.19993028271759095
+189,51.534288141320175,-0.16907998328235982
+190,51.60827113054155,-0.7243097293487956
+191,50.86900148253168,-0.852849584560334
+192,56.71895196108792,-2.984310171283939
+193,56.49221926836653,-3.0078420821546845
+194,53.24754582105187,-2.1032504353625754
+195,57.64760901855897,-2.5174859278136275
+196,51.62157595820984,-3.620655300504345
+197,50.899953970428456,-1.9877288610230908
+198,55.2096911732598,-4.712275642183986
+199,51.34499576399909,-2.008303332586673
+200,56.51107599139895,-2.873137719369804
+201,51.602217986258744,-0.2202038554567407
+202,51.676278254049826,-1.7955907631108847
+203,58.370447196737835,-3.97161100042104
+204,57.34594224152706,-3.7577059605207053
+205,55.81071414141518,-3.784780630749349
+206,52.489709034475865,-1.839680107160318
+207,53.414615966793015,-1.5902618293584774
+208,54.93886406468102,-4.479582250999347
+209,51.47648929763379,-0.06632678123240003
+210,51.260633891368336,0.21777152253113188
+211,51.49435720702496,-0.40586800361681963
+212,50.96247975289178,0.34737638998055903
+213,53.361947516881536,-1.52526151539822
+214,52.66953640401893,1.3167447856586103
+215,54.36951240858125,-1.6730049811143168
+216,51.008036470554764,-1.4610689001123724
+217,52.53774599309822,-2.0240129719976063
+218,54.88165637314494,-1.4526483403782777
+219,52.494228123962195,-2.0336606173052294
+220,54.45345869983049,-1.253767932166746
+221,53.92855685023935,-2.3147001006409473
+222,55.845579900403926,-4.423027089067133
+223,55.87743394473377,-4.318558844063477
+224,57.51653846036386,-1.8634093484124161
+225,57.52908714738853,-2.1646200165181133
+226,51.999114116609746,0.41331965044295876
+227,51.823969320376534,-4.945045144948275
+228,55.420941900357974,-1.8830489648333113
+229,52.52368252722568,-2.19113840253687
+230,53.086564193762364,-4.2552138089271585
+231,53.767469910303575,-2.823559359035836
+232,57.19236860307619,-2.1341997946676425
+233,53.41700857991712,-1.4480396474365629
+234,55.86226511498659,-3.0601173481542383
+235,53.99336561860267,-1.1936025347259882
+236,55.84231564498689,-5.806096457493757
+237,55.88096957917806,-4.229220148197532
+238,51.504942981303536,-3.631216946015672
+239,57.06981908001384,-5.248479400035539
+240,53.4460167384588,-2.6716239311050636
+241,56.61939536751675,-4.688378274626171
+242,52.03651661770262,-3.3118109111268974
+243,52.804572537817556,-1.093939437440976
+244,55.739209341072545,-3.9618455354189006
+245,51.71294078680873,0.7292759033419889
+246,51.56361382847432,0.07299926887082712
+247,56.15295196821997,-3.1322770755398874
+248,57.05658036505092,-2.1646049783384522
+249,53.68747366264027,-2.284133986776166
+250,52.720714758211976,-1.8968335690279727
+251,51.774008857553284,0.05192214119954359
+252,55.741700192590834,-4.7781950774878466
+253,54.62921064063282,-1.2084585707460367
+254,53.69987113715329,-0.06623167012315737
+255,55.84592021284949,-4.269426482635865
+256,52.95415951848228,-1.073298264182393
+257,51.30720380390985,-2.6385661510922014
+258,51.11582533176885,0.9266720349285045
+259,57.57672911907792,-1.8560109364727628
+260,56.362014533233186,-4.052426202489593
+261,53.36577618868204,-1.4463614175629202
+262,57.45539172229343,-5.111381708130045
+263,55.97408973303068,-3.186772193718422
+264,52.66421455842416,-2.827933415871064
+265,55.94334178747468,-3.2904710801828516
+266,51.510617481954,-0.1622258646140644
+267,53.7524473445445,-1.6075925777082745
+268,56.24255811061385,-4.682107704014756
+269,53.33464958253043,-2.2444623338755045
+270,55.915442572905704,-4.850696185802986
+271,54.648620384950334,-1.8627343112874104
+272,54.984390747762085,-1.436541858653582
+273,53.41302762480835,-1.9613279283476641
+274,53.15586288354437,-0.8425108138810233
+275,54.97358884457689,-1.7115363678015703
+276,55.02577293808564,-2.021076195776186
+277,56.08764800107761,-4.435760613697441
+278,56.10211602585332,-3.9543515568016994
+279,55.972104484483104,-4.541946089856799
+280,51.9480898336319,-0.5892570127948631
+281,51.93346177190775,-4.10360808602242
+282,57.10195750776059,-3.0887847897677885
+283,51.07805204643276,-3.3946529441440765
+284,56.42549443948404,-5.698226899673246
+285,55.9773203872407,-3.2587896647952963
+286,53.413051941479885,-1.3347965326765294
+287,53.54655759574791,-1.1243566582253843
+288,58.852467792027056,-3.164878283885756
+289,53.40598989220629,-1.1996685761525336
+290,51.533173797695454,0.43073157171016
+291,51.62881319270748,-0.10073136666880451
+292,52.76905276634672,-3.794259872926864
+293,56.72610355859715,-3.9513362205934524
+294,55.02373222429596,-1.5094499326405595
+295,51.647908783190395,-3.357417556193029
+296,51.86329556515295,-2.3038731466626134
+297,52.661408022840845,0.4056696078858301
+298,52.82811159037331,-0.15053257547035617
+299,51.59835096636695,0.30118468883617033
+300,51.690587595045095,-0.5493644181471224
+301,54.93645382400767,-1.4768584018484647
+302,53.22961879283787,-0.5251783331431347
+303,56.130246607780556,-3.3106185461484134
+304,55.86251762825322,-4.275797353988976
+305,53.549600126069635,-2.1338918304860717
+306,55.91746159775159,-3.2031510705995503
+307,53.43239836339646,-1.408496571915699
+308,52.58401937865336,-2.054745254036951
+309,53.0090055596229,-1.5815669795057368
+310,51.434709910344985,-0.19271557787215945
+311,57.16005350688278,-2.103515128530736
+312,53.501647863119274,-1.5961384434889816
+313,57.15096888969699,-2.205977925529348
+314,51.47995354714101,0.3083908109491153
+315,51.28128519182304,-0.4509044723020023
+316,53.316604778015424,-4.464006214356469
+317,51.972289374441246,-0.14212115851630353