From 454027679adc067c3327f5ef3aed792e6b33aa0f Mon Sep 17 00:00:00 2001 From: Heinz-Alexander Fuetterer Date: Thu, 28 Sep 2023 15:58:35 +0200 Subject: [PATCH] chore: convert json files to yaml --- fuji_server/data/repodois.json | 236 --------------------------- fuji_server/data/repodois.yaml | 253 +++++++++++++++++++++++++++++ fuji_server/helper/preprocessor.py | 7 +- tests/helper/test_preprocessor.py | 8 +- 4 files changed, 264 insertions(+), 240 deletions(-) delete mode 100644 fuji_server/data/repodois.json create mode 100644 fuji_server/data/repodois.yaml diff --git a/fuji_server/data/repodois.json b/fuji_server/data/repodois.json deleted file mode 100644 index 8bb4ceb6..00000000 --- a/fuji_server/data/repodois.json +++ /dev/null @@ -1,236 +0,0 @@ -{ - "ardcx.aims": "https://doi.org/10.17616/R3X908", - "ardcx.curtin": "https://doi.org/10.17616/R3WM02", - "ardcx.griffith": "https://doi.org/10.17616/R3FG92", - "ardcx.usq": "https://doi.org/10.17616/R36Q18", - "arfm.ufzdrp": "https://doi.org/10.17616/R31NJN1M", - "awzy.fednnv": "https://doi.org/10.17616/R3PM1K", - "bamf.bamf": "https://doi.org/10.17616/R31NJMZ5", - "bf.discover": "https://doi.org/10.17616/R31NJMN0", - "bibsys.uit-ord": "https://doi.org/10.17616/R3TV17", - "bkmx.azjwzc": "https://doi.org/10.17616/R3VK72", - "bl.ads": "https://doi.org/10.17616/R3MW23", - "bl.bath": "https://doi.org/10.17616/R32928", - "bl.birkbeck": "https://doi.org/10.17616/R31NJLW9", - "bl.bristol": "https://doi.org/10.17616/R3CP84", - "bl.brookes": "https://doi.org/10.17616/R31NJMGX", - "bl.brunel": "https://doi.org/10.17616/R3DP9T", - "bl.cam": "https://doi.org/10.17616/R3SW4D", - "bl.cran": "https://doi.org/10.17616/R3MH0J", - "bl.dri": "https://doi.org/10.17616/R3W632", - "bl.dundee": "https://doi.org/10.17616/R31NJN0E", - "bl.ecmwf": "https://doi.org/10.17616/R3WS8T", - "bl.ed": "https://doi.org/10.17616/R3RG6Q", - "bl.essex": "https://doi.org/10.17616/R3SV0J", - "bl.exeter": "https://doi.org/10.17616/R3033Z", - "bl.hallam": "https://doi.org/10.17616/R3S92S", - "bl.herts": "https://doi.org/10.17616/R31NJMM4", - "bl.iita": "https://doi.org/10.17616/R3K22H", - "bl.imperial": "http://doi.org/10.17616/R3K64N", - "bl.lboro": "https://doi.org/10.17616/R3564X", - "bl.leeds": "https://doi.org/10.17616/R3JH19", - "bl.lincoln": "https://doi.org/10.17616/R3MZ01", - "bl.lshtm": "https://doi.org/10.17616/R3705V", - "bl.mendeley": "https://doi.org/10.17616/R3DD11", - "bl.nhm": "https://doi.org/10.17616/R3PS7K", - "bl.oxdb": "https://doi.org/10.17616/R3504V", - "bl.reading": "https://doi.org/10.17616/R3D075", - "bl.salford": "https://doi.org/10.17616/R31D20", - "bl.shef": "https://doi.org/10.17616/R3P64K", - "bl.soton": "https://doi.org/10.17616/R3H05B", - "bl.standrew": "https://doi.org/10.17616/R3878Q", - "bl.strath": "https://doi.org/10.17616/R3G18H", - "bl.surrey": "https://doi.org/10.17616/R34N3Q", - "bl.uclan": "https://doi.org/10.17616/R38D3J", - "bl.ucld": "https://doi.org/10.17616/R34F41", - "bl.uel": "https://doi.org/10.17616/R3V48Z", - "bl.ukda": "https://doi.org/10.17616/R3088K", - "brown.bdr": "https://doi.org/10.17616/R3193B", - "brvz.rdr": "https://doi.org/10.17616/R31NJN5S", - "caltech.data": "https://doi.org/10.17616/R3SW99", - "carl.frdr": "https://doi.org/10.17616/R3X50W", - "cbg.datasets": "https://doi.org/10.17616/R3PP7J", - "ccdc.csd": "https://doi.org/10.17616/R36011", - "cern.hepdata": "https://doi.org/10.17616/R30W2H", - "cern.inspire": "https://doi.org/10.17616/R3JC9Z", - "cern.zenodo": "https://doi.org/10.17616/R3QP53", - "clarin.clarin": "https://doi.org/10.17616/R3RP5D", - "cngb.gigadb": "https://doi.org/10.17616/R3TG83", - "cnic.sciencedb": "https://doi.org/10.17616/R31NJMNT", - "concor.kcydcu": "https://doi.org/10.17616/R3FJ70", - "cornell.ciser": "https://doi.org/10.17616/R3ZK70", - "cornell.library": "https://doi.org/10.17616/R3FP94", - "cornell.ndacan": "https://doi.org/10.17616/R3N90S", - "cos.osf": "https://doi.org/10.17616/R3N03T", - "crui.ogsts": "https://doi.org/10.17616/R3TR3Z", - "crui.unibo": "https://doi.org/10.17616/R3P19R", - "csc.nrd": "https://doi.org/10.17616/R33649", - "csic.digital": "https://doi.org/10.17616/R3P34J", - "cul.ciesin": "https://doi.org/10.17616/R31605", - "dafi.client": "https://doi.org/10.17616/R3QS3C", - "dans.archive": "https://doi.org/10.17616/R3401D", - "dans.dataversenl": "https://doi.org/10.17616/R33W6Z", - "dartlib.crawdad": "https://doi.org/10.17616/R3CD0B", - "delft.data4tu": "https://doi.org/10.17616/R3VG6N", - "delft.isric": "https://doi.org/10.17616/R3X01J", - "delft.knmi": "https://doi.org/10.17616/R3QH0G", - "delft.maastro": "https://doi.org/10.17616/R38W4R", - "delft.ru": "https://doi.org/10.17616/R31NJMZR", - "delft.surfsara": "https://doi.org/10.17616/R31NJML8", - "delft.uu": "https://doi.org/10.17616/R3ZB7K", - "delft.vliz": "https://doi.org/10.17616/R3WG7B", - "dipf.rdcedu": "https://doi.org/10.17616/R3R010", - "dk.dtic": "https://doi.org/10.17616/R31NJMWG", - "dk.sb": "https://doi.org/10.17616/R31NJM89", - "dkrz.esgf": "https://doi.org/10.17616/R3BK7D", - "dkrz.wdcc": "https://doi.org/10.17616/R3989R", - "dryad.dryad": "https://doi.org/10.17616/R34S33", - "dzhw.fdz-dzhw": "https://doi.org/10.17616/R3BR0G", - "elsevier.md": "https://doi.org/10.17616/R3DD11", - "esdis.ornl": "https://doi.org/10.17616/R3SG61", - "estdoi.keel": "https://doi.org/10.17616/R3P636", - "estdoi.qdb": "https://doi.org/10.17616/R31NJMVF", - "estdoi.repo": "https://doi.org/10.17616/R30D32", - "ethz.da-rd": "https://doi.org/10.17616/R34D05", - "ethz.e-coll": "https://doi.org/10.17616/R3HF66", - "ethz.sicas": "https://doi.org/10.17616/R3HP8C", - "ethz.wgms": "https://doi.org/10.17616/R3WP5N", - "ethz.wsl": "https://doi.org/10.17616/R3G49X", - "europ.odin": "https://doi.org/10.17616/R3J917", - "fbtk.dmxfnr": "https://doi.org/10.17616/R31NJMV8", - "fct.uminho": "https://doi.org/10.17616/R31NJMNP", - "figshare.ariz": "https://doi.org/10.17616/R31NJMSF", - "figshare.ars": "https://doi.org/10.17616/R3PK5R", - "figshare.iastate": "https://doi.org/10.17616/R3GV3W", - "figshare.uct": "https://doi.org/10.17616/R36R4R", - "fzj.b2share": "https://doi.org/10.17616/R3VK72", - "fzj.tereno": "https://doi.org/10.17616/R39G9T", - "gbif.gbif": "https://doi.org/10.17616/R3J014", - "gdcc.csuc": "https://doi.org/10.17616/R31NJMYF", - "gdcc.harvard-dv": "https://doi.org/10.17616/R3C880", - "gdcc.harvard-sbgr": "https://doi.org/10.17616/R3N92J", - "gdcc.harvard-slp": "https://doi.org/10.17616/R3XD09", - "gdcc.jhu": "https://doi.org/10.17616/R3RW77", - "gdcc.ntu": "https://doi.org/10.17616/R3N77R", - "gdcc.odum-dv": "https://doi.org/10.17616/R3RP41", - "gdcc.odum-library": "https://doi.org/10.17616/R3RP41", - "gdcc.syr-qdr": "https://doi.org/10.17616/R3CS51", - "gesis.adp": "https://doi.org/10.17616/R3SG7D", - "gesis.aussda": "https://doi.org/10.17616/R39G72", - "gesis.bibb-fdz": "https://doi.org/10.17616/R3359V", - "gesis.csda": "https://doi.org/10.17616/R35P6H", - "gesis.dezim": "https://doi.org/10.17616/R31NJMNK", - "gesis.gesis": "https://doi.org/10.17616/R3BB70", - "gesis.icpsr": "https://doi.org/10.17616/R3BC8Q", - "gesis.indepth": "https://doi.org/10.17616/R3432T", - "gesis.neps": "https://doi.org/10.17616/R35P7W", - "gesis.rki": "https://doi.org/10.17616/R3K31G", - "gesis.share": "https://doi.org/10.17616/R3BK5N", - "gesis.sodanet": "https://doi.org/10.17616/R38G7R", - "gesis.srda": "https://doi.org/10.17616/R31NJMLA", - "gesis.ssri": "https://doi.org/10.17616/R31NJMWM", - "gesis.zpid": "https://doi.org/10.17616/R3FS48", - "hlqc.znxeli": "https://doi.org/10.17616/R31NJMJP", - "ieee.dataport": "https://doi.org/10.17616/R3H77H", - "iiasa.dare": "https://doi.org/10.17616/R31NJMH0", - "illinois.databank": "https://doi.org/10.17616/R3C93F", - "inist.adisp": "https://doi.org/10.17616/R3W90Z", - "inist.cds": "https://doi.org/10.17616/R3690R", - "inist.humanum": "https://doi.org/10.17616/R3JM1B", - "inist.ifremer": "https://doi.org/10.17616/R31NJMHX", - "inist.ifsttar": "https://doi.org/10.17616/R31NJMKM", - "inist.ill": "https://doi.org/10.17616/R33H18", - "inist.inra": "https://doi.org/10.17616/R3DJ4J", - "inist.otelo": "https://doi.org/10.17616/R3F19K", - "inist.resif": "https://doi.org/10.17616/R37Q06", - "iris.iris": "https://doi.org/10.17616/R3X607", - "ist.rex": "https://doi.org/10.17616/R3877B", - "jcvi.eivbwb": "https://doi.org/10.17616/R30P93", - "jcvi.gxpwaq": "https://doi.org/10.17616/R3S634", - "kaggle.kaggle": "https://doi.org/10.17616/R32N5V", - "kim.openkim": "https://doi.org/10.17616/R3SK8T", - "kmot.rdr": "https://doi.org/10.17616/R31NJMN6", - "ktsw.aezvvv": "https://doi.org/10.17616/R31NJMVX", - "landcare.general": "https://doi.org/10.17616/R3092N", - "landcare.soils": "https://doi.org/10.17616/R3660Q", - "ldc.catalog": "https://doi.org/10.17616/R3T06H", - "lmu.ub": "https://doi.org/10.17616/R3PP65", - "lxkc.dskyfi": "https://doi.org/10.17616/R3W331", - "mdw.repository": "https://doi.org/10.17616/R30M00", - "mit.physio": "https://doi.org/10.17616/R3D06S", - "mlbs.skuxgs": "https://doi.org/10.17616/R3ZP8D", - "nasapds.nasapds": "https://doi.org/10.17616/R37593", - "nkn.nkn": "https://doi.org/10.17616/R3JK91", - "nlqq.xoncsf": "https://doi.org/10.17616/R31NJMYB", - "noaa.ncei": "https://doi.org/10.17616/R3362J", - "nsidc.datactr": "https://doi.org/10.17616/R3HP4V", - "nu.arch": "https://doi.org/10.17616/R31NJMGT", - "nus.sb": "https://doi.org/10.17616/R3FF4T", - "nzau.data": "https://doi.org/10.17616/R3BW7H", - "ocean.ocean": "https://doi.org/10.17616/R38F5N", - "ocul.spdv": "https://doi.org/10.17616/R32326", - "ohsu.ohsu": "https://doi.org/10.17616/R3MS9Q", - "oibk.ohycfa": "https://doi.org/10.17616/R31NJMWR", - "pangaea.repository": "https://doi.org/10.17616/R3XS37", - "pds.data": "https://doi.org/10.17616/R31NJMJB", - "phbi.repo": "https://doi.org/10.17616/R3PC7F", - "psc.bil": "https://doi.org/10.17616/R31NJN9G", - "psu.datacom": "https://doi.org/10.17616/R31NJMGV", - "psu.scholar": "https://doi.org/10.17616/R34W4H", - "pu.dataspace": "https://doi.org/10.17616/R33B6F", - "purdue.purduelib": "https://doi.org/10.17616/R3V90N", - "rg.rg": "https://doi.org/10.17616/R36H16", - "rice.kinder": "https://doi.org/10.17616/R3GB90", - "sagebio.synapse": "https://doi.org/10.17616/R3B934", - "si.cda": "https://doi.org/10.17616/R3201S", - "si.si": "https://doi.org/10.17616/R3W49N", - "sml.tcia": "https://doi.org/10.17616/R3NH0V", - "sml.tdar": "https://doi.org/10.17616/R3HK56", - "snd.bolin": "https://doi.org/10.17616/R3PP99", - "snd.icos": "https://doi.org/10.17616/R3235B", - "snd.smhi": "https://doi.org/10.17616/R3R05H", - "snd.snd": "https://doi.org/10.17616/R3CS38", - "snd.su": "https://doi.org/10.17616/R3N648", - "stsci.mast": "https://doi.org/10.17616/R3K897", - "subgoe.dariah": "https://doi.org/10.17616/R30G8N", - "sul.openneuro": "https://doi.org/10.17616/R33047", - "sul.sdr": "https://doi.org/10.17616/R3RK6R", - "sul.simtk": "https://doi.org/10.17616/R3QJ4B", - "tib.bafg": "https://doi.org/10.17616/R3H35C", - "tib.beilst": "https://doi.org/10.17616/R3536N", - "tib.bikf": "https://doi.org/10.17616/R3GM23", - "tib.floss": "https://doi.org/10.17616/R3K905", - "tib.gfz": "https://doi.org/10.17616/R3VQ0S", - "tib.hzdr": "https://doi.org/10.17616/R3BR40", - "tib.inp": "https://doi.org/10.17616/R31NJMM8", - "tib.ipk": "https://doi.org/10.17616/R3CP9H", - "tib.kit-imk": "https://doi.org/10.17616/R3N347", - "tib.kit-ioc": "https://doi.org/10.17616/R34P5T", - "tib.ldeo": "https://doi.org/10.17616/R3PV10", - "tib.luis": "https://doi.org/10.17616/R3C519", - "tib.radar": "https://doi.org/10.17616/R3ZX96", - "tib.tuhh": "https://doi.org/10.17616/R31NJML0", - "tib.ub-hro": "https://doi.org/10.17616/R31NJMQH", - "tib.ubbs": "https://doi.org/10.17616/R31NJMTU", - "tib.ukon": "https://doi.org/10.17616/R3M024", - "tuw.tethys": "https://doi.org/10.17616/R31NJMU0", - "uchile.dataverse": "https://doi.org/10.17616/R31NJMLW", - "udr.researchdata": "https://doi.org/10.17616/R31NJMOS", - "ujcm.zntzqv": "https://doi.org/10.17616/R31NJNDH", - "umn.drum": "https://doi.org/10.17616/R30D19", - "umn.ipums": "https://doi.org/10.17616/R3ZS59", - "unimelb.repo1": "https://doi.org/10.17616/R3HH10", - "unm.dataone": "https://doi.org/10.17616/R3101G", - "usda.usda": "https://doi.org/10.17616/R3G051", - "usgs.prod": "https://doi.org/10.17616/R33S3S", - "vivli.search": "https://doi.org/10.17616/R3SB9S", - "vt.vtechdata": "https://doi.org/10.17616/R3JF54", - "wh.whoas": "https://doi.org/10.17616/R34302", - "wwpx.intr2": "https://doi.org/10.17616/R37M1J", - "ykdk.zuysqi": "https://doi.org/10.17616/R3XP4K", - "zbmed.biofresh": "https://doi.org/10.17616/R3DG9R", - "zbmed.dsmz": "https://doi.org/10.17616/R3G88X", - "zbw.ifo": "https://doi.org/10.17616/R3P01C", - "zbw.zbw-jda": "https://doi.org/10.17616/R3BM03", - "zbw.zew": "https://doi.org/10.17616/R3389W" -} diff --git a/fuji_server/data/repodois.yaml b/fuji_server/data/repodois.yaml new file mode 100644 index 00000000..594c844d --- /dev/null +++ b/fuji_server/data/repodois.yaml @@ -0,0 +1,253 @@ +arch.avanjj: https://doi.org/10.17616/R31NJMYW +ardcx.aims: https://doi.org/10.17616/R3X908 +ardcx.curtin: https://doi.org/10.17616/R3WM02 +ardcx.griffith: https://doi.org/10.17616/R3FG92 +ardcx.usq: https://doi.org/10.17616/R36Q18 +arfm.ufzdrp: https://doi.org/10.17616/R31NJN1M +awzy.fednnv: https://doi.org/10.17616/R3PM1K +bamf.bamf: https://doi.org/10.17616/R31NJMZ5 +bf.discover: https://doi.org/10.17616/R31NJMN0 +bibsys.uit-ord: https://doi.org/10.17616/R3TV17 +bkmx.azjwzc: https://doi.org/10.17616/R3VK72 +bl.ads: https://doi.org/10.17616/R3MW23 +bl.bath: https://doi.org/10.17616/R32928 +bl.birkbeck: https://doi.org/10.17616/R31NJLW9 +bl.bristol: https://doi.org/10.17616/R3CP84 +bl.brookes: https://doi.org/10.17616/R31NJMGX +bl.brunel: https://doi.org/10.17616/R3DP9T +bl.cam: https://doi.org/10.17616/R3SW4D +bl.cran: https://doi.org/10.17616/R3MH0J +bl.dri: https://doi.org/10.17616/R3W632 +bl.dundee: https://doi.org/10.17616/R31NJN0E +bl.ecmwf: https://doi.org/10.17616/R3WS8T +bl.ed: https://doi.org/10.17616/R3RG6Q +bl.essex: https://doi.org/10.17616/R3SV0J +bl.exeter: https://doi.org/10.17616/R3033Z +bl.hallam: https://doi.org/10.17616/R3S92S +bl.herts: https://doi.org/10.17616/R31NJMM4 +bl.iita: https://doi.org/10.17616/R3K22H +bl.imperial: http://doi.org/10.17616/R3K64N +bl.lboro: https://doi.org/10.17616/R3564X +bl.leeds: https://doi.org/10.17616/R3JH19 +bl.lincoln: https://doi.org/10.17616/R3MZ01 +bl.lshtm: https://doi.org/10.17616/R3705V +bl.mendeley: https://doi.org/10.17616/R3DD11 +bl.nhm: https://doi.org/10.17616/R3PS7K +bl.oxdb: https://doi.org/10.17616/R3504V +bl.reading: https://doi.org/10.17616/R3D075 +bl.salford: https://doi.org/10.17616/R31D20 +bl.shef: https://doi.org/10.17616/R3P64K +bl.soton: https://doi.org/10.17616/R3H05B +bl.standrew: https://doi.org/10.17616/R3878Q +bl.strath: https://doi.org/10.17616/R3G18H +bl.surrey: https://doi.org/10.17616/R34N3Q +bl.uclan: https://doi.org/10.17616/R38D3J +bl.ucld: https://doi.org/10.17616/R34F41 +bl.uel: https://doi.org/10.17616/R3V48Z +bl.ukda: https://doi.org/10.17616/R3088K +brown.bdr: https://doi.org/10.17616/R3193B +brvz.rdr: https://doi.org/10.17616/R31NJN5S +caltech.data: https://doi.org/10.17616/R3SW99 +carl.frdr: https://doi.org/10.17616/R3X50W +cbg.datasets: https://doi.org/10.17616/R3PP7J +ccdc.csd: https://doi.org/10.17616/R36011 +cern.hepdata: https://doi.org/10.17616/R30W2H +cern.inspire: https://doi.org/10.17616/R3JC9Z +cern.zenodo: https://doi.org/10.17616/R3QP53 +clarin.clarin: https://doi.org/10.17616/R3RP5D +cngb.gigadb: https://doi.org/10.17616/R3TG83 +cnic.sciencedb: https://doi.org/10.17616/R31NJMNT +concor.kcydcu: https://doi.org/10.17616/R3FJ70 +cornell.ciser: https://doi.org/10.17616/R3ZK70 +cornell.library: https://doi.org/10.17616/R3FP94 +cornell.ndacan: https://doi.org/10.17616/R3N90S +cos.osf: https://doi.org/10.17616/R3N03T +crui.ogsts: https://doi.org/10.17616/R3TR3Z +crui.unibo: https://doi.org/10.17616/R3P19R +crui.unipd: https://doi.org/10.17616/R31NJMHN +csc.nrd: https://doi.org/10.17616/R33649 +csic.digital: https://doi.org/10.17616/R3P34J +cul.ciesin: https://doi.org/10.17616/R31605 +dafi.client: https://doi.org/10.17616/R3QS3C +dans.archive: https://doi.org/10.17616/R3401D +dans.dataversenl: https://doi.org/10.17616/R33W6Z +dartlib.crawdad: https://doi.org/10.17616/R3CD0B +delft.data4tu: https://doi.org/10.17616/R3VG6N +delft.isric: https://doi.org/10.17616/R3X01J +delft.knmi: https://doi.org/10.17616/R3QH0G +delft.maastro: https://doi.org/10.17616/R38W4R +delft.ru: https://doi.org/10.17616/R31NJMZR +delft.surfsara: https://doi.org/10.17616/R31NJML8 +delft.uu: https://doi.org/10.17616/R3ZB7K +delft.vliz: https://doi.org/10.17616/R3WG7B +dipf.rdcedu: https://doi.org/10.17616/R3R010 +dk.dtic: https://doi.org/10.17616/R31NJMWG +dk.sb: https://doi.org/10.17616/R31NJM89 +dkrz.esgf: https://doi.org/10.17616/R3BK7D +dkrz.wdcc: https://doi.org/10.17616/R3989R +dryad.dryad: https://doi.org/10.17616/R34S33 +dzhw.fdz-dzhw: https://doi.org/10.17616/R3BR0G +elsevier.md: https://doi.org/10.17616/R3DD11 +esdis.ornl: https://doi.org/10.17616/R3SG61 +estdoi.keel: https://doi.org/10.17616/R3P636 +estdoi.qdb: https://doi.org/10.17616/R31NJMVF +estdoi.repo: https://doi.org/10.17616/R30D32 +ethz.da-rd: https://doi.org/10.17616/R34D05 +ethz.e-coll: https://doi.org/10.17616/R3HF66 +ethz.sicas: https://doi.org/10.17616/R3HP8C +ethz.wgms: https://doi.org/10.17616/R3WP5N +ethz.wsl: https://doi.org/10.17616/R3G49X +europ.odin: https://doi.org/10.17616/R3J917 +euvi.seanoe: https://doi.org/10.17616/R3J33X +fbtk.dmxfnr: https://doi.org/10.17616/R31NJMV8 +fct.uminho: https://doi.org/10.17616/R31NJMNP +fhstp.phaidra: https://doi.org/10.17616/R31NJMSE +figshare.ariz: https://doi.org/10.17616/R31NJMSF +figshare.ars: https://doi.org/10.17616/R3PK5R +figshare.iastate: https://doi.org/10.17616/R3GV3W +figshare.uct: https://doi.org/10.17616/R36R4R +fzj.b2share: https://doi.org/10.17616/R3VK72 +fzj.tereno: https://doi.org/10.17616/R39G9T +gbif.ec: https://doi.org/10.17616/R31NJNFS +gbif.gbif: https://doi.org/10.17616/R3J014 +gdcc.csuc: https://doi.org/10.17616/R31NJMYF +gdcc.harvard-dv: https://doi.org/10.17616/R3C880 +gdcc.harvard-sbgr: https://doi.org/10.17616/R3N92J +gdcc.harvard-slp: https://doi.org/10.17616/R3XD09 +gdcc.icraf: https://doi.org/10.17616/R3V042 +gdcc.jhu: https://doi.org/10.17616/R3RW77 +gdcc.ntu: https://doi.org/10.17616/R3N77R +gdcc.odum-dv: https://doi.org/10.17616/R3RP41 +gdcc.odum-library: https://doi.org/10.17616/R3RP41 +gdcc.syr-qdr: https://doi.org/10.17616/R3CS51 +gesis.adp: https://doi.org/10.17616/R3SG7D +gesis.aussda: https://doi.org/10.17616/R39G72 +gesis.bibb-fdz: https://doi.org/10.17616/R3359V +gesis.csda: https://doi.org/10.17616/R35P6H +gesis.dezim: https://doi.org/10.17616/R31NJMNK +gesis.gesis: https://doi.org/10.17616/R3BB70 +gesis.icpsr: https://doi.org/10.17616/R3BC8Q +gesis.indepth: https://doi.org/10.17616/R3432T +gesis.neps: https://doi.org/10.17616/R35P7W +gesis.rki: https://doi.org/10.17616/R3K31G +gesis.share: https://doi.org/10.17616/R3BK5N +gesis.sodanet: https://doi.org/10.17616/R38G7R +gesis.srda: https://doi.org/10.17616/R31NJMLA +gesis.ssri: https://doi.org/10.17616/R31NJMWM +gesis.zpid: https://doi.org/10.17616/R3FS48 +hain.njltcl: https://doi.org/10.17616/R31NJNAM +heliophy.spdf: https://doi.org/10.17616/R3P301 +hlqc.znxeli: https://doi.org/10.17616/R31NJMJP +ieee.dataport: https://doi.org/10.17616/R3H77H +iiasa.dare: https://doi.org/10.17616/R31NJMH0 +illinois.databank: https://doi.org/10.17616/R3C93F +inist.adisp: https://doi.org/10.17616/R3W90Z +inist.cds: https://doi.org/10.17616/R3690R +inist.humanum: https://doi.org/10.17616/R3JM1B +inist.ifremer: https://doi.org/10.17616/R31NJMHX +inist.ifsttar: https://doi.org/10.17616/R31NJMKM +inist.ill: https://doi.org/10.17616/R33H18 +inist.inra: https://doi.org/10.17616/R3DJ4J +inist.otelo: https://doi.org/10.17616/R3F19K +inist.resif: https://doi.org/10.17616/R37Q06 +iris.iris: https://doi.org/10.17616/R3X607 +ist.rex: https://doi.org/10.17616/R3877B +ivuw.dadosipb: https://doi.org/10.17616/R31NJN3K +ivuw.ipbdados: https://doi.org/10.17616/R31NJN3K +jbru.bbees: https://doi.org/10.17616/R31NJNEN +jcvi.eivbwb: https://doi.org/10.17616/R30P93 +jcvi.gxpwaq: https://doi.org/10.17616/R3S634 +kaggle.kaggle: https://doi.org/10.17616/R32N5V +kim.openkim: https://doi.org/10.17616/R3SK8T +kmot.rdr: https://doi.org/10.17616/R31NJMN6 +ktsw.aezvvv: https://doi.org/10.17616/R31NJMVX +landcare.general: https://doi.org/10.17616/R3092N +landcare.soils: https://doi.org/10.17616/R3660Q +ldc.catalog: https://doi.org/10.17616/R3T06H +lmu.ub: https://doi.org/10.17616/R3PP65 +lxkc.dskyfi: https://doi.org/10.17616/R3W331 +mdw.repository: https://doi.org/10.17616/R30M00 +mit.physio: https://doi.org/10.17616/R3D06S +mlbs.skuxgs: https://doi.org/10.17616/R3ZP8D +nasapds.nasapds: https://doi.org/10.17616/R37593 +nkn.nkn: https://doi.org/10.17616/R3JK91 +nlqq.xoncsf: https://doi.org/10.17616/R31NJMYB +noaa.ncei: https://doi.org/10.17616/R3362J +nsidc.datactr: https://doi.org/10.17616/R3HP4V +nu.arch: https://doi.org/10.17616/R31NJMGT +nus.sb: https://doi.org/10.17616/R3FF4T +nzau.data: https://doi.org/10.17616/R3BW7H +ocean.ocean: https://doi.org/10.17616/R38F5N +ocul.spdv: https://doi.org/10.17616/R32326 +ohsu.ohsu: https://doi.org/10.17616/R3MS9Q +oibk.ohycfa: https://doi.org/10.17616/R31NJMWR +pangaea.repository: https://doi.org/10.17616/R3XS37 +pds.data: https://doi.org/10.17616/R31NJMJB +phbi.repo: https://doi.org/10.17616/R3PC7F +psc.bil: https://doi.org/10.17616/R31NJN9G +psu.datacom: https://doi.org/10.17616/R31NJMGV +psu.scholar: https://doi.org/10.17616/R34W4H +pu.dataspace: https://doi.org/10.17616/R33B6F +purdue.purduelib: https://doi.org/10.17616/R3V90N +rg.rg: https://doi.org/10.17616/R36H16 +rice.kinder: https://doi.org/10.17616/R3GB90 +sagebio.synapse: https://doi.org/10.17616/R3B934 +si.cda: https://doi.org/10.17616/R3201S +si.si: https://doi.org/10.17616/R3W49N +sml.tcia: https://doi.org/10.17616/R3NH0V +sml.tdar: https://doi.org/10.17616/R3HK56 +snd.bolin: https://doi.org/10.17616/R3PP99 +snd.icos: https://doi.org/10.17616/R3235B +snd.smhi: https://doi.org/10.17616/R3R05H +snd.snd: https://doi.org/10.17616/R3CS38 +snd.su: https://doi.org/10.17616/R3N648 +stsci.mast: https://doi.org/10.17616/R3K897 +subgoe.dariah: https://doi.org/10.17616/R30G8N +sul.openneuro: https://doi.org/10.17616/R33047 +sul.sdr: https://doi.org/10.17616/R3RK6R +sul.simtk: https://doi.org/10.17616/R3QJ4B +tib.bafg: https://doi.org/10.17616/R3H35C +tib.beilst: https://doi.org/10.17616/R3536N +tib.bikf: https://doi.org/10.17616/R3GM23 +tib.floss: https://doi.org/10.17616/R3K905 +tib.gfz: https://doi.org/10.17616/R3VQ0S +tib.hzdr: https://doi.org/10.17616/R3BR40 +tib.igb: https://doi.org/10.17616/R31NJMKS +tib.inp: https://doi.org/10.17616/R31NJMM8 +tib.ipk: https://doi.org/10.17616/R3CP9H +tib.kit-imk: https://doi.org/10.17616/R3N347 +tib.kit-ioc: https://doi.org/10.17616/R34P5T +tib.kit4radar: https://doi.org/10.17616/R31NJMVU +tib.ldeo: https://doi.org/10.17616/R3PV10 +tib.luis: https://doi.org/10.17616/R3C519 +tib.radar: https://doi.org/10.17616/R3ZX96 +tib.tuhh: https://doi.org/10.17616/R31NJML0 +tib.ub-hro: https://doi.org/10.17616/R31NJMQH +tib.ubbs: https://doi.org/10.17616/R31NJMTU +tib.ukon: https://doi.org/10.17616/R3M024 +tib.zih: https://doi.org/10.17616/R31NJMHY +tuw.tethys: https://doi.org/10.17616/R31NJMU0 +uchile.dataverse: https://doi.org/10.17616/R31NJMLW +udr.researchdata: https://doi.org/10.17616/R31NJMOS +ujcm.zntzqv: https://doi.org/10.17616/R31NJNDH +umn.drum: https://doi.org/10.17616/R30D19 +umn.ipums: https://doi.org/10.17616/R3ZS59 +undr.undr: https://doi.org/10.17616/R31NJN9K +unimelb.repo1: https://doi.org/10.17616/R3HH10 +unm.dataone: https://doi.org/10.17616/R3101G +usda.usda: https://doi.org/10.17616/R3G051 +usgs.prod: https://doi.org/10.17616/R33S3S +uva.libra: https://doi.org/10.17616/R3TS86 +vivli.search: https://doi.org/10.17616/R3SB9S +vt.vtechdata: https://doi.org/10.17616/R3JF54 +wh.whoas: https://doi.org/10.17616/R34302 +wuua.vozxlg: https://doi.org/10.17616/R31NJNEM +wwgp.myyddx: https://doi.org/10.17616/R31NJNDK +wwpx.intr2: https://doi.org/10.17616/R37M1J +ykdk.zuysqi: https://doi.org/10.17616/R3XP4K +zbmed.biofresh: https://doi.org/10.17616/R3DG9R +zbmed.dsmz: https://doi.org/10.17616/R31NJMKK +zbw.ifo: https://doi.org/10.17616/R3P01C +zbw.zbw-jda: https://doi.org/10.17616/R3BM03 +zbw.zew: https://doi.org/10.17616/R3389W +zyhg.elabour: https://doi.org/10.17616/R31NJMGQ diff --git a/fuji_server/helper/preprocessor.py b/fuji_server/helper/preprocessor.py index f318353b..a8190dad 100644 --- a/fuji_server/helper/preprocessor.py +++ b/fuji_server/helper/preprocessor.py @@ -234,7 +234,7 @@ def retrieve_metrics_yaml(cls, yaml_metric_path): def retrieve_datacite_re3repos(cls): # retrieve all client id and re3data doi from datacite isDebugMode = True - re3dict_path = os.path.join(cls.fuji_server_dir, "data", "repodois.json") + re3dict_path = os.path.join(cls.fuji_server_dir, "data", "repodois.yaml") repolistdate = os.path.getmtime(re3dict_path) try: # update once a day @@ -244,7 +244,7 @@ def retrieve_datacite_re3repos(cls): pass if isDebugMode: with open(re3dict_path) as f: - cls.re3repositories = json.load(f) + cls.re3repositories = yaml.safe_load(f) else: print("updating re3data dois") p = {"query": "re3data_id:*"} @@ -261,7 +261,8 @@ def retrieve_datacite_re3repos(cls): # fix wrong entry cls.re3repositories["bl.imperial"] = "http://doi.org/10.17616/R3K64N" with open(re3dict_path, "w") as f2: - json.dump(cls.re3repositories, f2) + yaml.dump(cls.re3repositories, f2) + except requests.exceptions.RequestException as e: print("Preprocessor Error: " + str(e)) cls.logger.error(e) diff --git a/tests/helper/test_preprocessor.py b/tests/helper/test_preprocessor.py index be176d15..f17dc624 100644 --- a/tests/helper/test_preprocessor.py +++ b/tests/helper/test_preprocessor.py @@ -19,6 +19,7 @@ from typing import Any, Dict, List import pytest +import yaml from fuji_server.helper.preprocessor import Preprocessor from tests.conftest import DATA_DIR @@ -31,6 +32,11 @@ def load_json_from_data_directory(filename: str): return json.loads(path.read_text()) +def load_yaml_from_data_directory(filename: str): + path = DATA_DIR.joinpath(filename) + return yaml.safe_load(path.read_text()) + + def load_txt_from_data_directory(filename: str): path = DATA_DIR.joinpath(filename) return path.read_text() @@ -48,7 +54,7 @@ def metadata_standards(): @pytest.fixture(scope="session") def repodois(): - return load_json_from_data_directory("repodois.json") + return load_yaml_from_data_directory("repodois.yaml") @pytest.fixture(scope="session")