From 440c0bef2aa866226fd0e67269d52a065ae8ccc2 Mon Sep 17 00:00:00 2001 From: Gavin Date: Tue, 17 Dec 2024 15:13:34 -0800 Subject: [PATCH] Generate manifest files for job submission Will be downloaded to NERSC in later steps Also sorted the Pipfile --- Pipfile | 14 +- Pipfile.lock | 406 +++++++++++++------------- cdmtaskservice/app_state.py | 5 +- cdmtaskservice/jobflows/nersc_jaws.py | 47 ++- cdmtaskservice/models.py | 23 ++ cdmtaskservice/routes.py | 2 +- 6 files changed, 275 insertions(+), 222 deletions(-) diff --git a/Pipfile b/Pipfile index da84553..56f317f 100644 --- a/Pipfile +++ b/Pipfile @@ -6,21 +6,21 @@ name = "pypi" [packages] aiobotocore = "==2.15.2" aiohttp = "==3.11.10" -sfapi-client = "==0.3.1" -requests = "==2.32.3" +bidict = "==0.23.1" +cacheout = "==0.16.0" docker-image-py = "==0.1.13" fastapi = {extras = ["standard"], version = "==0.115.6"} -marisa-trie = "==1.2.1" -bidict = "==0.23.1" -uvicorn = {extras = ["standard"], version = "==0.32.1"} jinja-cli = "==1.2.2" -cacheout = "==0.16.0" +marisa-trie = "==1.2.1" motor = "==3.6.0" +requests = "==2.32.3" +sfapi-client = "==0.3.1" +uvicorn = {extras = ["standard"], version = "==0.32.1"} [dev-packages] -pytest = "==8.3.4" coverage = "==7.6.9" ipython = "==8.30.0" +pytest = "==8.3.4" pytest-asyncio = "==0.24.0" pytest-cov = "==6.0.0" diff --git a/Pipfile.lock b/Pipfile.lock index 5746a5b..9a7176b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "af7e6f455fccb78e3cd9a0ceabdcf862baa6d4db5b8038d5174ad2a5b237d027" + "sha256": "46e4a1f9a38b215bf752bd03c2f19434ab03076f4393316077c4819252a7bea4" }, "pipfile-spec": 6, "requires": { @@ -124,11 +124,11 @@ }, "aiosignal": { "hashes": [ - "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc", - "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17" + "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", + "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54" ], - "markers": "python_version >= '3.7'", - "version": "==1.3.1" + "markers": "python_version >= '3.9'", + "version": "==1.3.2" }, "annotated-types": { "hashes": [ @@ -155,11 +155,11 @@ }, "attrs": { "hashes": [ - "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", - "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff", + "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308" ], - "markers": "python_version >= '3.7'", - "version": "==24.2.0" + "markers": "python_version >= '3.8'", + "version": "==24.3.0" }, "authlib": { "hashes": [ @@ -195,11 +195,11 @@ }, "certifi": { "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", + "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db" ], "markers": "python_version >= '3.6'", - "version": "==2024.8.30" + "version": "==2024.12.14" }, "cffi": { "hashes": [ @@ -395,36 +395,36 @@ }, "cryptography": { "hashes": [ - "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362", - "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4", - "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa", - "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83", - "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff", - "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805", - "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6", - "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664", - "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08", - "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e", - "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18", - "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f", - "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73", - "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5", - "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984", - "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd", - "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3", - "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e", - "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405", - "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2", - "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c", - "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995", - "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73", - "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16", - "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7", - "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd", - "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7" - ], - "markers": "python_version >= '3.7'", - "version": "==43.0.3" + "sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7", + "sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731", + "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b", + "sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc", + "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543", + "sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c", + "sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591", + "sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede", + "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb", + "sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f", + "sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123", + "sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c", + "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c", + "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285", + "sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd", + "sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092", + "sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa", + "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289", + "sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02", + "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64", + "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053", + "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417", + "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e", + "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e", + "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7", + "sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756", + "sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4" + ], + "markers": "python_version >= '3.7' and python_full_version not in '3.9.0, 3.9.1'", + "version": "==44.0.0" }, "dnspython": { "hashes": [ @@ -465,10 +465,10 @@ "standard" ], "hashes": [ - "sha256:2835a8f0c44b68e464d5cafe5ec205265f02dc1ad1d640db33a994ba3338003b", - "sha256:43288efee46338fae8902f9bf4559aed3aed639f9516f5d394a7ff19edcc8faf" + "sha256:02b3b65956f526412515907a0793c9094abd4bfb5457b389f645b0ea6ba3605e", + "sha256:d549368ff584b2804336c61f192d86ddea080c11255f375959627911944804f4" ], - "version": "==0.0.6" + "version": "==0.0.7" }, "frozenlist": { "hashes": [ @@ -1155,11 +1155,11 @@ }, "pydantic-settings": { "hashes": [ - "sha256:7fb0637c786a558d3103436278a7c4f1cfd29ba8973238a50c5bb9a55387da87", - "sha256:e0f92546d8a9923cb8941689abf85d6601a8c19a23e97a34b2964a2e3f813ca0" + "sha256:ac4bfd4a36831a48dbf8b2d9325425b549a0a6f18cea118436d728eb4f1c4d66", + "sha256:e00c05d5fa6cbbb227c84bd7487c5c1065084119b750df7c8c1a554aed236eb5" ], "markers": "python_version >= '3.8'", - "version": "==2.6.1" + "version": "==2.7.0" }, "pygments": { "hashes": [ @@ -1252,10 +1252,10 @@ }, "python-multipart": { "hashes": [ - "sha256:905502ef39050557b7a6af411f454bc19526529ca46ae6831508438890ce12cc", - "sha256:f8d5b0b9c618575bf9df01c684ded1d94a338839bdd8223838afacfb4bb2082d" + "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", + "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13" ], - "version": "==0.0.19" + "version": "==0.0.20" }, "pyyaml": { "hashes": [ @@ -1466,11 +1466,11 @@ }, "six": { "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" + "version": "==1.17.0" }, "sniffio": { "hashes": [ @@ -1575,79 +1575,79 @@ }, "watchfiles": { "hashes": [ - "sha256:06d828fe2adc4ac8a64b875ca908b892a3603d596d43e18f7948f3fef5fc671c", - "sha256:074c7618cd6c807dc4eaa0982b4a9d3f8051cd0b72793511848fd64630174b17", - "sha256:09551237645d6bff3972592f2aa5424df9290e7a2e15d63c5f47c48cde585935", - "sha256:0fc3bf0effa2d8075b70badfdd7fb839d7aa9cea650d17886982840d71fdeabf", - "sha256:12ab123135b2f42517f04e720526d41448667ae8249e651385afb5cda31fedc0", - "sha256:13a4f9ee0cd25682679eea5c14fc629e2eaa79aab74d963bc4e21f43b8ea1877", - "sha256:1d19df28f99d6a81730658fbeb3ade8565ff687f95acb59665f11502b441be5f", - "sha256:1e176b6b4119b3f369b2b4e003d53a226295ee862c0962e3afd5a1c15680b4e3", - "sha256:1ee5edc939f53466b329bbf2e58333a5461e6c7b50c980fa6117439e2c18b42d", - "sha256:1f73c2147a453315d672c1ad907abe6d40324e34a185b51e15624bc793f93cc6", - "sha256:1ff236d7a3f4b0a42f699a22fc374ba526bc55048a70cbb299661158e1bb5e1f", - "sha256:245fab124b9faf58430da547512d91734858df13f2ddd48ecfa5e493455ffccb", - "sha256:28babb38cf2da8e170b706c4b84aa7e4528a6fa4f3ee55d7a0866456a1662041", - "sha256:28fb64b5843d94e2c2483f7b024a1280662a44409bedee8f2f51439767e2d107", - "sha256:29cf884ad4285d23453c702ed03d689f9c0e865e3c85d20846d800d4787de00f", - "sha256:2a825ba4b32c214e3855b536eb1a1f7b006511d8e64b8215aac06eb680642d84", - "sha256:2ac778a460ea22d63c7e6fb0bc0f5b16780ff0b128f7f06e57aaec63bd339285", - "sha256:2c2696611182c85eb0e755b62b456f48debff484b7306b56f05478b843ca8ece", - "sha256:2d9c0518fabf4a3f373b0a94bb9e4ea7a1df18dec45e26a4d182aa8918dee855", - "sha256:2de52b499e1ab037f1a87cb8ebcb04a819bf087b1015a4cf6dcf8af3c2a2613e", - "sha256:37566c844c9ce3b5deb964fe1a23378e575e74b114618d211fbda8f59d7b5dab", - "sha256:3d94fd83ed54266d789f287472269c0def9120a2022674990bd24ad989ebd7a0", - "sha256:48051d1c504448b2fcda71c5e6e3610ae45de6a0b8f5a43b961f250be4bdf5a8", - "sha256:487d15927f1b0bd24e7df921913399bb1ab94424c386bea8b267754d698f8f0e", - "sha256:4a3b33c3aefe9067ebd87846806cd5fc0b017ab70d628aaff077ab9abf4d06b3", - "sha256:4ff9c7e84e8b644a8f985c42bcc81457240316f900fc72769aaedec9d088055a", - "sha256:533a7cbfe700e09780bb31c06189e39c65f06c7f447326fee707fd02f9a6e945", - "sha256:53ae447f06f8f29f5ab40140f19abdab822387a7c426a369eb42184b021e97eb", - "sha256:550109001920a993a4383b57229c717fa73627d2a4e8fcb7ed33c7f1cddb0c85", - "sha256:5bbd0311588c2de7f9ea5cf3922ccacfd0ec0c1922870a2be503cc7df1ca8be7", - "sha256:5dccfc70480087567720e4e36ec381bba1ed68d7e5f368fe40c93b3b1eba0105", - "sha256:5f75cd42e7e2254117cf37ff0e68c5b3f36c14543756b2da621408349bd9ca7c", - "sha256:648e2b6db53eca6ef31245805cd528a16f56fa4cc15aeec97795eaf713c11435", - "sha256:774ef36b16b7198669ce655d4f75b4c3d370e7f1cbdfb997fb10ee98717e2058", - "sha256:8a2127cd68950787ee36753e6d401c8ea368f73beaeb8e54df5516a06d1ecd82", - "sha256:90004553be36427c3d06ec75b804233f8f816374165d5225b93abd94ba6e7234", - "sha256:905f69aad276639eff3893759a07d44ea99560e67a1cf46ff389cd62f88872a2", - "sha256:9122b8fdadc5b341315d255ab51d04893f417df4e6c1743b0aac8bf34e96e025", - "sha256:9272fdbc0e9870dac3b505bce1466d386b4d8d6d2bacf405e603108d50446940", - "sha256:936f362e7ff28311b16f0b97ec51e8f2cc451763a3264640c6ed40fb252d1ee4", - "sha256:947ccba18a38b85c366dafeac8df2f6176342d5992ca240a9d62588b214d731f", - "sha256:95dc785bc284552d044e561b8f4fe26d01ab5ca40d35852a6572d542adfeb4bc", - "sha256:95de85c254f7fe8cbdf104731f7f87f7f73ae229493bebca3722583160e6b152", - "sha256:9b4fb98100267e6a5ebaff6aaa5d20aea20240584647470be39fe4823012ac96", - "sha256:9c01446626574561756067f00b37e6b09c8622b0fc1e9fdbc7cbcea328d4e514", - "sha256:9c9a8d8fd97defe935ef8dd53d562e68942ad65067cd1c54d6ed8a088b1d931d", - "sha256:9e1d9284cc84de7855fcf83472e51d32daf6f6cecd094160192628bc3fee1b78", - "sha256:a0abf173975eb9dd17bb14c191ee79999e650997cc644562f91df06060610e62", - "sha256:a2218e78e2c6c07b1634a550095ac2a429026b2d5cbcd49a594f893f2bb8c936", - "sha256:a5a7a06cfc65e34fd0a765a7623c5ba14707a0870703888e51d3d67107589817", - "sha256:b2bca898c1dc073912d3db7fa6926cc08be9575add9e84872de2c99c688bac4e", - "sha256:b46e15c34d4e401e976d6949ad3a74d244600d5c4b88c827a3fdf18691a46359", - "sha256:b551c465a59596f3d08170bd7e1c532c7260dd90ed8135778038e13c5d48aa81", - "sha256:b555a93c15bd2c71081922be746291d776d47521a00703163e5fbe6d2a402399", - "sha256:bc338ce9f8846543d428260fa0f9a716626963148edc937d71055d01d81e1525", - "sha256:bedf84835069f51c7b026b3ca04e2e747ea8ed0a77c72006172c72d28c9f69fc", - "sha256:c3d258d78341d5d54c0c804a5b7faa66cd30ba50b2756a7161db07ce15363b8d", - "sha256:c83a6d33a9eda0af6a7470240d1af487807adc269704fe76a4972dd982d16236", - "sha256:c9a13ac46b545a7d0d50f7641eefe47d1597e7d1783a5d89e09d080e6dff44b0", - "sha256:cf517701a4a872417f4e02a136e929537743461f9ec6cdb8184d9a04f4843545", - "sha256:d2b39aa8edd9e5f56f99a2a2740a251dc58515398e9ed5a4b3e5ff2827060755", - "sha256:d3572d4c34c4e9c33d25b3da47d9570d5122f8433b9ac6519dca49c2740d23cd", - "sha256:d562a6114ddafb09c33246c6ace7effa71ca4b6a2324a47f4b09b6445ea78941", - "sha256:e1ed613ee107269f66c2df631ec0fc8efddacface85314d392a4131abe299f00", - "sha256:e3750434c83b61abb3163b49c64b04180b85b4dabb29a294513faec57f2ffdb7", - "sha256:eba98901a2eab909dbd79681190b9049acc650f6111fde1845484a4450761e98", - "sha256:f159ac795785cde4899e0afa539f4c723fb5dd336ce5605bc909d34edd00b79b", - "sha256:f8c4f3a1210ed099a99e6a710df4ff2f8069411059ffe30fa5f9467ebed1256b", - "sha256:fa13d604fcb9417ae5f2e3de676e66aa97427d888e83662ad205bed35a313176", - "sha256:fbd0ab7a9943bbddb87cbc2bf2f09317e74c77dc55b1f5657f81d04666c25269", - "sha256:ffd98a299b0a74d1b704ef0ed959efb753e656a4e0425c14e46ae4c3cbdd2919" - ], - "version": "==1.0.0" + "sha256:0179252846be03fa97d4d5f8233d1c620ef004855f0717712ae1c558f1974a16", + "sha256:06ce08549e49ba69ccc36fc5659a3d0ff4e3a07d542b895b8a9013fcab46c2dc", + "sha256:0b90651b4cf9e158d01faa0833b073e2e37719264bcee3eac49fc3c74e7d304b", + "sha256:0d1ec043f02ca04bf21b1b32cab155ce90c651aaf5540db8eb8ad7f7e645cba8", + "sha256:0fe4e740ea94978b2b2ab308cbf9270a246bcbb44401f77cc8740348cbaeac3d", + "sha256:127de3883bdb29dbd3b21f63126bb8fa6e773b74eaef46521025a9ce390e1073", + "sha256:1550be1a5cb3be08a3fb84636eaafa9b7119b70c71b0bed48726fd1d5aa9b868", + "sha256:160eff7d1267d7b025e983ca8460e8cc67b328284967cbe29c05f3c3163711a3", + "sha256:16db2d7e12f94818cbf16d4c8938e4d8aaecee23826344addfaaa671a1527b07", + "sha256:1c6cf7709ed3e55704cc06f6e835bf43c03bc8e3cb8ff946bf69a2e0a78d9d77", + "sha256:1da46bb1eefb5a37a8fb6fd52ad5d14822d67c498d99bda8754222396164ae42", + "sha256:1df924ba82ae9e77340101c28d56cbaff2c991bd6fe8444a545d24075abb0a87", + "sha256:1e263cc718545b7f897baeac1f00299ab6fabe3e18caaacacb0edf6d5f35513c", + "sha256:228e2247de583475d4cebf6b9af5dc9918abb99d1ef5ee737155bb39fb33f3c0", + "sha256:275c1b0e942d335fccb6014d79267d1b9fa45b5ac0639c297f1e856f2f532552", + "sha256:29b9cb35b7f290db1c31fb2fdf8fc6d3730cfa4bca4b49761083307f441cac5a", + "sha256:2b4691234d31686dca133c920f94e478b548a8e7c750f28dbbc2e4333e0d3da9", + "sha256:2b961b86cd3973f5822826017cad7f5a75795168cb645c3a6b30c349094e02e3", + "sha256:2dcc3f60c445f8ce14156854a072ceb36b83807ed803d37fdea2a50e898635d6", + "sha256:2f492d2907263d6d0d52f897a68647195bc093dafed14508a8d6817973586b6b", + "sha256:310505ad305e30cb6c5f55945858cdbe0eb297fc57378f29bacceb534ac34199", + "sha256:34e87c7b3464d02af87f1059fedda5484e43b153ef519e4085fe1a03dd94801e", + "sha256:418c5ce332f74939ff60691e5293e27c206c8164ce2b8ce0d9abf013003fb7fe", + "sha256:46e86ed457c3486080a72bc837300dd200e18d08183f12b6ca63475ab64ed651", + "sha256:48681c86f2cb08348631fed788a116c89c787fdf1e6381c5febafd782f6c3b44", + "sha256:489b80812f52a8d8c7b0d10f0d956db0efed25df2821c7a934f6143f76938bd6", + "sha256:48c9f3bc90c556a854f4cab6a79c16974099ccfa3e3e150673d82d47a4bc92c9", + "sha256:49bc1bc26abf4f32e132652f4b3bfeec77d8f8f62f57652703ef127e85a3e38d", + "sha256:52bb50a4c4ca2a689fdba84ba8ecc6a4e6210f03b6af93181bb61c4ec3abaf86", + "sha256:5691340f259b8f76b45fb31b98e594d46c36d1dc8285efa7975f7f50230c9093", + "sha256:62691f1c0894b001c7cde1195c03b7801aaa794a837bd6eef24da87d1542838d", + "sha256:632a52dcaee44792d0965c17bdfe5dc0edad5b86d6a29e53d6ad4bf92dc0ff49", + "sha256:65ab1fb635476f6170b07e8e21db0424de94877e4b76b7feabfe11f9a5fc12b5", + "sha256:6a5bc3ca468bb58a2ef50441f953e1f77b9a61bd1b8c347c8223403dc9b4ac9a", + "sha256:6a76494d2c5311584f22416c5a87c1e2cb954ff9b5f0988027bc4ef2a8a67181", + "sha256:6f8dc09ae69af50bead60783180f656ad96bd33ffbf6e7a6fce900f6d53b08f1", + "sha256:703aa5e50e465be901e0e0f9d5739add15e696d8c26c53bc6fc00eb65d7b9469", + "sha256:713f67132346bdcb4c12df185c30cf04bdf4bf6ea3acbc3ace0912cab6b7cb8c", + "sha256:75d3bcfa90454dba8df12adc86b13b6d85fda97d90e708efc036c2760cc6ba44", + "sha256:7ca05cacf2e5c4a97d02a2878a24020daca21dbb8823b023b978210a75c79098", + "sha256:80bf4b459d94a0387617a1b499f314aa04d8a64b7a0747d15d425b8c8b151da0", + "sha256:84fac88278f42d61c519a6c75fb5296fd56710b05bbdcc74bdf85db409a03780", + "sha256:889a37e2acf43c377b5124166bece139b4c731b61492ab22e64d371cce0e6e80", + "sha256:8af4b582d5fc1b8465d1d2483e5e7b880cc1a4e99f6ff65c23d64d070867ac58", + "sha256:90b0fe1fcea9bd6e3084b44875e179b4adcc4057a3b81402658d0eb58c98edf8", + "sha256:93436ed550e429da007fbafb723e0769f25bae178fbb287a94cb4ccdf42d3af3", + "sha256:995c374e86fa82126c03c5b4630c4e312327ecfe27761accb25b5e1d7ab50ec8", + "sha256:9af037d3df7188ae21dc1c7624501f2f90d81be6550904e07869d8d0e6766655", + "sha256:9e080cf917b35b20c889225a13f290f2716748362f6071b859b60b8847a6aa43", + "sha256:a2ec98e31e1844eac860e70d9247db9d75440fc8f5f679c37d01914568d18721", + "sha256:abd85de513eb83f5ec153a802348e7a5baa4588b818043848247e3e8986094e8", + "sha256:ac1be85fe43b4bf9a251978ce5c3bb30e1ada9784290441f5423a28633a958a7", + "sha256:be37f9b1f8934cd9e7eccfcb5612af9fb728fecbe16248b082b709a9d1b348bf", + "sha256:bfcae6aecd9e0cb425f5145afee871465b98b75862e038d42fe91fd753ddd780", + "sha256:c05b021f7b5aa333124f2a64d56e4cb9963b6efdf44e8d819152237bbd93ba15", + "sha256:c14a07bdb475eb696f85c715dbd0f037918ccbb5248290448488a0b4ef201aad", + "sha256:c18f3502ad0737813c7dad70e3e1cc966cc147fbaeef47a09463bbffe70b0a00", + "sha256:c2e9fe695ff151b42ab06501820f40d01310fbd58ba24da8923ace79cf6d702d", + "sha256:c68be72b1666d93b266714f2d4092d78dc53bd11cf91ed5a3c16527587a52e29", + "sha256:ca94c85911601b097d53caeeec30201736ad69a93f30d15672b967558df02885", + "sha256:cf745cbfad6389c0e331786e5fe9ae3f06e9d9c2ce2432378e1267954793975c", + "sha256:d9dd2b89a16cf7ab9c1170b5863e68de6bf83db51544875b25a5f05a7269e678", + "sha256:ddff3f8b9fa24a60527c137c852d0d9a7da2a02cf2151650029fdc97c852c974", + "sha256:e153a690b7255c5ced17895394b4f109d5dcc2a4f35cb809374da50f0e5c456a", + "sha256:ea2b51c5f38bad812da2ec0cd7eec09d25f521a8b6b6843cbccedd9a1d8a5c15", + "sha256:ef9ec8068cf23458dbf36a08e0c16f0a2df04b42a8827619646637be1769300a", + "sha256:f280b02827adc9d87f764972fbeb701cf5611f80b619c20568e1982a277d6146", + "sha256:f3ff7da165c99a5412fe5dd2304dd2dbaaaa5da718aad942dcb3a178eaa70c56", + "sha256:f58d3bfafecf3d81c15d99fc0ecf4319e80ac712c77cf0ce2661c8cf8bf84066", + "sha256:f79fe7993e230a12172ce7d7c7db061f046f672f2b946431c81aff8f60b2758b", + "sha256:ffe709b1d0bc2e9921257569675674cafb3a5f8af689ab9f3f2b3f88775b960f" + ], + "version": "==1.0.3" }, "websockets": { "hashes": [ @@ -1725,79 +1725,74 @@ }, "wrapt": { "hashes": [ - "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc", - "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81", - "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", - "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e", - "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca", - "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0", - "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", - "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487", - "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40", - "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c", - "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", - "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202", - "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41", - "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9", - "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b", - "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", - "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", - "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", - "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00", - "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc", - "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", - "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267", - "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", - "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966", - "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", - "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228", - "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72", - "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", - "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292", - "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0", - "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0", - "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36", - "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c", - "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5", - "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f", - "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73", - "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b", - "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", - "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593", - "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39", - "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", - "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf", - "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf", - "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", - "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c", - "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", - "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f", - "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440", - "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465", - "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136", - "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b", - "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", - "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", - "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8", - "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6", - "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e", - "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", - "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c", - "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e", - "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8", - "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2", - "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020", - "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35", - "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", - "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3", - "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", - "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809", - "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", - "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", - "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" + "sha256:0229b247b0fc7dee0d36176cbb79dbaf2a9eb7ecc50ec3121f40ef443155fb1d", + "sha256:0698d3a86f68abc894d537887b9bbf84d29bcfbc759e23f4644be27acf6da301", + "sha256:0a0a1a1ec28b641f2a3a2c35cbe86c00051c04fffcfcc577ffcdd707df3f8635", + "sha256:0b48554952f0f387984da81ccfa73b62e52817a4386d070c75e4db7d43a28c4a", + "sha256:0f2a28eb35cf99d5f5bd12f5dd44a0f41d206db226535b37b0c60e9da162c3ed", + "sha256:140ea00c87fafc42739bd74a94a5a9003f8e72c27c47cd4f61d8e05e6dec8721", + "sha256:16187aa2317c731170a88ef35e8937ae0f533c402872c1ee5e6d079fcf320801", + "sha256:17fcf043d0b4724858f25b8826c36e08f9fb2e475410bece0ec44a22d533da9b", + "sha256:18b956061b8db634120b58f668592a772e87e2e78bc1f6a906cfcaa0cc7991c1", + "sha256:2399408ac33ffd5b200480ee858baa58d77dd30e0dd0cab6a8a9547135f30a88", + "sha256:2a0c23b8319848426f305f9cb0c98a6e32ee68a36264f45948ccf8e7d2b941f8", + "sha256:2dfb7cff84e72e7bf975b06b4989477873dcf160b2fd89959c629535df53d4e0", + "sha256:2f495b6754358979379f84534f8dd7a43ff8cff2558dcdea4a148a6e713a758f", + "sha256:33539c6f5b96cf0b1105a0ff4cf5db9332e773bb521cc804a90e58dc49b10578", + "sha256:3c34f6896a01b84bab196f7119770fd8466c8ae3dfa73c59c0bb281e7b588ce7", + "sha256:498fec8da10e3e62edd1e7368f4b24aa362ac0ad931e678332d1b209aec93045", + "sha256:4d63f4d446e10ad19ed01188d6c1e1bb134cde8c18b0aa2acfd973d41fcc5ada", + "sha256:4e4b4385363de9052dac1a67bfb535c376f3d19c238b5f36bddc95efae15e12d", + "sha256:4e547b447073fc0dbfcbff15154c1be8823d10dab4ad401bdb1575e3fdedff1b", + "sha256:4f643df3d4419ea3f856c5c3f40fec1d65ea2e89ec812c83f7767c8730f9827a", + "sha256:4f763a29ee6a20c529496a20a7bcb16a73de27f5da6a843249c7047daf135977", + "sha256:5ae271862b2142f4bc687bdbfcc942e2473a89999a54231aa1c2c676e28f29ea", + "sha256:5d8fd17635b262448ab8f99230fe4dac991af1dabdbb92f7a70a6afac8a7e346", + "sha256:69c40d4655e078ede067a7095544bcec5a963566e17503e75a3a3e0fe2803b13", + "sha256:69d093792dc34a9c4c8a70e4973a3361c7a7578e9cd86961b2bbf38ca71e4e22", + "sha256:6a9653131bda68a1f029c52157fd81e11f07d485df55410401f745007bd6d339", + "sha256:6ff02a91c4fc9b6a94e1c9c20f62ea06a7e375f42fe57587f004d1078ac86ca9", + "sha256:714c12485aa52efbc0fc0ade1e9ab3a70343db82627f90f2ecbc898fdf0bb181", + "sha256:7264cbb4a18dc4acfd73b63e4bcfec9c9802614572025bdd44d0721983fc1d9c", + "sha256:73a96fd11d2b2e77d623a7f26e004cc31f131a365add1ce1ce9a19e55a1eef90", + "sha256:74bf625b1b4caaa7bad51d9003f8b07a468a704e0644a700e936c357c17dd45a", + "sha256:81b1289e99cf4bad07c23393ab447e5e96db0ab50974a280f7954b071d41b489", + "sha256:8425cfce27b8b20c9b89d77fb50e368d8306a90bf2b6eef2cdf5cd5083adf83f", + "sha256:875d240fdbdbe9e11f9831901fb8719da0bd4e6131f83aa9f69b96d18fae7504", + "sha256:879591c2b5ab0a7184258274c42a126b74a2c3d5a329df16d69f9cee07bba6ea", + "sha256:89fc28495896097622c3fc238915c79365dd0ede02f9a82ce436b13bd0ab7569", + "sha256:8a5e7cc39a45fc430af1aefc4d77ee6bad72c5bcdb1322cfde852c15192b8bd4", + "sha256:8f8909cdb9f1b237786c09a810e24ee5e15ef17019f7cecb207ce205b9b5fcce", + "sha256:914f66f3b6fc7b915d46c1cc424bc2441841083de01b90f9e81109c9759e43ab", + "sha256:92a3d214d5e53cb1db8b015f30d544bc9d3f7179a05feb8f16df713cecc2620a", + "sha256:948a9bd0fb2c5120457b07e59c8d7210cbc8703243225dbd78f4dfc13c8d2d1f", + "sha256:9c900108df470060174108012de06d45f514aa4ec21a191e7ab42988ff42a86c", + "sha256:9f2939cd4a2a52ca32bc0b359015718472d7f6de870760342e7ba295be9ebaf9", + "sha256:a4192b45dff127c7d69b3bdfb4d3e47b64179a0b9900b6351859f3001397dabf", + "sha256:a8fc931382e56627ec4acb01e09ce66e5c03c384ca52606111cee50d931a342d", + "sha256:ad47b095f0bdc5585bced35bd088cbfe4177236c7df9984b3cc46b391cc60627", + "sha256:b1ca5f060e205f72bec57faae5bd817a1560fcfc4af03f414b08fa29106b7e2d", + "sha256:ba1739fb38441a27a676f4de4123d3e858e494fac05868b7a281c0a383c098f4", + "sha256:baa7ef4e0886a6f482e00d1d5bcd37c201b383f1d314643dfb0367169f94f04c", + "sha256:bb90765dd91aed05b53cd7a87bd7f5c188fcd95960914bae0d32c5e7f899719d", + "sha256:bc7f729a72b16ee21795a943f85c6244971724819819a41ddbaeb691b2dd85ad", + "sha256:bdf62d25234290db1837875d4dceb2151e4ea7f9fff2ed41c0fde23ed542eb5b", + "sha256:c30970bdee1cad6a8da2044febd824ef6dc4cc0b19e39af3085c763fdec7de33", + "sha256:d2c63b93548eda58abf5188e505ffed0229bf675f7c3090f8e36ad55b8cbc371", + "sha256:d751300b94e35b6016d4b1e7d0e7bbc3b5e1751e2405ef908316c2a9024008a1", + "sha256:da427d311782324a376cacb47c1a4adc43f99fd9d996ffc1b3e8529c4074d393", + "sha256:daba396199399ccabafbfc509037ac635a6bc18510ad1add8fd16d4739cdd106", + "sha256:e185ec6060e301a7e5f8461c86fb3640a7beb1a0f0208ffde7a65ec4074931df", + "sha256:e4a557d97f12813dc5e18dad9fa765ae44ddd56a672bb5de4825527c847d6379", + "sha256:e5ed16d95fd142e9c72b6c10b06514ad30e846a0d0917ab406186541fe68b451", + "sha256:e711fc1acc7468463bc084d1b68561e40d1eaa135d8c509a65dd534403d83d7b", + "sha256:f28b29dc158ca5d6ac396c8e0a2ef45c4e97bb7e65522bfc04c989e6fe814575", + "sha256:f335579a1b485c834849e9075191c9898e0731af45705c2ebf70e0cd5d58beed", + "sha256:fce6fee67c318fdfb7f285c29a82d84782ae2579c0e1b385b7f36c6e8074fffb", + "sha256:fd136bb85f4568fffca995bd3c8d52080b1e5b225dbf1c2b17b66b4c5fa02838" ], - "markers": "python_version >= '3.6'", - "version": "==1.16.0" + "markers": "python_version >= '3.8'", + "version": "==1.17.0" }, "xmltodict": { "hashes": [ @@ -1971,7 +1966,6 @@ "sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3" ], "index": "pypi", - "markers": "python_version >= '3.9'", "version": "==7.6.9" }, "decorator": { @@ -2004,7 +1998,6 @@ "sha256:cb0a405a306d2995a5cbb9901894d240784a9f341394c6ba3f4fe8c6eb89ff6e" ], "index": "pypi", - "markers": "python_version >= '3.10'", "version": "==8.30.0" }, "jedi": { @@ -2091,7 +2084,6 @@ "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761" ], "index": "pypi", - "markers": "python_version >= '3.8'", "version": "==8.3.4" }, "pytest-asyncio": { @@ -2110,14 +2102,6 @@ "index": "pypi", "version": "==6.0.0" }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, "stack-data": { "hashes": [ "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", @@ -2138,7 +2122,7 @@ "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" ], - "markers": "python_version < '3.12'", + "markers": "python_version >= '3.8'", "version": "==4.12.2" }, "wcwidth": { diff --git a/cdmtaskservice/app_state.py b/cdmtaskservice/app_state.py index 578f2bb..dedc9d0 100644 --- a/cdmtaskservice/app_state.py +++ b/cdmtaskservice/app_state.py @@ -91,12 +91,13 @@ async def build_app( try: mongodao = await MongoDAO.create(mongocli[cfg.mongo_db]) job_state = JobState(mongodao) - nerscjawsflow = NERSCJAWSRunner( + nerscjawsflow = NERSCJAWSRunner( # this has a lot of required args, yech nerscman, - job_state, + job_state, # TODO CODE if this isn't necessary, remove and recombine with job_submit mongodao, s3, s3_external, + coman, cfg.jaws_token, cfg.jaws_group, cfg.service_root_url, diff --git a/cdmtaskservice/jobflows/nersc_jaws.py b/cdmtaskservice/jobflows/nersc_jaws.py index a251407..8cf78a2 100644 --- a/cdmtaskservice/jobflows/nersc_jaws.py +++ b/cdmtaskservice/jobflows/nersc_jaws.py @@ -3,12 +3,15 @@ """ import logging +import os from cdmtaskservice import models from cdmtaskservice import timestamp from cdmtaskservice.arg_checkers import not_falsy as _not_falsy, require_string as _require_string from cdmtaskservice.callback_url_paths import get_download_complete_callback +from cdmtaskservice.coroutine_manager import CoroutineWrangler from cdmtaskservice.exceptions import InvalidJobStateError +from cdmtaskservice.input_file_locations import determine_file_locations from cdmtaskservice.job_state import JobState from cdmtaskservice.mongo import MongoDAO from cdmtaskservice.nersc.manager import NERSCManager @@ -31,6 +34,7 @@ def __init__( mongodao: MongoDAO, s3_client: S3Client, s3_external_client: S3Client, + coro_manager: CoroutineWrangler, jaws_token: str, jaws_group: str, service_root_url: str, @@ -46,6 +50,7 @@ def __init__( s3_external_client - an S3 client pointing to an external URL for the S3 data stores that may not be accessible from the current process, but is accessible to remote processes at NERSC. + coro_manager - a coroutine manager. jaws_token - a token for the JGI JAWS system. jaws_group - the group to use for running JAWS jobs. service_root_url - the URL of the service root, used for constructing service callbacks. @@ -58,6 +63,7 @@ def __init__( self._s3 = _not_falsy(s3_client, "s3_client") self._s3ext = _not_falsy(s3_external_client, "s3_external_client") self._s3insecure = s3_insecure_ssl + self._coman = _not_falsy(coro_manager, "coro_manager") self._jtoken = _require_string(jaws_token, "jaws_token") self._jgroup = _require_string(jaws_group, "jaws_group") self._callback_root = _require_string(service_root_url, "service_root_url") @@ -116,7 +122,7 @@ async def start_job(self, job: models.Job, objmeta: list[S3ObjectMeta]): async def download_complete(self, job: models.AdminJobDetails): """ Continue a job after the download is complete. The job is expected to be in the - download submitted satate. + download submitted state. """ if _not_falsy(job, "job").state != models.JobState.DOWNLOAD_SUBMITTED: raise InvalidJobStateError("Job must be in the download submitted state") @@ -132,3 +138,42 @@ async def download_complete(self, job: models.AdminJobDetails): models.JobState.JOB_SUBMITTING, timestamp.utcdatetime() ) + await self._coman.run_coroutine(self._download_complete(job)) + + async def _download_complete(self, job: models.AdminJobDetails): + logr = logging.getLogger(__name__) + manifest_files = self._generate_manifest_files(job) + # TODO REMOVE these lines + logr.info(f"*** manifiles: {len(manifest_files)}") + for m in manifest_files: + logr.info(m) + logr.info("***") + + def _generate_manifest_files(self, job: models.AdminJobDetails) -> list[str]: + # If we support multiple compute sites this should be moved to a general code location + # Currently leave here rather than creating yet another module + manifests = [] + mani_spec = job.job_input.params.get_file_parameter() + if not mani_spec or mani_spec.type is not models.ParameterType.MANIFEST_FILE: + return manifests + file_to_rel_path = determine_file_locations(job.job_input) + for files in job.job_input.get_files_per_container().files: + manifest = "" + if mani_spec.manifest_file_header: + manifest = f"{mani_spec.manifest_file_header}\n" + for f in files: + match mani_spec.manifest_file_format: + case models.ManifestFileFormat.DATA_IDS: + manifest += f"{f.data_id}\n" + case models.ManifestFileFormat.FILES: + f = os.path.join( + job.job_input.params.input_mount_point, file_to_rel_path[f] + ) + manifest += f"{f}\n" + case _: + # Can't currently happen but for future safety + raise ValueError( + f"Unknown manifest file format: {manifest.manifest_file_format}" + ) + manifests.append(manifest) + return manifests diff --git a/cdmtaskservice/models.py b/cdmtaskservice/models.py index 986a8d6..7a3298a 100644 --- a/cdmtaskservice/models.py +++ b/cdmtaskservice/models.py @@ -314,6 +314,9 @@ def get_file_parameter(self) -> Parameter | None: If a file specification parameter is present in the arguments, get it. Returns None otherwise. """ + # tried methodtools.lru_cache here but Pydantic complained + if hasattr(self, "_param"): # memoize + return self._param param = None if self.positional_args: for i, p in enumerate(self.positional_args): @@ -327,6 +330,7 @@ def get_file_parameter(self) -> Parameter | None: # Space separated lists in environment variables don't really make sense. # Come back to this if needed. param, p, f"Environmental parameter at key {key}", no_space=True) + self._param = param return param def _check_parameter( @@ -575,6 +579,25 @@ def _check_input_roots(cls, v): @classmethod def _check_outdir(cls, v): return _validate_s3_path(v) + + @model_validator(mode="after") + def _check_manifest_data_ids(self) -> Self: + fp = self.params.get_file_parameter() + # This seems really sketchy. Not sure how else to do it though + if (fp + # manifest file format is ignored, but not necc None, if type is not manifest file + and fp.type is ParameterType.MANIFEST_FILE + and fp.manifest_file_format is ManifestFileFormat.DATA_IDS + and ( + not isinstance(self.input_files[0], S3File) + or not self.input_files[0].data_id + ) + ): + raise ValueError( + "If a manifest file with data IDs is specified, " + + "data IDs must be supplied for all files" + ) + return self def inputs_are_S3File(self) -> bool: """ diff --git a/cdmtaskservice/routes.py b/cdmtaskservice/routes.py index 37d21b0..1fbfd8a 100644 --- a/cdmtaskservice/routes.py +++ b/cdmtaskservice/routes.py @@ -38,7 +38,7 @@ _AUTH = KBaseHTTPBearer() # * isn't allowed in KBase user names -_SERVICE_USER = "***service***" +_SERVICE_USER = "***SERVICE***" def _ensure_admin(user: kb_auth.KBaseUser, err_msg: str):