From ea628e631447d67ebdd55641fc2c923651fdc875 Mon Sep 17 00:00:00 2001 From: "CDORON@il.ibm.com" Date: Wed, 2 Dec 2020 15:18:58 +0200 Subject: [PATCH 1/4] add support for httpfs Signed-off-by: CDORON@il.ibm.com --- Pipfile | 1 + Pipfile.lock | 273 +++++++++++++++++++++++++++++---------------- afm/asset.py | 9 +- sample/sample.yaml | 4 +- 4 files changed, 190 insertions(+), 97 deletions(-) diff --git a/Pipfile b/Pipfile index 26e4769..fce7dc4 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ pandas = "==1.0.5" pyarrow = "==1.0.1" schemed-yaml-config = "==0.0.3" requests = "*" +httpfs = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 66edc25..7348bb0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "86c1dda9674248e3f12976d3b28a3a6ba2953757ee7315fadb5c66686cee376a" + "sha256": "26f28bed8b06290fa7d182094994446b9ef5d6930ed9e733f63bd078e9099b86" }, "pipfile-spec": 6, "requires": { @@ -16,20 +16,27 @@ ] }, "default": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, "attrs": { "hashes": [ - "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", - "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.2.0" + "version": "==20.3.0" }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" ], - "version": "==2020.6.20" + "version": "==2020.11.8" }, "chardet": { "hashes": [ @@ -38,6 +45,44 @@ ], "version": "==3.0.4" }, + "fs": { + "hashes": [ + "sha256:cc99d476b500f993df8ef697b96dc70928ca2946a455c396a566efe021126767", + "sha256:cd6b178f373a0370feac8612fd3c142aa6a5cadd3d471b525b08db4d3b511c9c" + ], + "version": "==2.4.11" + }, + "h11": { + "hashes": [ + "sha256:3c6c61d69c6f13d41f1b80ab0322f1872702a3ba26e12aa864c928f6a43fbaab", + "sha256:ab6c335e1b6ef34b205d5ca3e228c9299cc7218b049819ec84a388c2525e5d87" + ], + "version": "==0.11.0" + }, + "httpcore": { + "hashes": [ + "sha256:420700af11db658c782f7e8fda34f9dcd95e3ee93944dd97d78cb70247e0cd06", + "sha256:dd1d762d4f7c2702149d06be2597c35fb154c5eff9789a8c5823fbcf4d2978d6" + ], + "markers": "python_version >= '3.6'", + "version": "==0.12.2" + }, + "httpfs": { + "hashes": [ + "sha256:4748d727c65947ff13b74860b006c1f69d72b3c171c09f1118e4a1963383e2fd", + "sha256:a2c5e8f8772a48b12f50f5f6b8754494b72692ecebf73cee422a3b5a376e48a8" + ], + "index": "pypi", + "version": "==0.0.1" + }, + "httpx": { + "hashes": [ + "sha256:126424c279c842738805974687e0518a94c7ae8d140cd65b9c4f77ac46ffa537", + "sha256:9cffb8ba31fac6536f2c8cde30df859013f59e4bcc5b8d43901cb3654a8e0a5b" + ], + "markers": "python_version >= '3.6'", + "version": "==0.16.1" + }, "idna": { "hashes": [ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", @@ -55,35 +100,43 @@ }, "numpy": { "hashes": [ - "sha256:082f8d4dd69b6b688f64f509b91d482362124986d98dc7dc5f5e9f9b9c3bb983", - "sha256:1bc0145999e8cb8aed9d4e65dd8b139adf1919e521177f198529687dbf613065", - "sha256:309cbcfaa103fc9a33ec16d2d62569d541b79f828c382556ff072442226d1968", - "sha256:3673c8b2b29077f1b7b3a848794f8e11f401ba0b71c49fbd26fb40b71788b132", - "sha256:480fdd4dbda4dd6b638d3863da3be82873bba6d32d1fc12ea1b8486ac7b8d129", - "sha256:56ef7f56470c24bb67fb43dae442e946a6ce172f97c69f8d067ff8550cf782ff", - "sha256:5a936fd51049541d86ccdeef2833cc89a18e4d3808fe58a8abeb802665c5af93", - "sha256:5b6885c12784a27e957294b60f97e8b5b4174c7504665333c5e94fbf41ae5d6a", - "sha256:667c07063940e934287993366ad5f56766bc009017b4a0fe91dbd07960d0aba7", - "sha256:7ed448ff4eaffeb01094959b19cbaf998ecdee9ef9932381420d514e446601cd", - "sha256:8343bf67c72e09cfabfab55ad4a43ce3f6bf6e6ced7acf70f45ded9ebb425055", - "sha256:92feb989b47f83ebef246adabc7ff3b9a59ac30601c3f6819f8913458610bdcc", - "sha256:935c27ae2760c21cd7354402546f6be21d3d0c806fffe967f745d5f2de5005a7", - "sha256:aaf42a04b472d12515debc621c31cf16c215e332242e7a9f56403d814c744624", - "sha256:b12e639378c741add21fbffd16ba5ad25c0a1a17cf2b6fe4288feeb65144f35b", - "sha256:b1cca51512299841bf69add3b75361779962f9cee7d9ee3bb446d5982e925b69", - "sha256:b8456987b637232602ceb4d663cb34106f7eb780e247d51a260b84760fd8f491", - "sha256:b9792b0ac0130b277536ab8944e7b754c69560dac0415dd4b2dbd16b902c8954", - "sha256:c9591886fc9cbe5532d5df85cb8e0cc3b44ba8ce4367bd4cf1b93dc19713da72", - "sha256:cf1347450c0b7644ea142712619533553f02ef23f92f781312f6a3553d031fc7", - "sha256:de8b4a9b56255797cbddb93281ed92acbc510fb7b15df3f01bd28f46ebc4edae", - "sha256:e1b1dc0372f530f26a03578ac75d5e51b3868b9b76cd2facba4c9ee0eb252ab1", - "sha256:e45f8e981a0ab47103181773cc0a54e650b2aef8c7b6cd07405d0fa8d869444a", - "sha256:e4f6d3c53911a9d103d8ec9518190e52a8b945bab021745af4939cfc7c0d4a9e", - "sha256:ed8a311493cf5480a2ebc597d1e177231984c818a86875126cfd004241a73c3e", - "sha256:ef71a1d4fd4858596ae80ad1ec76404ad29701f8ca7cdcebc50300178db14dfc" + "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db", + "sha256:09c12096d843b90eafd01ea1b3307e78ddd47a55855ad402b157b6c4862197ce", + "sha256:13d166f77d6dc02c0a73c1101dd87fdf01339febec1030bd810dcd53fff3b0f1", + "sha256:141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512", + "sha256:16c1b388cc31a9baa06d91a19366fb99ddbe1c7b205293ed072211ee5bac1ed2", + "sha256:18bed2bcb39e3f758296584337966e68d2d5ba6aab7e038688ad53c8f889f757", + "sha256:1aeef46a13e51931c0b1cf8ae1168b4a55ecd282e6688fdb0a948cc5a1d5afb9", + "sha256:27d3f3b9e3406579a8af3a9f262f5339005dd25e0ecf3cf1559ff8a49ed5cbf2", + "sha256:2a2740aa9733d2e5b2dfb33639d98a64c3b0f24765fed86b0fd2aec07f6a0a08", + "sha256:4377e10b874e653fe96985c05feed2225c912e328c8a26541f7fc600fb9c637b", + "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb", + "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc", + "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac", + "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83", + "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36", + "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387", + "sha256:8cac8790a6b1ddf88640a9267ee67b1aee7a57dfa2d2dd33999d080bc8ee3a0f", + "sha256:8ece138c3a16db8c1ad38f52eb32be6086cc72f403150a79336eb2045723a1ad", + "sha256:9eeb7d1d04b117ac0d38719915ae169aa6b61fca227b0b7d198d43728f0c879c", + "sha256:a09f98011236a419ee3f49cedc9ef27d7a1651df07810ae430a6b06576e0b414", + "sha256:a5d897c14513590a85774180be713f692df6fa8ecf6483e561a6d47309566f37", + "sha256:ad6f2ff5b1989a4899bf89800a671d71b1612e5ff40866d1f4d8bcf48d4e5764", + "sha256:c42c4b73121caf0ed6cd795512c9c09c52a7287b04d105d112068c1736d7c753", + "sha256:cb1017eec5257e9ac6209ac172058c430e834d5d2bc21961dceeb79d111e5909", + "sha256:d6c7bb82883680e168b55b49c70af29b84b84abb161cbac2800e8fcb6f2109b6", + "sha256:e452dc66e08a4ce642a961f134814258a082832c78c90351b75c41ad16f79f63", + "sha256:e5b6ed0f0b42317050c88022349d994fe72bfe35f5908617512cd8c8ef9da2a9", + "sha256:e9b30d4bd69498fc0c3fe9db5f62fffbb06b8eb9321f92cc970f2969be5e3949", + "sha256:ec149b90019852266fec2341ce1db513b843e496d5a8e8cdb5ced1923a92faab", + "sha256:edb01671b3caae1ca00881686003d16c2209e07b7ef8b7639f1867852b948f7c", + "sha256:f0d3929fe88ee1c155129ecd82f981b8856c5d97bcb0d5f23e9b4242e79d1de3", + "sha256:f29454410db6ef8126c83bd3c968d143304633d45dc57b51252afbd79d700893", + "sha256:fe45becb4c2f72a0907c1d0246ea6449fe7a9e2293bb0e11c4e9a32bb0930a15", + "sha256:fedbd128668ead37f33917820b704784aff695e0019309ad446a6d0b065b57e4" ], "markers": "python_version >= '3.6'", - "version": "==1.19.1" + "version": "==1.19.4" }, "pandas": { "hashes": [ @@ -136,9 +189,10 @@ }, "pyrsistent": { "hashes": [ - "sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3" + "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" ], - "version": "==0.16.0" + "markers": "python_version >= '3.5'", + "version": "==0.17.3" }, "python-dateutil": { "hashes": [ @@ -150,10 +204,10 @@ }, "pytz": { "hashes": [ - "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", - "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268", + "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd" ], - "version": "==2020.1" + "version": "==2020.4" }, "pyyaml": { "hashes": [ @@ -175,11 +229,21 @@ }, "requests": { "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8", + "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998" ], "index": "pypi", - "version": "==2.24.0" + "version": "==2.25.0" + }, + "rfc3986": { + "extras": [ + "idna2008" + ], + "hashes": [ + "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d", + "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50" + ], + "version": "==1.4.0" }, "schemed-yaml-config": { "hashes": [ @@ -196,13 +260,21 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, + "sniffio": { + "hashes": [ + "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663", + "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de" + ], + "markers": "python_version >= '3.5'", + "version": "==1.2.0" + }, "urllib3": { "hashes": [ - "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", - "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.25.10" + "version": "==1.26.2" }, "yamlloader": { "hashes": [ @@ -230,26 +302,26 @@ }, "attrs": { "hashes": [ - "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", - "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.2.0" + "version": "==20.3.0" }, "iniconfig": { "hashes": [ - "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437", - "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69" + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" ], - "version": "==1.0.1" + "version": "==1.1.1" }, "isort": { "hashes": [ - "sha256:171c5f365791073426b5ed3a156c2081a47f88c329161fd28228ff2da4c97ddb", - "sha256:ba91218eee31f1e300ecc079ef0c524cea3fc41bfbb979cbdf5fd3a889e3cfed" + "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7", + "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58" ], "markers": "python_version >= '3.6' and python_version < '4.0'", - "version": "==5.5.2" + "version": "==5.6.4" }, "lazy-object-proxy": { "hashes": [ @@ -293,21 +365,13 @@ ], "version": "==0.6.1" }, - "more-itertools": { - "hashes": [ - "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", - "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" - ], - "markers": "python_version >= '3.5'", - "version": "==8.5.0" - }, "packaging": { "hashes": [ - "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", - "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236", + "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.4" + "version": "==20.7" }, "pluggy": { "hashes": [ @@ -343,37 +407,57 @@ }, "pytest": { "hashes": [ - "sha256:85228d75db9f45e06e57ef9bf4429267f81ac7c0d742cc9ed63d09886a9fe6f4", - "sha256:8b6007800c53fdacd5a5c192203f4e531eb2a1540ad9c752e052ec0f7143dbad" + "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe", + "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e" ], "index": "pypi", - "version": "==6.0.1" + "version": "==6.1.2" }, "regex": { "hashes": [ - "sha256:0dc64ee3f33cd7899f79a8d788abfbec168410be356ed9bd30bbd3f0a23a7204", - "sha256:1269fef3167bb52631ad4fa7dd27bf635d5a0790b8e6222065d42e91bede4162", - "sha256:14a53646369157baa0499513f96091eb70382eb50b2c82393d17d7ec81b7b85f", - "sha256:3a3af27a8d23143c49a3420efe5b3f8cf1a48c6fc8bc6856b03f638abc1833bb", - "sha256:46bac5ca10fb748d6c55843a931855e2727a7a22584f302dd9bb1506e69f83f6", - "sha256:4c037fd14c5f4e308b8370b447b469ca10e69427966527edcab07f52d88388f7", - "sha256:51178c738d559a2d1071ce0b0f56e57eb315bcf8f7d4cf127674b533e3101f88", - "sha256:5ea81ea3dbd6767873c611687141ec7b06ed8bab43f68fad5b7be184a920dc99", - "sha256:6961548bba529cac7c07af2fd4d527c5b91bb8fe18995fed6044ac22b3d14644", - "sha256:75aaa27aa521a182824d89e5ab0a1d16ca207318a6b65042b046053cfc8ed07a", - "sha256:7a2dd66d2d4df34fa82c9dc85657c5e019b87932019947faece7983f2089a840", - "sha256:8a51f2c6d1f884e98846a0a9021ff6861bdb98457879f412fdc2b42d14494067", - "sha256:9c568495e35599625f7b999774e29e8d6b01a6fb684d77dee1f56d41b11b40cd", - "sha256:9eddaafb3c48e0900690c1727fba226c4804b8e6127ea409689c3bb492d06de4", - "sha256:bbb332d45b32df41200380fff14712cb6093b61bd142272a10b16778c418e98e", - "sha256:bc3d98f621898b4a9bc7fecc00513eec8f40b5b83913d74ccb445f037d58cd89", - "sha256:c11d6033115dc4887c456565303f540c44197f4fc1a2bfb192224a301534888e", - "sha256:c50a724d136ec10d920661f1442e4a8b010a4fe5aebd65e0c2241ea41dbe93dc", - "sha256:d0a5095d52b90ff38592bbdc2644f17c6d495762edf47d876049cfd2968fbccf", - "sha256:d6cff2276e502b86a25fd10c2a96973fdb45c7a977dca2138d661417f3728341", - "sha256:e46d13f38cfcbb79bfdb2964b0fe12561fe633caf964a77a5f8d4e45fe5d2ef7" - ], - "version": "==2020.7.14" + "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538", + "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4", + "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc", + "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa", + "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444", + "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1", + "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af", + "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8", + "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9", + "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88", + "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba", + "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364", + "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e", + "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7", + "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0", + "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31", + "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683", + "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee", + "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b", + "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884", + "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c", + "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e", + "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562", + "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85", + "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c", + "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6", + "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d", + "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b", + "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70", + "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b", + "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b", + "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f", + "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0", + "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5", + "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5", + "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f", + "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e", + "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512", + "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d", + "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917", + "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f" + ], + "version": "==2020.11.13" }, "six": { "hashes": [ @@ -385,10 +469,11 @@ }, "toml": { "hashes": [ - "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", - "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "version": "==0.10.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" }, "wrapt": { "hashes": [ diff --git a/afm/asset.py b/afm/asset.py index 333c7a3..56071ab 100644 --- a/afm/asset.py +++ b/afm/asset.py @@ -2,13 +2,18 @@ # Copyright 2020 IBM Corp. # SPDX-License-Identifier: Apache-2.0 # +import httpfs import os from afm.config import Config from afm.pep import registry, consolidate_actions from afm.s3 import s3filesystem_from_config -from pyarrow.fs import LocalFileSystem +from pyarrow.fs import LocalFileSystem, PyFileSystem, FSSpecHandler + +def httpfs_from_config(httpfs_config): + fs = httpfs.fs.HttpFs(httpfs_config.get('endpoint_url')) + return PyFileSystem(FSSpecHandler(fs)) class Asset: def __init__(self, config: Config, asset_name: str): @@ -47,6 +52,8 @@ def _filesystem_for_asset(asset_config: dict): return s3filesystem_from_config(connection["s3"]) elif connection_type == "localfs": return LocalFileSystem() + elif connection_type == "httpfs": + return httpfs_from_config(connection["httpfs"]) raise ValueError( "Unsupported connection type: {}".format(connection_type)) diff --git a/sample/sample.yaml b/sample/sample.yaml index ab73c15..ba589bc 100644 --- a/sample/sample.yaml +++ b/sample/sample.yaml @@ -3,8 +3,8 @@ data: format: parquet path: "ursa-labs-taxi-data/2019/06/data.parquet" connection: - type: s3 - s3: + type: httpfs + httpfs: endpoint_url: "https://s3.us-east-2.amazonaws.com" transformations: - action: "Redact" From 9dba1a3a4c9a12955d67ef427780a43427be46e8 Mon Sep 17 00:00:00 2001 From: "CDORON@il.ibm.com" Date: Sun, 6 Dec 2020 09:46:44 +0200 Subject: [PATCH 2/4] create filesystems module for s3 and httpfs Signed-off-by: CDORON@il.ibm.com --- afm/asset.py | 10 +++------- afm/filesystems/httpfs.py | 10 ++++++++++ afm/{ => filesystems}/s3.py | 0 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 afm/filesystems/httpfs.py rename afm/{ => filesystems}/s3.py (100%) diff --git a/afm/asset.py b/afm/asset.py index 56071ab..8a6ea3e 100644 --- a/afm/asset.py +++ b/afm/asset.py @@ -2,18 +2,14 @@ # Copyright 2020 IBM Corp. # SPDX-License-Identifier: Apache-2.0 # -import httpfs import os from afm.config import Config from afm.pep import registry, consolidate_actions -from afm.s3 import s3filesystem_from_config +from afm.filesystems.s3 import s3filesystem_from_config +from afm.filesystems.httpfs import httpfs_from_config -from pyarrow.fs import LocalFileSystem, PyFileSystem, FSSpecHandler - -def httpfs_from_config(httpfs_config): - fs = httpfs.fs.HttpFs(httpfs_config.get('endpoint_url')) - return PyFileSystem(FSSpecHandler(fs)) +from pyarrow.fs import LocalFileSystem class Asset: def __init__(self, config: Config, asset_name: str): diff --git a/afm/filesystems/httpfs.py b/afm/filesystems/httpfs.py new file mode 100644 index 0000000..4e217c6 --- /dev/null +++ b/afm/filesystems/httpfs.py @@ -0,0 +1,10 @@ +# +# Copyright 2020 IBM Corp. +# SPDX-License-Identifier: Apache-2.0 +# +import httpfs +from pyarrow.fs import PyFileSystem, FSSpecHandler + +def httpfs_from_config(httpfs_config): + fs = httpfs.fs.HttpFs(httpfs_config.get('endpoint_url')) + return PyFileSystem(FSSpecHandler(fs)) diff --git a/afm/s3.py b/afm/filesystems/s3.py similarity index 100% rename from afm/s3.py rename to afm/filesystems/s3.py From 69ab57969bcda97cb31f44b433630a1eb6633a14 Mon Sep 17 00:00:00 2001 From: "CDORON@il.ibm.com" Date: Mon, 7 Dec 2020 10:23:10 +0200 Subject: [PATCH 3/4] replace httpfs with fsspec.implementations.http.HTTPFileSystem Signed-off-by: CDORON@il.ibm.com --- Pipfile | 3 +- Pipfile.lock | 220 +++++++++++++++++++++++++++----------- afm/filesystems/httpfs.py | 4 +- sample/sample.yaml | 4 +- 4 files changed, 161 insertions(+), 70 deletions(-) diff --git a/Pipfile b/Pipfile index fce7dc4..1616c6c 100644 --- a/Pipfile +++ b/Pipfile @@ -17,7 +17,8 @@ pandas = "==1.0.5" pyarrow = "==1.0.1" schemed-yaml-config = "==0.0.3" requests = "*" -httpfs = "*" +fsspec = "*" +aiohttp = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 7348bb0..6709f18 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "26f28bed8b06290fa7d182094994446b9ef5d6930ed9e733f63bd078e9099b86" + "sha256": "ce233bca67788ad55d7bab8cf5c52cb35a61fcdea6dfeefb0bfdb368aef0e56c" }, "pipfile-spec": 6, "requires": { @@ -16,12 +16,56 @@ ] }, "default": { - "appdirs": { + "aiohttp": { + "hashes": [ + "sha256:0b795072bb1bf87b8620120a6373a3c61bfcb8da7e5c2377f4bb23ff4f0b62c9", + "sha256:0d438c8ca703b1b714e82ed5b7a4412c82577040dadff479c08405e2a715564f", + "sha256:16a3cb5df5c56f696234ea9e65e227d1ebe9c18aa774d36ff42f532139066a5f", + "sha256:1edfd82a98c5161497bbb111b2b70c0813102ad7e0aa81cbeb34e64c93863005", + "sha256:2406dc1dda01c7f6060ab586e4601f18affb7a6b965c50a8c90ff07569cf782a", + "sha256:2858b2504c8697beb9357be01dc47ef86438cc1cb36ecb6991796d19475faa3e", + "sha256:2a7b7640167ab536c3cb90cfc3977c7094f1c5890d7eeede8b273c175c3910fd", + "sha256:3228b7a51e3ed533f5472f54f70fd0b0a64c48dc1649a0f0e809bec312934d7a", + "sha256:328b552513d4f95b0a2eea4c8573e112866107227661834652a8984766aa7656", + "sha256:39f4b0a6ae22a1c567cb0630c30dd082481f95c13ca528dc501a7766b9c718c0", + "sha256:3b0036c978cbcc4a4512278e98e3e6d9e6b834dc973206162eddf98b586ef1c6", + "sha256:3ea8c252d8df5e9166bcf3d9edced2af132f4ead8ac422eac723c5781063709a", + "sha256:41608c0acbe0899c852281978492f9ce2c6fbfaf60aff0cefc54a7c4516b822c", + "sha256:59d11674964b74a81b149d4ceaff2b674b3b0e4d0f10f0be1533e49c4a28408b", + "sha256:5e479df4b2d0f8f02133b7e4430098699450e1b2a826438af6bec9a400530957", + "sha256:684850fb1e3e55c9220aad007f8386d8e3e477c4ec9211ae54d968ecdca8c6f9", + "sha256:6ccc43d68b81c424e46192a778f97da94ee0630337c9bbe5b2ecc9b0c1c59001", + "sha256:6d42debaf55450643146fabe4b6817bb2a55b23698b0434107e892a43117285e", + "sha256:710376bf67d8ff4500a31d0c207b8941ff4fba5de6890a701d71680474fe2a60", + "sha256:756ae7efddd68d4ea7d89c636b703e14a0c686688d42f588b90778a3c2fc0564", + "sha256:77149002d9386fae303a4a162e6bce75cc2161347ad2ba06c2f0182561875d45", + "sha256:78e2f18a82b88cbc37d22365cf8d2b879a492faedb3f2975adb4ed8dfe994d3a", + "sha256:7d9b42127a6c0bdcc25c3dcf252bb3ddc70454fac593b1b6933ae091396deb13", + "sha256:8389d6044ee4e2037dca83e3f6994738550f6ee8cfb746762283fad9b932868f", + "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4", + "sha256:c1e0920909d916d3375c7a1fdb0b1c78e46170e8bb42792312b6eb6676b2f87f", + "sha256:c68fdf21c6f3573ae19c7ee65f9ff185649a060c9a06535e9c3a0ee0bbac9235", + "sha256:c733ef3bdcfe52a1a75564389bad4064352274036e7e234730526d155f04d914", + "sha256:c9c58b0b84055d8bc27b7df5a9d141df4ee6ff59821f922dd73155861282f6a3", + "sha256:d03abec50df423b026a5aa09656bd9d37f1e6a49271f123f31f9b8aed5dc3ea3", + "sha256:d2cfac21e31e841d60dc28c0ec7d4ec47a35c608cb8906435d47ef83ffb22150", + "sha256:dcc119db14757b0c7bce64042158307b9b1c76471e655751a61b57f5a0e4d78e", + "sha256:df3a7b258cc230a65245167a202dd07320a5af05f3d41da1488ba0fa05bc9347", + "sha256:df48a623c58180874d7407b4d9ec06a19b84ed47f60a3884345b1a5099c1818b", + "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7", + "sha256:f326b3c1bbfda5b9308252ee0dcb30b612ee92b0e105d4abec70335fab5b1245", + "sha256:f411cb22115cb15452d099fec0ee636b06cf81bfb40ed9c02d30c8dc2bc2e3d1" + ], + "index": "pypi", + "version": "==3.7.3" + }, + "async-timeout": { "hashes": [ - "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", - "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" ], - "version": "==1.4.4" + "markers": "python_full_version >= '3.5.3'", + "version": "==3.0.1" }, "attrs": { "hashes": [ @@ -33,10 +77,10 @@ }, "certifi": { "hashes": [ - "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", - "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" ], - "version": "==2020.11.8" + "version": "==2020.12.5" }, "chardet": { "hashes": [ @@ -45,43 +89,13 @@ ], "version": "==3.0.4" }, - "fs": { - "hashes": [ - "sha256:cc99d476b500f993df8ef697b96dc70928ca2946a455c396a566efe021126767", - "sha256:cd6b178f373a0370feac8612fd3c142aa6a5cadd3d471b525b08db4d3b511c9c" - ], - "version": "==2.4.11" - }, - "h11": { - "hashes": [ - "sha256:3c6c61d69c6f13d41f1b80ab0322f1872702a3ba26e12aa864c928f6a43fbaab", - "sha256:ab6c335e1b6ef34b205d5ca3e228c9299cc7218b049819ec84a388c2525e5d87" - ], - "version": "==0.11.0" - }, - "httpcore": { - "hashes": [ - "sha256:420700af11db658c782f7e8fda34f9dcd95e3ee93944dd97d78cb70247e0cd06", - "sha256:dd1d762d4f7c2702149d06be2597c35fb154c5eff9789a8c5823fbcf4d2978d6" - ], - "markers": "python_version >= '3.6'", - "version": "==0.12.2" - }, - "httpfs": { + "fsspec": { "hashes": [ - "sha256:4748d727c65947ff13b74860b006c1f69d72b3c171c09f1118e4a1963383e2fd", - "sha256:a2c5e8f8772a48b12f50f5f6b8754494b72692ecebf73cee422a3b5a376e48a8" + "sha256:404194b0cf93128d980689fe9a49013ee9676edd1ab7f0f15b737b2cedb0ab88", + "sha256:e1e494d4814f6804769f3c7bfd7a722a15113cc0339d14755297f09306b8f21f" ], "index": "pypi", - "version": "==0.0.1" - }, - "httpx": { - "hashes": [ - "sha256:126424c279c842738805974687e0518a94c7ae8d140cd65b9c4f77ac46ffa537", - "sha256:9cffb8ba31fac6536f2c8cde30df859013f59e4bcc5b8d43901cb3654a8e0a5b" - ], - "markers": "python_version >= '3.6'", - "version": "==0.16.1" + "version": "==0.8.4" }, "idna": { "hashes": [ @@ -98,6 +112,49 @@ ], "version": "==3.0.2" }, + "multidict": { + "hashes": [ + "sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a", + "sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93", + "sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632", + "sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656", + "sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79", + "sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7", + "sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d", + "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5", + "sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224", + "sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26", + "sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea", + "sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348", + "sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6", + "sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76", + "sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1", + "sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f", + "sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952", + "sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a", + "sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37", + "sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9", + "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359", + "sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8", + "sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da", + "sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3", + "sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d", + "sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf", + "sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841", + "sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d", + "sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93", + "sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f", + "sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647", + "sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635", + "sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456", + "sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda", + "sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5", + "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281", + "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80" + ], + "markers": "python_version >= '3.6'", + "version": "==5.1.0" + }, "numpy": { "hashes": [ "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db", @@ -199,7 +256,7 @@ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.8.1" }, "pytz": { @@ -235,16 +292,6 @@ "index": "pypi", "version": "==2.25.0" }, - "rfc3986": { - "extras": [ - "idna2008" - ], - "hashes": [ - "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d", - "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50" - ], - "version": "==1.4.0" - }, "schemed-yaml-config": { "hashes": [ "sha256:53a755672364a0a9274401cb3254c5a86aee0654087fcf55079f7cda2d98082d" @@ -257,16 +304,16 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.15.0" }, - "sniffio": { + "typing-extensions": { "hashes": [ - "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663", - "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de" + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" ], - "markers": "python_version >= '3.5'", - "version": "==1.2.0" + "version": "==3.7.4.3" }, "urllib3": { "hashes": [ @@ -289,6 +336,49 @@ "sha256:7f30f0b99ea3f877f7cb340c570921fa9d639b7f69cba18be051e27f8de2080e" ], "version": "==0.4.0" + }, + "yarl": { + "hashes": [ + "sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e", + "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434", + "sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366", + "sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3", + "sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec", + "sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959", + "sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e", + "sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c", + "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6", + "sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a", + "sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6", + "sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424", + "sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e", + "sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f", + "sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50", + "sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2", + "sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc", + "sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4", + "sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970", + "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10", + "sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0", + "sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406", + "sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896", + "sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643", + "sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721", + "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478", + "sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724", + "sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e", + "sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8", + "sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96", + "sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25", + "sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76", + "sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2", + "sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2", + "sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c", + "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a", + "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" + ], + "markers": "python_version >= '3.6'", + "version": "==1.6.3" } }, "develop": { @@ -352,11 +442,11 @@ }, "licenseheaders": { "hashes": [ - "sha256:7e1c34164e2e355aa5ab518028f66cd15177635220c60ec693b0ccfdf3ec71a3", - "sha256:834d54f43fa088ec704d6f5f1b4b1e117c158c3eebeb20cb180d6c5490067e14" + "sha256:a6de1568c4e13c60abfb71e4ec42b4ae0edcb490a6e08bf55403dc0e3f09d309", + "sha256:a988f44144f403c8acc22983fc785b5483acd3575eaf8a3b9be5f90742657d1c" ], "index": "pypi", - "version": "==0.8.5" + "version": "==0.8.6" }, "mccabe": { "hashes": [ @@ -402,7 +492,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.4.7" }, "pytest": { @@ -464,7 +554,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.15.0" }, "toml": { @@ -472,7 +562,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==0.10.2" }, "wrapt": { diff --git a/afm/filesystems/httpfs.py b/afm/filesystems/httpfs.py index 4e217c6..be04d5a 100644 --- a/afm/filesystems/httpfs.py +++ b/afm/filesystems/httpfs.py @@ -2,9 +2,9 @@ # Copyright 2020 IBM Corp. # SPDX-License-Identifier: Apache-2.0 # -import httpfs +from fsspec.implementations.http import HTTPFileSystem from pyarrow.fs import PyFileSystem, FSSpecHandler def httpfs_from_config(httpfs_config): - fs = httpfs.fs.HttpFs(httpfs_config.get('endpoint_url')) + fs = HTTPFileSystem() return PyFileSystem(FSSpecHandler(fs)) diff --git a/sample/sample.yaml b/sample/sample.yaml index ba589bc..ab73c15 100644 --- a/sample/sample.yaml +++ b/sample/sample.yaml @@ -3,8 +3,8 @@ data: format: parquet path: "ursa-labs-taxi-data/2019/06/data.parquet" connection: - type: httpfs - httpfs: + type: s3 + s3: endpoint_url: "https://s3.us-east-2.amazonaws.com" transformations: - action: "Redact" From 050447577fc25a52ea80f947d2681c5ef3468333 Mon Sep 17 00:00:00 2001 From: "CDORON@il.ibm.com" Date: Mon, 7 Dec 2020 11:22:17 +0200 Subject: [PATCH 4/4] fixed versions for fsspec and aiohttp. httpfs does not require config Signed-off-by: CDORON@il.ibm.com --- Pipfile | 4 ++-- Pipfile.lock | 12 ++++++------ afm/asset.py | 2 +- afm/filesystems/httpfs.py | 5 ++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Pipfile b/Pipfile index 1616c6c..9e6ee3d 100644 --- a/Pipfile +++ b/Pipfile @@ -17,8 +17,8 @@ pandas = "==1.0.5" pyarrow = "==1.0.1" schemed-yaml-config = "==0.0.3" requests = "*" -fsspec = "*" -aiohttp = "*" +fsspec = "==0.8.4" +aiohttp = "==3.7.3" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 6709f18..e3ee9f0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ce233bca67788ad55d7bab8cf5c52cb35a61fcdea6dfeefb0bfdb368aef0e56c" + "sha256": "78f82ec8f064ddaec1d90330ba65ea2d2874b21c9541df01991d63c672eaca24" }, "pipfile-spec": 6, "requires": { @@ -256,7 +256,7 @@ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.1" }, "pytz": { @@ -304,7 +304,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "typing-extensions": { @@ -492,7 +492,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pytest": { @@ -554,7 +554,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "toml": { @@ -562,7 +562,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "wrapt": { diff --git a/afm/asset.py b/afm/asset.py index 8a6ea3e..eebb45c 100644 --- a/afm/asset.py +++ b/afm/asset.py @@ -49,7 +49,7 @@ def _filesystem_for_asset(asset_config: dict): elif connection_type == "localfs": return LocalFileSystem() elif connection_type == "httpfs": - return httpfs_from_config(connection["httpfs"]) + return httpfs_from_config() raise ValueError( "Unsupported connection type: {}".format(connection_type)) diff --git a/afm/filesystems/httpfs.py b/afm/filesystems/httpfs.py index be04d5a..a4ad6b1 100644 --- a/afm/filesystems/httpfs.py +++ b/afm/filesystems/httpfs.py @@ -5,6 +5,5 @@ from fsspec.implementations.http import HTTPFileSystem from pyarrow.fs import PyFileSystem, FSSpecHandler -def httpfs_from_config(httpfs_config): - fs = HTTPFileSystem() - return PyFileSystem(FSSpecHandler(fs)) +def httpfs_from_config(): + return PyFileSystem(FSSpecHandler(HTTPFileSystem()))