From a39e07797f1a19fc3aa0cd424d26ff9da6c72ab2 Mon Sep 17 00:00:00 2001 From: Raya Kurinnoy Date: Tue, 8 Mar 2022 15:54:08 +0200 Subject: [PATCH] New quick-start and workflow --- quick-start/quick-start.ipynb | 625 ++++++++++++++++++++++++++++++---- quick-start/workflow.py | 39 +++ 2 files changed, 605 insertions(+), 59 deletions(-) create mode 100644 quick-start/workflow.py diff --git a/quick-start/quick-start.ipynb b/quick-start/quick-start.ipynb index 8a47d5d..4e6f31c 100644 --- a/quick-start/quick-start.ipynb +++ b/quick-start/quick-start.ipynb @@ -23,6 +23,428 @@ "* Nuclio - serverless function framework used for deploying an online serrving function\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Install MLRun package\n", + "To install the MLRun package or upgrade the version of an installed package run the command below.
" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: Package(s) not found: mlrun\u001b[0m\n", + "Collecting mlrun[complete]==0.10.1-rc1\n", + " Downloading mlrun-0.10.1rc1-py3-none-any.whl (804 kB)\n", + "\u001b[K |████████████████████████████████| 804 kB 5.3 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting distributed~=2021.11.2\n", + " Downloading distributed-2021.11.2-py3-none-any.whl (802 kB)\n", + "\u001b[K |████████████████████████████████| 802 kB 87.8 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: urllib3<1.27,>=1.25.4 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (1.25.11)\n", + "Collecting fastapi~=0.67.0\n", + " Downloading fastapi-0.67.0-py3-none-any.whl (51 kB)\n", + "\u001b[K |████████████████████████████████| 51 kB 25.4 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting pydantic~=1.5\n", + " Downloading pydantic-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.9 MB)\n", + "\u001b[K |████████████████████████████████| 10.9 MB 111.9 MB/s eta 0:00:01 | 2.6 MB 111.9 MB/s eta 0:00:01▏ | 9.3 MB 111.9 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: click~=7.0 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (7.1.2)\n", + "Collecting kfp~=1.8.0\n", + " Downloading kfp-1.8.11.tar.gz (298 kB)\n", + "\u001b[K |████████████████████████████████| 298 kB 124.8 MB/s eta 0:00:01��████████████████ | 215 kB 124.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting dask~=2021.11.2\n", + " Downloading dask-2021.11.2-py3-none-any.whl (1.0 MB)\n", + "\u001b[K |████████████████████████████████| 1.0 MB 113.4 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting kubernetes~=12.0\n", + " Downloading kubernetes-12.0.1-py2.py3-none-any.whl (1.7 MB)\n", + "\u001b[K |████████████████████████████████| 1.7 MB 93.7 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting python-dotenv~=0.17.0\n", + " Downloading python_dotenv-0.17.1-py2.py3-none-any.whl (18 kB)\n", + "Collecting fsspec~=2021.8.1\n", + " Downloading fsspec-2021.8.1-py3-none-any.whl (119 kB)\n", + "\u001b[K |████████████████████████████████| 119 kB 122.1 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting v3io-frames~=0.10.2\n", + " Downloading v3io_frames-0.10.2-py3-none-any.whl (35 kB)\n", + "Collecting orjson<3.4,>=3\n", + " Downloading orjson-3.3.1-cp37-cp37m-manylinux2014_x86_64.whl (208 kB)\n", + "\u001b[K |████████████████████████████████| 208 kB 124.6 MB/s eta 0:00:01███████████ | 194 kB 124.6 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: nest-asyncio~=1.0 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (1.4.1)\n", + "Requirement already satisfied: pyyaml~=5.1 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (5.3.1)\n", + "Requirement already satisfied: sqlalchemy~=1.3 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (1.3.19)\n", + "Collecting v3io~=0.5.13\n", + " Downloading v3io-0.5.15-py3-none-any.whl (49 kB)\n", + "\u001b[K |████████████████████████████████| 49 kB 61.0 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting pandas~=1.2\n", + " Downloading pandas-1.3.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.3 MB)\n", + "\u001b[K |████████████████████████████████| 11.3 MB 57.6 MB/s eta 0:00:01 | 4.4 MB 57.6 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: ipython~=7.0 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (7.16.1)\n", + "Collecting aiohttp~=3.8\n", + " Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n", + "\u001b[K |████████████████████████████████| 1.1 MB 118.7 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting deepdiff~=5.0\n", + " Downloading deepdiff-5.7.0-py3-none-any.whl (68 kB)\n", + "\u001b[K |████████████████████████████████| 68 kB 107.2 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: chardet<4.0,>=3.0.2 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (3.0.4)\n", + "Collecting storey~=0.10.5\n", + " Downloading storey-0.10.5-py3-none-any.whl (117 kB)\n", + "\u001b[K |████████████████████████████████| 117 kB 128.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting tabulate~=0.8.6\n", + " Downloading tabulate-0.8.9-py3-none-any.whl (25 kB)\n", + "Collecting nuclio-jupyter~=0.8.22\n", + " Downloading nuclio_jupyter-0.8.22-py3-none-any.whl (49 kB)\n", + "\u001b[K |████████████████████████████████| 49 kB 97.2 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting humanfriendly~=8.2\n", + " Downloading humanfriendly-8.2-py2.py3-none-any.whl (86 kB)\n", + "\u001b[K |████████████████████████████████| 86 kB 105.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting mergedeep~=1.3\n", + " Downloading mergedeep-1.3.4-py3-none-any.whl (6.4 kB)\n", + "Collecting google-auth<2.0dev,>=1.25.0\n", + " Downloading google_auth-1.35.0-py2.py3-none-any.whl (152 kB)\n", + "\u001b[K |████████████████████████████████| 152 kB 119.3 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting semver~=2.13\n", + " Downloading semver-2.13.0-py2.py3-none-any.whl (12 kB)\n", + "Collecting inflection~=0.5.0\n", + " Downloading inflection-0.5.1-py2.py3-none-any.whl (9.5 kB)\n", + "Collecting cryptography<3.4,~=3.0\n", + " Downloading cryptography-3.3.2-cp36-abi3-manylinux2010_x86_64.whl (2.6 MB)\n", + "\u001b[K |████████████████████████████████| 2.6 MB 117.3 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting v3iofs~=0.1.7\n", + " Downloading v3iofs-0.1.10-py3-none-any.whl (13 kB)\n", + "Requirement already satisfied: pyarrow<6,>=1 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (1.0.1)\n", + "Requirement already satisfied: numpy<1.22.0,>=1.16.5 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (1.19.2)\n", + "Requirement already satisfied: ipykernel~=5.0 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (5.3.4)\n", + "Collecting pymysql~=1.0\n", + " Downloading PyMySQL-1.0.2-py3-none-any.whl (43 kB)\n", + "\u001b[K |████████████████████████████████| 43 kB 72.6 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: GitPython~=3.0 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (3.1.26)\n", + "Requirement already satisfied: requests~=2.22 in /conda/lib/python3.7/site-packages (from mlrun[complete]==0.10.1-rc1) (2.24.0)\n", + "Collecting alembic<1.6.0,~=1.4\n", + " Downloading alembic-1.5.8-py2.py3-none-any.whl (159 kB)\n", + "\u001b[K |████████████████████████████████| 159 kB 124.0 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting plotly~=5.4; extra == \"complete\"\n", + " Downloading plotly-5.6.0-py2.py3-none-any.whl (27.7 MB)\n", + "\u001b[K |████████████████████████████████| 27.7 MB 64.8 MB/s eta 0:00:01████████▏ | 7.9 MB 64.8 MB/s eta 0:00:01��███████▍ | 15.0 MB 64.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting gcsfs~=2021.8.1; extra == \"complete\"\n", + " Downloading gcsfs-2021.8.1-py2.py3-none-any.whl (23 kB)\n", + "Collecting boto3<1.17.107,~=1.9; extra == \"complete\"\n", + " Downloading boto3-1.17.106-py2.py3-none-any.whl (131 kB)\n", + "\u001b[K |████████████████████████████████| 131 kB 115.7 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting s3fs~=2021.8.1; extra == \"complete\"\n", + " Downloading s3fs-2021.8.1-py3-none-any.whl (26 kB)\n", + "Collecting adlfs~=2021.8.1; extra == \"complete\"\n", + " Downloading adlfs-2021.8.2.tar.gz (38 kB)\n", + "Collecting azure-keyvault-secrets~=4.2; extra == \"complete\"\n", + " Downloading azure_keyvault_secrets-4.3.0-py2.py3-none-any.whl (233 kB)\n", + "\u001b[K |████████████████████████████████| 233 kB 118.7 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting azure-storage-blob~=12.0; extra == \"complete\"\n", + " Downloading azure_storage_blob-12.9.0-py2.py3-none-any.whl (356 kB)\n", + "\u001b[K |████████████████████████████████| 356 kB 116.0 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting azure-identity~=1.5; extra == \"complete\"\n", + " Downloading azure_identity-1.8.0-py3-none-any.whl (130 kB)\n", + "\u001b[K |████████████████████████████████| 130 kB 110.9 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting botocore<1.20.107,>=1.20.106; extra == \"complete\"\n", + " Downloading botocore-1.20.106-py2.py3-none-any.whl (7.7 MB)\n", + "\u001b[K |████████████████████████████████| 7.7 MB 53.8 MB/s eta 0:00:01 | 276 kB 53.8 MB/s eta 0:00:01�▉| 7.6 MB 53.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting aiobotocore~=1.4.0; extra == \"complete\"\n", + " Downloading aiobotocore-1.4.2.tar.gz (52 kB)\n", + "\u001b[K |████████████████████████████████| 52 kB 58.0 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: psutil>=5.0 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (5.7.2)\n", + "Requirement already satisfied: zict>=0.1.3 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (2.0.0)\n", + "Requirement already satisfied: jinja2 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (2.11.2)\n", + "Requirement already satisfied: setuptools in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (41.0.0)\n", + "Requirement already satisfied: toolz>=0.8.2 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (0.11.1)\n", + "Requirement already satisfied: cloudpickle>=1.5.0 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (1.6.0)\n", + "Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (2.2.2)\n", + "Requirement already satisfied: msgpack>=0.6.0 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (1.0.0)\n", + "Requirement already satisfied: tblib>=1.6.0 in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (1.7.0)\n", + "Requirement already satisfied: tornado>=5; python_version < \"3.8\" in /conda/lib/python3.7/site-packages (from distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (6.0.4)\n", + "Collecting starlette==0.14.2\n", + " Downloading starlette-0.14.2-py3-none-any.whl (60 kB)\n", + "\u001b[K |████████████████████████████████| 60 kB 98.0 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /conda/lib/python3.7/site-packages (from pydantic~=1.5->mlrun[complete]==0.10.1-rc1) (3.7.4.3)\n", + "Requirement already satisfied: absl-py<2,>=0.9 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (0.10.0)\n", + "Requirement already satisfied: google-cloud-storage<2,>=1.20.0 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.32.0)\n", + "Collecting google-api-python-client<2,>=1.7.8\n", + " Downloading google_api_python_client-1.12.10-py2.py3-none-any.whl (61 kB)\n", + "\u001b[K |████████████████████████████████| 61 kB 8.5 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: requests-toolbelt<1,>=0.8.0 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (0.9.1)\n", + "Requirement already satisfied: kfp-server-api<2.0.0,>=1.1.2 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.5.0)\n", + "Requirement already satisfied: jsonschema<4,>=3.0.1 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (3.2.0)\n", + "Requirement already satisfied: Deprecated<2,>=1.2.7 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.2.10)\n", + "Requirement already satisfied: strip-hints<1,>=0.1.8 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (0.1.8)\n", + "Collecting docstring-parser<1,>=0.7.3\n", + " Downloading docstring_parser-0.13.tar.gz (23 kB)\n", + " Installing build dependencies ... \u001b[?25ldone\n", + "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", + "\u001b[?25h Preparing wheel metadata ... \u001b[?25ldone\n", + "\u001b[?25hCollecting kfp-pipeline-spec<0.2.0,>=0.1.13\n", + " Downloading kfp_pipeline_spec-0.1.13-py3-none-any.whl (18 kB)\n", + "Collecting fire<1,>=0.3.1\n", + " Downloading fire-0.4.0.tar.gz (87 kB)\n", + "\u001b[K |████████████████████████████████| 87 kB 106.2 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: protobuf<4,>=3.13.0 in /conda/lib/python3.7/site-packages (from kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (3.19.3)\n", + "Collecting uritemplate<4,>=3.0.1\n", + " Downloading uritemplate-3.0.1-py2.py3-none-any.whl (15 kB)\n", + "Collecting typer<1.0,>=0.3.2\n", + " Downloading typer-0.4.0-py3-none-any.whl (27 kB)\n", + "Requirement already satisfied: packaging>=20.0 in /conda/lib/python3.7/site-packages (from dask~=2021.11.2->mlrun[complete]==0.10.1-rc1) (20.4)\n", + "Requirement already satisfied: partd>=0.3.10 in /conda/lib/python3.7/site-packages (from dask~=2021.11.2->mlrun[complete]==0.10.1-rc1) (1.1.0)\n", + "Requirement already satisfied: six>=1.9.0 in /conda/lib/python3.7/site-packages (from kubernetes~=12.0->mlrun[complete]==0.10.1-rc1) (1.12.0)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /conda/lib/python3.7/site-packages (from kubernetes~=12.0->mlrun[complete]==0.10.1-rc1) (0.57.0)\n", + "Requirement already satisfied: python-dateutil>=2.5.3 in /conda/lib/python3.7/site-packages (from kubernetes~=12.0->mlrun[complete]==0.10.1-rc1) (2.8.1)\n", + "Requirement already satisfied: certifi>=14.05.14 in /conda/lib/python3.7/site-packages (from kubernetes~=12.0->mlrun[complete]==0.10.1-rc1) (2021.10.8)\n", + "Requirement already satisfied: requests-oauthlib in /conda/lib/python3.7/site-packages (from kubernetes~=12.0->mlrun[complete]==0.10.1-rc1) (1.3.0)\n", + "Requirement already satisfied: grpcio!=1.34.0,<1.42,>=1.30 in /conda/lib/python3.7/site-packages (from v3io-frames~=0.10.2->mlrun[complete]==0.10.1-rc1) (1.32.0)\n", + "Requirement already satisfied: grpcio-tools!=1.34.0,<1.42,>=1.30 in /conda/lib/python3.7/site-packages (from v3io-frames~=0.10.2->mlrun[complete]==0.10.1-rc1) (1.32.0)\n", + "Requirement already satisfied: googleapis-common-protos>=1.5.3 in /conda/lib/python3.7/site-packages (from v3io-frames~=0.10.2->mlrun[complete]==0.10.1-rc1) (1.54.0)\n", + "Requirement already satisfied: future>=0.18.2 in /conda/lib/python3.7/site-packages (from v3io~=0.5.13->mlrun[complete]==0.10.1-rc1) (0.18.2)\n", + "Requirement already satisfied: ujson>=3.0.0 in /conda/lib/python3.7/site-packages (from v3io~=0.5.13->mlrun[complete]==0.10.1-rc1) (5.1.0)\n", + "Requirement already satisfied: pytz>=2017.3 in /conda/lib/python3.7/site-packages (from pandas~=1.2->mlrun[complete]==0.10.1-rc1) (2020.1)\n", + "Requirement already satisfied: decorator in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (4.4.2)\n", + "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (4.8.0)\n", + "Requirement already satisfied: traitlets>=4.2 in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (4.3.3)\n", + "Requirement already satisfied: pygments in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (2.7.1)\n", + "Requirement already satisfied: backcall in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (0.2.0)\n", + "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (3.0.8)\n", + "Requirement already satisfied: pickleshare in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (0.7.5)\n", + "Requirement already satisfied: jedi>=0.10 in /conda/lib/python3.7/site-packages (from ipython~=7.0->mlrun[complete]==0.10.1-rc1) (0.17.2)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /conda/lib/python3.7/site-packages (from aiohttp~=3.8->mlrun[complete]==0.10.1-rc1) (1.5.1)\n", + "Collecting charset-normalizer<3.0,>=2.0\n", + " Downloading charset_normalizer-2.0.12-py3-none-any.whl (39 kB)\n", + "Collecting asynctest==0.13.0; python_version < \"3.8\"\n", + " Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n", + "Collecting frozenlist>=1.1.1\n", + " Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n", + "\u001b[K |████████████████████████████████| 144 kB 117.3 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n", + " Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n", + "Collecting aiosignal>=1.1.2\n", + " Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n", + "Requirement already satisfied: attrs>=17.3.0 in /conda/lib/python3.7/site-packages (from aiohttp~=3.8->mlrun[complete]==0.10.1-rc1) (20.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /conda/lib/python3.7/site-packages (from aiohttp~=3.8->mlrun[complete]==0.10.1-rc1) (4.7.6)\n", + "Collecting ordered-set==4.0.2\n", + " Downloading ordered-set-4.0.2.tar.gz (10 kB)\n", + "Requirement already satisfied: notebook>=5.2.0 in /conda/lib/python3.7/site-packages (from nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (6.1.6)\n", + "Requirement already satisfied: nbconvert>=5.4 in /conda/lib/python3.7/site-packages (from nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (6.0.7)\n", + "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /conda/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.10.1-rc1) (4.6)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /conda/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.10.1-rc1) (4.1.1)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /conda/lib/python3.7/site-packages (from google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.10.1-rc1) (0.2.8)\n", + "Requirement already satisfied: cffi>=1.12 in /conda/lib/python3.7/site-packages (from cryptography<3.4,~=3.0->mlrun[complete]==0.10.1-rc1) (1.12.2)\n", + "Requirement already satisfied: jupyter-client in /conda/lib/python3.7/site-packages (from ipykernel~=5.0->mlrun[complete]==0.10.1-rc1) (6.1.7)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /conda/lib/python3.7/site-packages (from GitPython~=3.0->mlrun[complete]==0.10.1-rc1) (4.0.9)\n", + "Requirement already satisfied: idna<3,>=2.5 in /conda/lib/python3.7/site-packages (from requests~=2.22->mlrun[complete]==0.10.1-rc1) (2.8)\n", + "Collecting Mako\n", + " Downloading Mako-1.1.6-py2.py3-none-any.whl (75 kB)\n", + "\u001b[K |████████████████████████████████| 75 kB 100.2 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting python-editor>=0.3\n", + " Downloading python_editor-1.0.4-py3-none-any.whl (4.9 kB)\n", + "Collecting tenacity>=6.2.0\n", + " Downloading tenacity-8.0.1-py3-none-any.whl (24 kB)\n", + "Requirement already satisfied: google-auth-oauthlib in /conda/lib/python3.7/site-packages (from gcsfs~=2021.8.1; extra == \"complete\"->mlrun[complete]==0.10.1-rc1) (0.4.1)\n", + "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /conda/lib/python3.7/site-packages (from boto3<1.17.107,~=1.9; extra == \"complete\"->mlrun[complete]==0.10.1-rc1) (0.10.0)\n", + "Collecting s3transfer<0.5.0,>=0.4.0\n", + " Downloading s3transfer-0.4.2-py2.py3-none-any.whl (79 kB)\n", + "\u001b[K |████████████████████████████████| 79 kB 99.5 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting azure-core>=1.7.0\n", + " Downloading azure_core-1.23.0-py3-none-any.whl (179 kB)\n", + "\u001b[K |████████████████████████████████| 179 kB 123.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting azure-datalake-store<0.1,>=0.0.46\n", + " Downloading azure_datalake_store-0.0.52-py2.py3-none-any.whl (61 kB)\n", + "\u001b[K |████████████████████████████████| 61 kB 21.5 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting azure-common~=1.1\n", + " Downloading azure_common-1.1.28-py2.py3-none-any.whl (14 kB)\n", + "Collecting msrest>=0.6.21\n", + " Downloading msrest-0.6.21-py2.py3-none-any.whl (85 kB)\n", + "\u001b[K |████████████████████████████████| 85 kB 102.8 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting msal-extensions~=0.3.0\n", + " Downloading msal_extensions-0.3.1-py2.py3-none-any.whl (18 kB)\n", + "Collecting msal<2.0.0,>=1.12.0\n", + " Downloading msal-1.17.0-py2.py3-none-any.whl (79 kB)\n", + "\u001b[K |████████████████████████████████| 79 kB 106.4 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: wrapt>=1.10.10 in /conda/lib/python3.7/site-packages (from aiobotocore~=1.4.0; extra == \"complete\"->mlrun[complete]==0.10.1-rc1) (1.12.1)\n", + "Collecting aioitertools>=0.5.1\n", + " Downloading aioitertools-0.10.0-py3-none-any.whl (23 kB)\n", + "Requirement already satisfied: heapdict in /conda/lib/python3.7/site-packages (from zict>=0.1.3->distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (1.0.1)\n", + "Requirement already satisfied: MarkupSafe>=0.23 in /conda/lib/python3.7/site-packages (from jinja2->distributed~=2021.11.2->mlrun[complete]==0.10.1-rc1) (1.1.1)\n", + "Requirement already satisfied: google-resumable-media<2.0dev,>=1.0.0 in /conda/lib/python3.7/site-packages (from google-cloud-storage<2,>=1.20.0->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.1.0)\n", + "Requirement already satisfied: google-cloud-core<2.0dev,>=1.4.1 in /conda/lib/python3.7/site-packages (from google-cloud-storage<2,>=1.20.0->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.4.3)\n", + "Requirement already satisfied: google-api-core<3dev,>=1.21.0; python_version >= \"3\" in /conda/lib/python3.7/site-packages (from google-api-python-client<2,>=1.7.8->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.22.2)\n", + "Collecting httplib2<1dev,>=0.15.0\n", + " Downloading httplib2-0.20.4-py3-none-any.whl (96 kB)\n", + "\u001b[K |████████████████████████████████| 96 kB 107.3 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting google-auth-httplib2>=0.0.3\n", + " Downloading google_auth_httplib2-0.1.0-py2.py3-none-any.whl (9.3 kB)\n", + "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /conda/lib/python3.7/site-packages (from jsonschema<4,>=3.0.1->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (2.0.0)\n", + "Requirement already satisfied: pyrsistent>=0.14.0 in /conda/lib/python3.7/site-packages (from jsonschema<4,>=3.0.1->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (0.17.3)\n", + "Requirement already satisfied: wheel in /conda/lib/python3.7/site-packages (from strip-hints<1,>=0.1.8->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (0.33.1)\n", + "Requirement already satisfied: termcolor in /conda/lib/python3.7/site-packages (from fire<1,>=0.3.1->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.1.0)\n", + "Requirement already satisfied: pyparsing>=2.0.2 in /conda/lib/python3.7/site-packages (from packaging>=20.0->dask~=2021.11.2->mlrun[complete]==0.10.1-rc1) (2.4.7)\n", + "Requirement already satisfied: locket in /conda/lib/python3.7/site-packages (from partd>=0.3.10->dask~=2021.11.2->mlrun[complete]==0.10.1-rc1) (0.2.0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /conda/lib/python3.7/site-packages (from requests-oauthlib->kubernetes~=12.0->mlrun[complete]==0.10.1-rc1) (3.1.0)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /conda/lib/python3.7/site-packages (from pexpect; sys_platform != \"win32\"->ipython~=7.0->mlrun[complete]==0.10.1-rc1) (0.6.0)\n", + "Requirement already satisfied: ipython-genutils in /conda/lib/python3.7/site-packages (from traitlets>=4.2->ipython~=7.0->mlrun[complete]==0.10.1-rc1) (0.2.0)\n", + "Requirement already satisfied: wcwidth in /conda/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython~=7.0->mlrun[complete]==0.10.1-rc1) (0.2.5)\n", + "Requirement already satisfied: parso<0.8.0,>=0.7.0 in /conda/lib/python3.7/site-packages (from jedi>=0.10->ipython~=7.0->mlrun[complete]==0.10.1-rc1) (0.7.0)\n", + "Requirement already satisfied: prometheus-client in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.8.0)\n", + "Requirement already satisfied: jupyter-core>=4.6.1 in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (4.6.3)\n", + "Requirement already satisfied: terminado>=0.8.3 in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.9.1)\n", + "Requirement already satisfied: pyzmq>=17 in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (19.0.2)\n", + "Requirement already satisfied: Send2Trash in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (1.5.0)\n", + "Requirement already satisfied: nbformat in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (5.0.8)\n", + "Requirement already satisfied: argon2-cffi in /conda/lib/python3.7/site-packages (from notebook>=5.2.0->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (20.1.0)\n", + "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.5.1)\n", + "Requirement already satisfied: entrypoints>=0.2.2 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.3)\n", + "Requirement already satisfied: pandocfilters>=1.4.1 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (1.4.2)\n", + "Requirement already satisfied: defusedxml in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.6.0)\n", + "Requirement already satisfied: jupyterlab-pygments in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.1.2)\n", + "Requirement already satisfied: bleach in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (3.2.1)\n", + "Requirement already satisfied: mistune<2,>=0.8.1 in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.8.4)\n", + "Requirement already satisfied: testpath in /conda/lib/python3.7/site-packages (from nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.4.4)\n", + "Requirement already satisfied: pyasn1>=0.1.3 in /conda/lib/python3.7/site-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2.0dev,>=1.25.0->mlrun[complete]==0.10.1-rc1) (0.4.8)\n", + "Requirement already satisfied: pycparser in /conda/lib/python3.7/site-packages (from cffi>=1.12->cryptography<3.4,~=3.0->mlrun[complete]==0.10.1-rc1) (2.19)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /conda/lib/python3.7/site-packages (from gitdb<5,>=4.0.1->GitPython~=3.0->mlrun[complete]==0.10.1-rc1) (5.0.0)\n", + "Requirement already satisfied: adal>=0.4.2 in /conda/lib/python3.7/site-packages (from azure-datalake-store<0.1,>=0.0.46->adlfs~=2021.8.1; extra == \"complete\"->mlrun[complete]==0.10.1-rc1) (1.2.4)\n", + "Collecting isodate>=0.6.0\n", + " Downloading isodate-0.6.1-py2.py3-none-any.whl (41 kB)\n", + "\u001b[K |████████████████████████████████| 41 kB 38.2 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting portalocker<3,>=1.0; python_version >= \"3.5\" and platform_system != \"Windows\"\n", + " Downloading portalocker-2.4.0-py2.py3-none-any.whl (16 kB)\n", + "Requirement already satisfied: PyJWT[crypto]<3,>=1.0.0 in /conda/lib/python3.7/site-packages (from msal<2.0.0,>=1.12.0->azure-identity~=1.5; extra == \"complete\"->mlrun[complete]==0.10.1-rc1) (1.7.1)\n", + "Requirement already satisfied: google-crc32c<2.0dev,>=1.0; python_version >= \"3.5\" in /conda/lib/python3.7/site-packages (from google-resumable-media<2.0dev,>=1.0.0->google-cloud-storage<2,>=1.20.0->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (1.0.0)\n", + "Requirement already satisfied: zipp>=0.5 in /conda/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->jsonschema<4,>=3.0.1->kfp~=1.8.0->mlrun[complete]==0.10.1-rc1) (3.3.1)\n", + "Requirement already satisfied: async-generator in /conda/lib/python3.7/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (1.10)\n", + "Requirement already satisfied: webencodings in /conda/lib/python3.7/site-packages (from bleach->nbconvert>=5.4->nuclio-jupyter~=0.8.22->mlrun[complete]==0.10.1-rc1) (0.5.1)\n", + "Building wheels for collected packages: kfp, adlfs, aiobotocore, docstring-parser, fire, ordered-set\n", + " Building wheel for kfp (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for kfp: filename=kfp-1.8.11-py3-none-any.whl size=414450 sha256=fb0464b7d95a95a6f16966c67ec84d6cbb8dbda6054422afade5746e91d5437c\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-zl92s5bu/wheels/85/1e/ee/a14b49663bddf9e72d1c269cbe53970167bfabb53cadbbea3a\n", + " Building wheel for adlfs (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for adlfs: filename=adlfs-2021.8.2-py3-none-any.whl size=21470 sha256=e0e330c3741fb0b004db542dddfc46a95c6716e8f77b17fbd6ef093198a5350d\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-zl92s5bu/wheels/0d/88/1d/e06072abb7fb4d59b5cf94e194e53017dfa2dc47af4dec88b7\n", + " Building wheel for aiobotocore (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for aiobotocore: filename=aiobotocore-1.4.2-py3-none-any.whl size=49909 sha256=b5aa08829131f975611cb79f49faec16e36436045cf31e8e4077f39038c87e3e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-zl92s5bu/wheels/33/e7/d9/b297a9aa9c43d56bc2463e6e2771655ff638f30b30f0b61fcb\n", + " Building wheel for docstring-parser (PEP 517) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for docstring-parser: filename=docstring_parser-0.13-py3-none-any.whl size=31866 sha256=4345d5c0304e35b9e0e36046bfde99d54eb8e21c30ab33dbd435062ef26ab441\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-zl92s5bu/wheels/bd/88/3c/d1aa049309f7945178cac9fbe6561a86424f432da57c18ca0f\n", + " Building wheel for fire (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for fire: filename=fire-0.4.0-py2.py3-none-any.whl size=115929 sha256=05b6e842d090dc02821d3dbf288f50c469e2362061c8d610b0abe0426e5a4f2c\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-zl92s5bu/wheels/8a/67/fb/2e8a12fa16661b9d5af1f654bd199366799740a85c64981226\n", + " Building wheel for ordered-set (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for ordered-set: filename=ordered_set-4.0.2-py2.py3-none-any.whl size=8209 sha256=7ceb690723e761ecae1e9381e8300bd13bacbb3a64e5905376a1392f8589b7c1\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-zl92s5bu/wheels/73/2b/f6/26e9f84153c25050fe7c09e88f8e32a6be3c7034a38c418319\n", + "Successfully built kfp adlfs aiobotocore docstring-parser fire ordered-set\n", + "Installing collected packages: fsspec, dask, distributed, pydantic, starlette, fastapi, google-auth, kubernetes, httplib2, uritemplate, google-auth-httplib2, google-api-python-client, tabulate, docstring-parser, kfp-pipeline-spec, fire, typer, kfp, python-dotenv, pandas, v3io-frames, orjson, v3io, charset-normalizer, asynctest, frozenlist, async-timeout, aiosignal, aiohttp, ordered-set, deepdiff, v3iofs, storey, botocore, s3transfer, boto3, nuclio-jupyter, humanfriendly, mergedeep, semver, inflection, cryptography, pymysql, Mako, python-editor, alembic, tenacity, plotly, gcsfs, aioitertools, aiobotocore, s3fs, azure-core, azure-datalake-store, msal, portalocker, msal-extensions, azure-identity, isodate, msrest, azure-storage-blob, adlfs, azure-common, azure-keyvault-secrets, mlrun\n", + " Attempting uninstall: fsspec\n", + " Found existing installation: fsspec 0.8.3\n", + " Uninstalling fsspec-0.8.3:\n", + " Successfully uninstalled fsspec-0.8.3\n", + " Attempting uninstall: dask\n", + " Found existing installation: dask 2.30.0\n", + " Uninstalling dask-2.30.0:\n", + " Successfully uninstalled dask-2.30.0\n", + " Attempting uninstall: distributed\n", + " Found existing installation: distributed 2.30.0\n", + " Uninstalling distributed-2.30.0:\n", + " Successfully uninstalled distributed-2.30.0\n", + " Attempting uninstall: google-auth\n", + " Found existing installation: google-auth 1.22.1\n", + " Uninstalling google-auth-1.22.1:\n", + " Successfully uninstalled google-auth-1.22.1\n", + " Attempting uninstall: kubernetes\n", + " Found existing installation: kubernetes 11.0.0\n", + " Uninstalling kubernetes-11.0.0:\n", + " Successfully uninstalled kubernetes-11.0.0\n", + " Attempting uninstall: tabulate\n", + " Found existing installation: tabulate 0.8.3\n", + " Uninstalling tabulate-0.8.3:\n", + " Successfully uninstalled tabulate-0.8.3\n", + " Attempting uninstall: kfp\n", + " Found existing installation: kfp 1.0.1\n", + " Uninstalling kfp-1.0.1:\n", + " Successfully uninstalled kfp-1.0.1\n", + " Attempting uninstall: pandas\n", + " Found existing installation: pandas 1.1.3\n", + " Uninstalling pandas-1.1.3:\n", + " Successfully uninstalled pandas-1.1.3\n", + " Attempting uninstall: v3io-frames\n", + " Found existing installation: v3io-frames 0.9.6\n", + " Uninstalling v3io-frames-0.9.6:\n", + " Successfully uninstalled v3io-frames-0.9.6\n", + " Attempting uninstall: v3io\n", + " Found existing installation: v3io 0.5.7\n", + " Uninstalling v3io-0.5.7:\n", + " Successfully uninstalled v3io-0.5.7\n", + " Attempting uninstall: async-timeout\n", + " Found existing installation: async-timeout 3.0.1\n", + " Uninstalling async-timeout-3.0.1:\n", + " Successfully uninstalled async-timeout-3.0.1\n", + " Attempting uninstall: aiohttp\n", + " Found existing installation: aiohttp 3.6.3\n", + " Uninstalling aiohttp-3.6.3:\n", + " Successfully uninstalled aiohttp-3.6.3\n", + " Attempting uninstall: botocore\n", + " Found existing installation: botocore 1.23.38\n", + " Uninstalling botocore-1.23.38:\n", + " Successfully uninstalled botocore-1.23.38\n", + " Attempting uninstall: s3transfer\n", + " Found existing installation: s3transfer 0.5.0\n", + " Uninstalling s3transfer-0.5.0:\n", + " Successfully uninstalled s3transfer-0.5.0\n", + " Attempting uninstall: boto3\n", + " Found existing installation: boto3 1.20.38\n", + " Uninstalling boto3-1.20.38:\n", + " Successfully uninstalled boto3-1.20.38\n", + " Attempting uninstall: nuclio-jupyter\n", + " Found existing installation: nuclio-jupyter 0.8.9\n", + " Uninstalling nuclio-jupyter-0.8.9:\n", + " Successfully uninstalled nuclio-jupyter-0.8.9\n", + " Attempting uninstall: cryptography\n", + " Found existing installation: cryptography 2.6.1\n", + " Uninstalling cryptography-2.6.1:\n", + " Successfully uninstalled cryptography-2.6.1\n", + " Attempting uninstall: pymysql\n", + " Found existing installation: PyMySQL 0.10.1\n", + " Uninstalling PyMySQL-0.10.1:\n", + " Successfully uninstalled PyMySQL-0.10.1\n", + "\u001b[31mERROR: After October 2020 you may experience errors when installing or updating packages. This is because pip will change the way that it resolves dependency conflicts.\n", + "\n", + "We recommend you use --use-feature=2020-resolver to test your packages with the new resolver before it becomes the default.\n", + "\n", + "kfp 1.8.11 requires cloudpickle<3,>=2.0.0, but you'll have cloudpickle 1.6.0 which is incompatible.\n", + "google-api-python-client 1.12.10 requires six<2dev,>=1.13.0, but you'll have six 1.12.0 which is incompatible.\n", + "storey 0.10.5 requires grpcio<1.42,>1.34.0, but you'll have grpcio 1.32.0 which is incompatible.\n", + "storey 0.10.5 requires grpcio-tools<1.42,>1.34.0, but you'll have grpcio-tools 1.32.0 which is incompatible.\n", + "aioitertools 0.10.0 requires typing_extensions>=4.0; python_version < \"3.10\", but you'll have typing-extensions 3.7.4.3 which is incompatible.\n", + "azure-core 1.23.0 requires typing-extensions>=4.0.1, but you'll have typing-extensions 3.7.4.3 which is incompatible.\u001b[0m\n", + "Successfully installed Mako adlfs aiobotocore aiohttp aioitertools aiosignal alembic async-timeout asynctest azure-common azure-core azure-datalake-store azure-identity azure-keyvault-secrets azure-storage-blob boto3 botocore charset-normalizer cryptography dask deepdiff distributed docstring-parser fastapi fire frozenlist fsspec gcsfs google-api-python-client google-auth google-auth-httplib2 httplib2 humanfriendly inflection isodate kfp kfp-pipeline-spec kubernetes mergedeep mlrun msal msal-extensions msrest nuclio-jupyter ordered-set orjson pandas plotly portalocker pydantic pymysql python-dotenv python-editor s3fs s3transfer semver starlette storey tabulate tenacity typer uritemplate v3io v3io-frames v3iofs\n" + ] + } + ], + "source": [ + "# uncomment the command below\n", + "!/User/align_mlrun.sh" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Kernel Restart\n", + "After installing or updating the MLRun package, restart the notebook kernel in your environment!" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -36,7 +458,7 @@ "source": [ "Project is a key entity in Iguazio's platform. Every object we create (e.g. runs, functions, workflows, features, artifacts, models, etc.) are associated with a `project` namespace.
\n", "Run the following code to initialize your MLRun environment to use a \"quick-start-tutorial-<username>\"project
\n", - "For more information about MLRun projects, see the MLRun [projects documentation](../projects/overview.md)." + "For more information about MLRun projects, see the MLRun [projects documentation](https://docs.mlrun.org/en/latest/projects/project.html)." ] }, { @@ -58,7 +480,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:40:55,401 [info] loaded project quick-start-tutorial from MLRun DB\n" + "> 2022-03-08 08:11:35,662 [info] loaded project quick-start-tutorial from MLRun DB\n" ] } ], @@ -151,9 +573,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:45:04,642 [info] starting run prep_data uid=79fb2d02cab74e22ad2d9f52f7230c83 DB=http://mlrun-api:8080\n", - "> 2022-03-06 20:45:04,774 [info] Job is running in the background, pod: prep-data-9j58x\n", - "> 2022-03-06 20:45:08,881 [info] run executed, status=completed\n", + "> 2022-03-08 08:11:49,795 [info] starting run prep_data uid=1b68f4020d884bdd83510fa4ce12a486 DB=http://mlrun-api:8080\n", + "> 2022-03-08 08:11:49,917 [info] Job is running in the background, pod: prep-data-tckvl\n", + "> 2022-03-08 08:12:28,712 [info] run executed, status=completed\n", "final state: completed\n" ] }, @@ -328,26 +750,26 @@ " \n", " \n", " quick-start-tutorial-admin\n", - "
...f7230c83
\n", + "
...ce12a486
\n", " 0\n", - " Mar 06 20:45:08\n", + " Mar 08 08:12:27\n", " completed\n", " prep_data\n", - "
v3io_user=admin
kind=job
owner=admin
mlrun/client_version=0.10.0
host=prep-data-9j58x
\n", + "
v3io_user=admin
kind=job
owner=admin
mlrun/client_version=0.10.0
host=prep-data-tckvl
\n", "
source_url
\n", " \n", "
num_rows=150
\n", - "
cleaned_data
\n", + "
cleaned_data
\n", " \n", " \n", "\n", "\n", - "
\n", + "
\n", "
\n", - " Title\n", - " ×\n", + " Title\n", + " ×\n", "
\n", - " \n", + " \n", "
\n", "
\n" ], @@ -368,7 +790,7 @@ { "data": { "text/html": [ - " > to track results use the .show() or .logs() methods or click here to open in UI" + " > to track results use the .show() or .logs() methods or click here to open in UI" ], "text/plain": [ "" @@ -381,7 +803,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:45:10,990 [info] run executed, status=completed\n" + "> 2022-03-08 08:12:29,320 [info] run executed, status=completed\n" ] } ], @@ -514,7 +936,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:46:26,619 [info] starting run train-iris-train_iris uid=f67d76c0e96f42c0a0e7437a666fd9b8 DB=http://mlrun-api:8080\n" + "> 2022-03-08 08:13:15,507 [info] starting run train-iris-train_iris uid=af978d254dc1496b8c1e2d2b4b6113f0 DB=http://mlrun-api:8080\n" ] }, { @@ -688,26 +1110,26 @@ " \n", " \n", " quick-start-tutorial-admin\n", - "
...666fd9b8
\n", + "
...4b6113f0
\n", " 0\n", - " Mar 06 20:46:26\n", + " Mar 08 08:13:15\n", " completed\n", " train-iris-train_iris\n", - "
v3io_user=admin
kind=
owner=admin
host=jupyter-admin-86d8bcc7dc-n6cbv
\n", + "
v3io_user=admin
kind=
owner=admin
host=jupyter-admin-5f8d9cd4d8-mz42n
\n", "
dataset
\n", "
label_column=label
\n", "
accuracy=1.0
f1_score=1.0
precision_score=1.0
recall_score=1.0
auc-micro=1.0
auc-macro=1.0
auc-weighted=1.0
\n", - "
feature-importance
test_set
confusion-matrix
roc-curves
model
\n", + "
feature-importance
test_set
confusion-matrix
roc-curves
model
\n", " \n", " \n", "\n", "\n", - "
\n", + "
\n", "
\n", - " Title\n", - " ×\n", + " Title\n", + " ×\n", "
\n", - " \n", + " \n", "
\n", "
\n" ], @@ -728,7 +1150,7 @@ { "data": { "text/html": [ - " > to track results use the .show() or .logs() methods or click here to open in UI" + " > to track results use the .show() or .logs() methods or click here to open in UI" ], "text/plain": [ "" @@ -741,7 +1163,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:46:28,664 [info] run executed, status=completed\n" + "> 2022-03-08 08:13:17,147 [info] run executed, status=completed\n" ] } ], @@ -854,7 +1276,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -917,24 +1339,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:48:56,501 [info] model my_model was loaded\n", - "> 2022-03-06 20:48:56,502 [info] Loaded ['my_model']\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/User/.pythonlibs/jupyter-admin/lib/python3.7/site-packages/sklearn/base.py:451: UserWarning:\n", - "\n", - "X does not have valid feature names, but RandomForestClassifier was fitted with feature names\n", - "\n" + "> 2022-03-08 08:13:34,013 [info] model my_model was loaded\n", + "> 2022-03-08 08:13:34,014 [info] Loaded ['my_model']\n" ] }, { "data": { "text/plain": [ - "{'id': 'f055818e62754ca68f149aab4a75f15b',\n", + "{'id': 'ac38eedf717443ee8daa4f3afda30fb0',\n", " 'model_name': 'my_model',\n", " 'outputs': [0, 2]}" ] @@ -974,13 +1386,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:49:12,321 [info] Starting remote function deploy\n", - "2022-03-06 20:49:12 (info) Deploying function\n", - "2022-03-06 20:49:12 (info) Building\n", - "2022-03-06 20:49:12 (info) Staging files and preparing base images\n", - "2022-03-06 20:49:12 (info) Building processor image\n", - "2022-03-06 20:49:16 (info) Build complete\n", - "> 2022-03-06 20:49:22,030 [info] successfully deployed function: {'internal_invocation_urls': ['nuclio-quick-start-tutorial-admin-serving.default-tenant.svc.cluster.local:8080'], 'external_invocation_urls': ['quick-start-tutorial-admin-serving-quick-start-tutorial-admin.default-tenant.app.product-3-2-2.iguazio-cd2.com/']}\n" + "> 2022-03-08 08:13:35,146 [info] Starting remote function deploy\n", + "2022-03-08 08:13:35 (info) Deploying function\n", + "2022-03-08 08:13:35 (info) Building\n", + "2022-03-08 08:13:35 (info) Staging files and preparing base images\n", + "2022-03-08 08:13:35 (info) Building processor image\n", + "2022-03-08 08:14:31 (info) Build complete\n", + "2022-03-08 08:14:36 (info) Function deploy complete\n", + "> 2022-03-08 08:14:36,606 [info] successfully deployed function: {'internal_invocation_urls': ['nuclio-quick-start-tutorial-admin-serving.default-tenant.svc.cluster.local:8080'], 'external_invocation_urls': ['quick-start-tutorial-admin-serving-quick-start-tutorial-admin.default-tenant.app.product-3-2-3-a.iguazio-cd0.com/']}\n" ] } ], @@ -1015,7 +1428,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "The address for the function is http://quick-start-tutorial-admin-serving-quick-start-tutorial-admin.default-tenant.app.product-3-2-2.iguazio-cd2.com/ \n", + "The address for the function is http://quick-start-tutorial-admin-serving-quick-start-tutorial-admin.default-tenant.app.product-3-2-3-a.iguazio-cd0.com/ \n", "\n", "{\"name\": \"ModelRouter\", \"version\": \"v2\", \"extensions\": []}" ] @@ -1051,13 +1464,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "> 2022-03-06 20:49:51,601 [info] invoking function: {'method': 'POST', 'path': 'http://nuclio-quick-start-tutorial-admin-serving.default-tenant.svc.cluster.local:8080/v2/models/my_model/infer'}\n" + "> 2022-03-08 08:14:40,613 [info] invoking function: {'method': 'POST', 'path': 'http://nuclio-quick-start-tutorial-admin-serving.default-tenant.svc.cluster.local:8080/v2/models/my_model/infer'}\n" ] }, { "data": { "text/plain": [ - "{'id': 'ebca197f-5d71-402c-9d8a-aaa5397239c9',\n", + "{'id': '1f0732d8-c668-4403-b9ef-e6b64d759092',\n", " 'model_name': 'my_model',\n", " 'outputs': [0, 2]}" ] @@ -1107,7 +1520,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -1121,7 +1534,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -1164,13 +1577,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
Pipeline running (id=20bfd2ed-7483-42e3-b22c-9d08c9a8e159), click here to view the details in MLRun UI
" + "
Pipeline running (id=49bb9ed7-8622-41db-9db1-3e354e755a23), click here to view the details in MLRun UI
" ], "text/plain": [ "" @@ -1188,16 +1601,110 @@ "\n", "\n", - "\n", - "\n", + "\n", + "\n", "kfp\n", - "\n", + "\n", + "\n", + "\n", + "getting-started-tutorial-sk72z-2409247289\n", + "\n", + "prep-data-prep-data\n", + "\n", + "\n", + "\n", + "getting-started-tutorial-sk72z-713919998\n", + "\n", + "train\n", + "\n", + "\n", + "\n", + "getting-started-tutorial-sk72z-2409247289->getting-started-tutorial-sk72z-713919998\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "getting-started-tutorial-sk72z-2948199946\n", + "\n", + "test-classifier\n", + "\n", + "\n", + "\n", + "getting-started-tutorial-sk72z-713919998->getting-started-tutorial-sk72z-2948199946\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "getting-started-tutorial-sk72z-3414330050\n", + "\n", + "\n", + "\n", + "\n", + "deploy-serving\n", + "\n", + "\n", + "\n", + "getting-started-tutorial-sk72z-713919998->getting-started-tutorial-sk72z-3414330050\n", + "\n", + "\n", + "\n", "\n", "\n" ], "text/plain": [ - "" + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "

Run Results

Workflow 49bb9ed7-8622-41db-9db1-3e354e755a23 finished, state=Succeeded
click the hyper links below to see detailed results
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
uidstartstatenameparametersresults
Mar 08 08:35:52completedtest-classifier
label_column=label
accuracy=1.0
test-error=0.0
auc-micro=1.0
auc-weighted=1.0
f1-score=1.0
precision_score=1.0
recall_score=1.0
Mar 08 08:35:40completedtrain
label_column=label
accuracy=1.0
f1_score=1.0
precision_score=1.0
recall_score=1.0
auc-micro=1.0
auc-macro=1.0
auc-weighted=1.0
Mar 08 08:35:28completedprep-data-prep_data
label_column=label
num_rows=150
" + ], + "text/plain": [ + "" ] }, "metadata": {}, diff --git a/quick-start/workflow.py b/quick-start/workflow.py new file mode 100644 index 0000000..d850f48 --- /dev/null +++ b/quick-start/workflow.py @@ -0,0 +1,39 @@ +from kfp import dsl +import mlrun + +DATASET = 'cleaned_data' + +# Create a Kubeflow Pipelines pipeline +@dsl.pipeline( + name="Getting-started-tutorial", + description="Demonstrate some of the main capabilities of MLRun" +) +def kfpipeline(source_url, model_name="iris"): + project = mlrun.get_current_project() + + # Ingest the data set + ingest = mlrun.run_function( + 'prep-data', + handler='prep_data', + inputs={'source_url': source_url}, + params={'label_column': project.get_param("label_column")}, + outputs=[DATASET]) + + # Train a model + train = mlrun.run_function( + "train", + params={"label_column": project.get_param("label_column")}, + inputs={"dataset": ingest.outputs[DATASET]}, + outputs=['model', 'test_set']) + + # Test and visualize the model + test = mlrun.run_function( + "hub://test_classifier", + params={"label_column": project.get_param("label_column")}, + inputs={"models_path": train.outputs['model'], + "test_set": train.outputs['test_set']}) + + # Deploy the model as a serverless function + serving = mlrun.import_function("hub://v2_model_server", new_name="serving") + deploy = mlrun.deploy_function(serving, + models=[{"key":model_name, "model_path": train.outputs['model']}]) \ No newline at end of file