pipeline_stock.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sagemaker in /opt/conda/lib/python3.7/site-packages (2.100.0)\n",
      "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.5.0)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: google-pasta in /opt/conda/lib/python3.7/site-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: boto3<2.0,>=1.20.21 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.24.12)\n",
      "Requirement already satisfied: protobuf<4.0,>=3.1 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (3.20.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (20.1)\n",
      "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (0.1.5)\n",
      "Requirement already satisfied: attrs<22,>=20.3.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (20.3.0)\n",
      "Requirement already satisfied: pathos in /opt/conda/lib/python3.7/site-packages (from sagemaker) (0.2.9)\n",
      "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: numpy<2.0,>=1.9.0 in /opt/conda/lib/python3.7/site-packages (from sagemaker) (1.21.6)\n",
      "Requirement already satisfied: botocore<1.28.0,>=1.27.12 in /opt/conda/lib/python3.7/site-packages (from boto3<2.0,>=1.20.21->sagemaker) (1.27.12)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.7/site-packages (from boto3<2.0,>=1.20.21->sagemaker) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from boto3<2.0,>=1.20.21->sagemaker) (0.6.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (2.2.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from packaging>=20.0->sagemaker) (1.14.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=20.0->sagemaker) (2.4.6)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /opt/conda/lib/python3.7/site-packages (from pandas->sagemaker) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas->sagemaker) (2019.3)\n",
      "Requirement already satisfied: multiprocess>=0.70.13 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (0.70.13)\n",
      "Requirement already satisfied: dill>=0.3.5.1 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (0.3.5.1)\n",
      "Requirement already satisfied: ppft>=1.7.6.5 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (1.7.6.5)\n",
      "Requirement already satisfied: pox>=0.3.1 in /opt/conda/lib/python3.7/site-packages (from pathos->sagemaker) (0.3.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/conda/lib/python3.7/site-packages (from botocore<1.28.0,>=1.27.12->boto3<2.0,>=1.20.21->sagemaker) (1.26.9)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m22.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install --upgrade sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "import sagemaker.session\n",
    "\n",
    "\n",
    "region = boto3.Session().region_name\n",
    "sagemaker_session = sagemaker.session.Session()\n",
    "role = sagemaker.get_execution_role()\n",
    "default_bucket = \"ds-stock\"\n",
    "model_package_group_name = f\"Stock\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_data_uri = f\"s3://{default_bucket}/dataset/dataset.csv\"\n",
    "inference_data_uri = f\"s3://{default_bucket}/dataset/inference.csv\"\n",
    "inference_data_output_uri = f\"s3://{default_bucket}/stock_out\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.workflow.parameters import (\n",
    "    ParameterInteger,\n",
    "    ParameterString,\n",
    ")\n",
    "\n",
    "\n",
    "processing_instance_count = ParameterInteger(\n",
    "    name=\"ProcessingInstanceCount\",\n",
    "    default_value=1\n",
    ")\n",
    "processing_instance_type = ParameterString(\n",
    "    name=\"ProcessingInstanceType\",\n",
    "    default_value=\"ml.m5.xlarge\"\n",
    ")\n",
    "training_instance_type = ParameterString(\n",
    "    name=\"TrainingInstanceType\",\n",
    "    default_value=\"ml.m5.xlarge\"\n",
    ")\n",
    "model_approval_status = ParameterString(\n",
    "    name=\"ModelApprovalStatus\",\n",
    "    default_value=\"PendingManualApproval\"\n",
    ")\n",
    "input_data = ParameterString(\n",
    "    name=\"InputData\",\n",
    "    default_value=input_data_uri,\n",
    ")\n",
    "inference_data = ParameterString(\n",
    "    name=\"InferenceData\",\n",
    "    default_value=inference_data_uri,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.workflow.steps import CacheConfig\n",
    "cache_config = CacheConfig(enable_caching=True, expire_after=\"PT3D\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing code/preprocessing.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile code/preprocessing.py\n",
    "import argparse\n",
    "import os\n",
    "import requests\n",
    "import tempfile\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    base_dir = \"/opt/ml/processing\"\n",
    "\n",
    "    df = pd.read_csv(\n",
    "        f\"{base_dir}/input/dataset.csv\",\n",
    "        index_col=0,\n",
    "    )\n",
    "    if 'y' in df:\n",
    "        df = df.drop('y', axis=1)\n",
    "        \n",
    "    df['date'] = [i.split('$')[0] for i in df.index]\n",
    "    dates = sorted(set(df['date']))\n",
    "    \n",
    "    train_dates, validation_dates, test_dates = np.split(dates, [len(dates) - 2 * 7, len(dates) - 7])\n",
    "    \n",
    "    train = df[df[\"date\"].isin(train_dates)].drop('date', axis=1)\n",
    "    validation = df[df[\"date\"].isin(validation_dates)].drop('date', axis=1)\n",
    "    test = df[df[\"date\"].isin(test_dates)].drop('date', axis=1)\n",
    "\n",
    "    \n",
    "    train.to_csv(f\"{base_dir}/train/train.csv\", header=False, index=False)\n",
    "    validation.to_csv(f\"{base_dir}/validation/validation.csv\", header=False, index=False)\n",
    "    test.to_csv(f\"{base_dir}/test/test.csv\", header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:instance_type should not be a pipeline variable (<class 'sagemaker.workflow.parameters.ParameterString'>). The default_value of this Parameter object will be used to override it. Please remove this pipeline variable and use python primitives instead.\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.sklearn.processing import SKLearnProcessor\n",
    "\n",
    "\n",
    "framework_version = \"0.23-1\"\n",
    "\n",
    "sklearn_processor = SKLearnProcessor(\n",
    "    framework_version=framework_version,\n",
    "    instance_type=processing_instance_type,\n",
    "    instance_count=processing_instance_count,\n",
    "    base_job_name=\"sklearn-stock-process\",\n",
    "    role=role\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.processing import ProcessingInput, ProcessingOutput\n",
    "from sagemaker.workflow.steps import ProcessingStep\n",
    "    \n",
    "\n",
    "step_process = ProcessingStep(\n",
    "    name=\"StockProcess\",\n",
    "    processor=sklearn_processor,\n",
    "    inputs=[\n",
    "      ProcessingInput(source=input_data, destination=\"/opt/ml/processing/input\"),  \n",
    "    ],\n",
    "    outputs=[\n",
    "        ProcessingOutput(output_name=\"train\", source=\"/opt/ml/processing/train\"),\n",
    "        ProcessingOutput(output_name=\"validation\", source=\"/opt/ml/processing/validation\"),\n",
    "        ProcessingOutput(output_name=\"test\", source=\"/opt/ml/processing/test\")\n",
    "    ],\n",
    "    code=\"code/preprocessing.py\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = f\"s3://{default_bucket}/StockTrain\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:instance_type should not be a pipeline variable (<class 'sagemaker.workflow.parameters.ParameterString'>). The default_value of this Parameter object will be used to override it. Please remove this pipeline variable and use python primitives instead.\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "\n",
    "image_uri = sagemaker.image_uris.retrieve(\n",
    "    framework=\"xgboost\",\n",
    "    region=region,\n",
    "    version=\"1.0-1\",\n",
    "    py_version=\"py3\",\n",
    "    instance_type=training_instance_type,\n",
    ")\n",
    "xgb_train = Estimator(\n",
    "    image_uri=image_uri,\n",
    "    instance_type=training_instance_type,\n",
    "    instance_count=1,\n",
    "    output_path=model_path,\n",
    "    role=role,\n",
    ")\n",
    "xgb_train.set_hyperparameters(\n",
    "    objective=\"binary:logistic\",\n",
    "    num_round=500,\n",
    "    max_depth=5,\n",
    "    eta=0.2,\n",
    "    gamma=4,\n",
    "    min_child_weight=6,\n",
    "    subsample=0.7,\n",
    "    silent=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.inputs import TrainingInput\n",
    "from sagemaker.workflow.steps import TrainingStep\n",
    "\n",
    "\n",
    "step_train = TrainingStep(\n",
    "    name=\"StockTrain\",\n",
    "    estimator=xgb_train,\n",
    "    inputs={\n",
    "        \"train\": TrainingInput(\n",
    "            s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\n",
    "                \"train\"\n",
    "            ].S3Output.S3Uri,\n",
    "            content_type=\"text/csv\"\n",
    "        ),\n",
    "        \"validation\": TrainingInput(\n",
    "            s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\n",
    "                \"test\"\n",
    "            ].S3Output.S3Uri,\n",
    "            content_type=\"text/csv\"\n",
    "        )\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting code/evaluation.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile code/evaluation.py\n",
    "import json\n",
    "import pathlib\n",
    "import pickle\n",
    "import tarfile\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost\n",
    "\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    model_path = f\"/opt/ml/processing/model/model.tar.gz\"\n",
    "    with tarfile.open(model_path) as tar:\n",
    "        tar.extractall(path=\".\")\n",
    "\n",
    "    model = pickle.load(open(\"xgboost-model\", \"rb\"))\n",
    "\n",
    "    test_path = \"/opt/ml/processing/test/test.csv\"\n",
    "    df = pd.read_csv(test_path, header=None)\n",
    "\n",
    "    y_test = df.iloc[:, 0].to_numpy()\n",
    "    df.drop(df.columns[0], axis=1, inplace=True)\n",
    "\n",
    "    X_test = xgboost.DMatrix(df.values)\n",
    "\n",
    "    predictions = model.predict(X_test)\n",
    "    y_pred = 1*(predictions>0.5)\n",
    "\n",
    "    report_dict = classification_report(y_test, y_pred, labels=[1], output_dict=True)['1']\n",
    "\n",
    "    output_dir = \"/opt/ml/processing/evaluation\"\n",
    "    pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "    evaluation_path = f\"{output_dir}/evaluation.json\"\n",
    "    with open(evaluation_path, \"w\") as f:\n",
    "        f.write(json.dumps(report_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.processing import ScriptProcessor\n",
    "\n",
    "\n",
    "script_eval = ScriptProcessor(\n",
    "    image_uri=image_uri,\n",
    "    command=[\"python3\"],\n",
    "    instance_type=processing_instance_type,\n",
    "    instance_count=1,\n",
    "    base_job_name=\"script-stock-eval\",\n",
    "    role=role,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "step_eval = ProcessingStep(\n",
    "    name=\"StockEval\",\n",
    "    processor=script_eval,\n",
    "    inputs=[\n",
    "        ProcessingInput(\n",
    "            source=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n",
    "            destination=\"/opt/ml/processing/model\"\n",
    "        ),\n",
    "        ProcessingInput(\n",
    "            source=step_process.properties.ProcessingOutputConfig.Outputs[\n",
    "                \"test\"\n",
    "            ].S3Output.S3Uri,\n",
    "            destination=\"/opt/ml/processing/test\"\n",
    "        )\n",
    "    ],\n",
    "    outputs=[\n",
    "        ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\"),\n",
    "    ],\n",
    "    code=\"code/evaluation.py\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting code/inference.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile code/inference.py\n",
    "import json\n",
    "import pathlib\n",
    "import pickle\n",
    "import tarfile\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost\n",
    "import os\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    model_path = f\"/opt/ml/processing/model/model.tar.gz\"\n",
    "    with tarfile.open(model_path) as tar:\n",
    "        tar.extractall(path=\".\")\n",
    "    \n",
    "    model = pickle.load(open(\"xgboost-model\", \"rb\"))\n",
    "\n",
    "    test_path = \"/opt/ml/processing/input/inference.csv\"\n",
    "    \n",
    "    df = pd.read_csv(test_path, index_col=0)\n",
    "    \n",
    "    X_test = xgboost.DMatrix(df.values)\n",
    "    predictions = model.predict(X_test)\n",
    "    \n",
    "    df.insert(0, \"prediction\", predictions)\n",
    "    \n",
    "    output_dir = \"/opt/ml/processing/output\"\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    \n",
    "    output_path = f\"{output_dir}/inference.csv\"\n",
    "    df.to_csv(output_path, index=True)\n",
    "    print(\"save ......\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.processing import ScriptProcessor\n",
    "\n",
    "\n",
    "script_inference = ScriptProcessor(\n",
    "    image_uri=image_uri,\n",
    "    command=[\"python3\"],\n",
    "    instance_type=processing_instance_type,\n",
    "    instance_count=1,\n",
    "    base_job_name=\"script-stock-inference\",\n",
    "    role=role,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "step_inference = ProcessingStep(\n",
    "    name=\"StockInference\",\n",
    "    processor=script_eval,\n",
    "    inputs=[\n",
    "        ProcessingInput(\n",
    "            source=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n",
    "            destination=\"/opt/ml/processing/model\"\n",
    "        ),\n",
    "        ProcessingInput(\n",
    "            source=inference_data, \n",
    "            destination=\"/opt/ml/processing/input\"\n",
    "        )\n",
    "    ],\n",
    "    outputs=[\n",
    "        ProcessingOutput(source=\"/opt/ml/processing/output\", destination=inference_data_output_uri),\n",
    "    ],\n",
    "    code=\"code/inference.py\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.workflow.pipeline import Pipeline\n",
    "\n",
    "\n",
    "pipeline_name = f\"StockPipeline\"\n",
    "pipeline = Pipeline(\n",
    "    name=pipeline_name,\n",
    "    parameters=[\n",
    "        processing_instance_type, \n",
    "        processing_instance_count,\n",
    "        training_instance_type,\n",
    "        model_approval_status,\n",
    "        input_data,\n",
    "        inference_data,\n",
    "    ],\n",
    "    steps=[step_process, step_train, step_eval, step_inference],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Version': '2020-12-01',\n",
       " 'Metadata': {},\n",
       " 'Parameters': [{'Name': 'ProcessingInstanceType',\n",
       "   'Type': 'String',\n",
       "   'DefaultValue': 'ml.m5.xlarge'},\n",
       "  {'Name': 'ProcessingInstanceCount', 'Type': 'Integer', 'DefaultValue': 1},\n",
       "  {'Name': 'TrainingInstanceType',\n",
       "   'Type': 'String',\n",
       "   'DefaultValue': 'ml.m5.xlarge'},\n",
       "  {'Name': 'ModelApprovalStatus',\n",
       "   'Type': 'String',\n",
       "   'DefaultValue': 'PendingManualApproval'},\n",
       "  {'Name': 'InputData',\n",
       "   'Type': 'String',\n",
       "   'DefaultValue': 's3://ds-stock/dataset/dataset.csv'},\n",
       "  {'Name': 'InferenceData',\n",
       "   'Type': 'String',\n",
       "   'DefaultValue': 's3://ds-stock/dataset/inference.csv'}],\n",
       " 'PipelineExperimentConfig': {'ExperimentName': {'Get': 'Execution.PipelineName'},\n",
       "  'TrialName': {'Get': 'Execution.PipelineExecutionId'}},\n",
       " 'Steps': [{'Name': 'StockProcess',\n",
       "   'Type': 'Processing',\n",
       "   'Arguments': {'ProcessingResources': {'ClusterConfig': {'InstanceType': {'Get': 'Parameters.ProcessingInstanceType'},\n",
       "      'InstanceCount': {'Get': 'Parameters.ProcessingInstanceCount'},\n",
       "      'VolumeSizeInGB': 30}},\n",
       "    'AppSpecification': {'ImageUri': '246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:0.23-1-cpu-py3',\n",
       "     'ContainerEntrypoint': ['python3',\n",
       "      '/opt/ml/processing/input/code/preprocessing.py']},\n",
       "    'RoleArn': 'arn:aws:iam::511640773671:role/service-role/AmazonSageMaker-ExecutionRole-20220726T174318',\n",
       "    'ProcessingInputs': [{'InputName': 'input-1',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': {'Get': 'Parameters.InputData'},\n",
       "       'LocalPath': '/opt/ml/processing/input',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}},\n",
       "     {'InputName': 'code',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': 's3://sagemaker-us-west-2-511640773671/StockProcess-bb34265535517c9c573822207904a776/input/code/preprocessing.py',\n",
       "       'LocalPath': '/opt/ml/processing/input/code',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}}],\n",
       "    'ProcessingOutputConfig': {'Outputs': [{'OutputName': 'train',\n",
       "       'AppManaged': False,\n",
       "       'S3Output': {'S3Uri': 's3://sagemaker-us-west-2-511640773671/StockProcess-bb34265535517c9c573822207904a776/output/train',\n",
       "        'LocalPath': '/opt/ml/processing/train',\n",
       "        'S3UploadMode': 'EndOfJob'}},\n",
       "      {'OutputName': 'validation',\n",
       "       'AppManaged': False,\n",
       "       'S3Output': {'S3Uri': 's3://sagemaker-us-west-2-511640773671/StockProcess-bb34265535517c9c573822207904a776/output/validation',\n",
       "        'LocalPath': '/opt/ml/processing/validation',\n",
       "        'S3UploadMode': 'EndOfJob'}},\n",
       "      {'OutputName': 'test',\n",
       "       'AppManaged': False,\n",
       "       'S3Output': {'S3Uri': 's3://sagemaker-us-west-2-511640773671/StockProcess-bb34265535517c9c573822207904a776/output/test',\n",
       "        'LocalPath': '/opt/ml/processing/test',\n",
       "        'S3UploadMode': 'EndOfJob'}}]}}},\n",
       "  {'Name': 'StockTrain',\n",
       "   'Type': 'Training',\n",
       "   'Arguments': {'AlgorithmSpecification': {'TrainingInputMode': 'File',\n",
       "     'TrainingImage': '246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-xgboost:1.0-1-cpu-py3'},\n",
       "    'OutputDataConfig': {'S3OutputPath': 's3://ds-stock/StockTrain'},\n",
       "    'StoppingCondition': {'MaxRuntimeInSeconds': 86400},\n",
       "    'ResourceConfig': {'VolumeSizeInGB': 30,\n",
       "     'InstanceCount': 1,\n",
       "     'InstanceType': {'Get': 'Parameters.TrainingInstanceType'}},\n",
       "    'RoleArn': 'arn:aws:iam::511640773671:role/service-role/AmazonSageMaker-ExecutionRole-20220726T174318',\n",
       "    'InputDataConfig': [{'DataSource': {'S3DataSource': {'S3DataType': 'S3Prefix',\n",
       "        'S3Uri': {'Get': \"Steps.StockProcess.ProcessingOutputConfig.Outputs['train'].S3Output.S3Uri\"},\n",
       "        'S3DataDistributionType': 'FullyReplicated'}},\n",
       "      'ContentType': 'text/csv',\n",
       "      'ChannelName': 'train'},\n",
       "     {'DataSource': {'S3DataSource': {'S3DataType': 'S3Prefix',\n",
       "        'S3Uri': {'Get': \"Steps.StockProcess.ProcessingOutputConfig.Outputs['test'].S3Output.S3Uri\"},\n",
       "        'S3DataDistributionType': 'FullyReplicated'}},\n",
       "      'ContentType': 'text/csv',\n",
       "      'ChannelName': 'validation'}],\n",
       "    'HyperParameters': {'objective': 'binary:logistic',\n",
       "     'num_round': '500',\n",
       "     'max_depth': '5',\n",
       "     'eta': '0.2',\n",
       "     'gamma': '4',\n",
       "     'min_child_weight': '6',\n",
       "     'subsample': '0.7',\n",
       "     'silent': '0'},\n",
       "    'ProfilerRuleConfigurations': [{'RuleConfigurationName': 'ProfilerReport-1658845968',\n",
       "      'RuleEvaluatorImage': '895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest',\n",
       "      'RuleParameters': {'rule_to_invoke': 'ProfilerReport'}}],\n",
       "    'ProfilerConfig': {'S3OutputPath': 's3://ds-stock/StockTrain'}}},\n",
       "  {'Name': 'StockEval',\n",
       "   'Type': 'Processing',\n",
       "   'Arguments': {'ProcessingResources': {'ClusterConfig': {'InstanceType': {'Get': 'Parameters.ProcessingInstanceType'},\n",
       "      'InstanceCount': 1,\n",
       "      'VolumeSizeInGB': 30}},\n",
       "    'AppSpecification': {'ImageUri': '246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-xgboost:1.0-1-cpu-py3',\n",
       "     'ContainerEntrypoint': ['python3',\n",
       "      '/opt/ml/processing/input/code/evaluation.py']},\n",
       "    'RoleArn': 'arn:aws:iam::511640773671:role/service-role/AmazonSageMaker-ExecutionRole-20220726T174318',\n",
       "    'ProcessingInputs': [{'InputName': 'input-1',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': {'Get': 'Steps.StockTrain.ModelArtifacts.S3ModelArtifacts'},\n",
       "       'LocalPath': '/opt/ml/processing/model',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}},\n",
       "     {'InputName': 'input-2',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': {'Get': \"Steps.StockProcess.ProcessingOutputConfig.Outputs['test'].S3Output.S3Uri\"},\n",
       "       'LocalPath': '/opt/ml/processing/test',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}},\n",
       "     {'InputName': 'code',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': 's3://sagemaker-us-west-2-511640773671/StockEval-bf80570b6a911e151eb5287242d428cb/input/code/evaluation.py',\n",
       "       'LocalPath': '/opt/ml/processing/input/code',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}}],\n",
       "    'ProcessingOutputConfig': {'Outputs': [{'OutputName': 'evaluation',\n",
       "       'AppManaged': False,\n",
       "       'S3Output': {'S3Uri': 's3://sagemaker-us-west-2-511640773671/StockEval-bf80570b6a911e151eb5287242d428cb/output/evaluation',\n",
       "        'LocalPath': '/opt/ml/processing/evaluation',\n",
       "        'S3UploadMode': 'EndOfJob'}}]}}},\n",
       "  {'Name': 'StockInference',\n",
       "   'Type': 'Processing',\n",
       "   'Arguments': {'ProcessingResources': {'ClusterConfig': {'InstanceType': {'Get': 'Parameters.ProcessingInstanceType'},\n",
       "      'InstanceCount': 1,\n",
       "      'VolumeSizeInGB': 30}},\n",
       "    'AppSpecification': {'ImageUri': '246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-xgboost:1.0-1-cpu-py3',\n",
       "     'ContainerEntrypoint': ['python3',\n",
       "      '/opt/ml/processing/input/code/inference.py']},\n",
       "    'RoleArn': 'arn:aws:iam::511640773671:role/service-role/AmazonSageMaker-ExecutionRole-20220726T174318',\n",
       "    'ProcessingInputs': [{'InputName': 'input-1',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': {'Get': 'Steps.StockTrain.ModelArtifacts.S3ModelArtifacts'},\n",
       "       'LocalPath': '/opt/ml/processing/model',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}},\n",
       "     {'InputName': 'input-2',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': {'Get': 'Parameters.InferenceData'},\n",
       "       'LocalPath': '/opt/ml/processing/input',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}},\n",
       "     {'InputName': 'code',\n",
       "      'AppManaged': False,\n",
       "      'S3Input': {'S3Uri': 's3://sagemaker-us-west-2-511640773671/StockInference-a0975e90eb5ac3df56d55d632bfecb98/input/code/inference.py',\n",
       "       'LocalPath': '/opt/ml/processing/input/code',\n",
       "       'S3DataType': 'S3Prefix',\n",
       "       'S3InputMode': 'File',\n",
       "       'S3DataDistributionType': 'FullyReplicated',\n",
       "       'S3CompressionType': 'None'}}],\n",
       "    'ProcessingOutputConfig': {'Outputs': [{'OutputName': 'output-1',\n",
       "       'AppManaged': False,\n",
       "       'S3Output': {'S3Uri': 's3://ds-stock/stock_out',\n",
       "        'LocalPath': '/opt/ml/processing/output',\n",
       "        'S3UploadMode': 'EndOfJob'}}]}}}]}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "json.loads(pipeline.definition())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PipelineArn': 'arn:aws:sagemaker:us-west-2:511640773671:pipeline/stockpipeline',\n",
       " 'ResponseMetadata': {'RequestId': 'b4a150fb-8968-495f-82da-6b1ed0cb68f0',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': 'b4a150fb-8968-495f-82da-6b1ed0cb68f0',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '81',\n",
       "   'date': 'Tue, 26 Jul 2022 14:32:50 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.upsert(role_arn=role)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-2:236514542706:image/datascience-1.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}