tutorials/profiles.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Profiles Basic Tutorial\n",
    "\n",
    "__Updated June 2024 to use a new dataset__\n",
    "\n",
    "This Jupyter notebook will give a basic tutorial on how to use PyDeequ's Profiles module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "# indicate your Spark version, here we use Spark 3.5 with pydeequ 1.4.0\n",
    "os.environ[\"SPARK_VERSION\"] = '3.5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ":: loading settings :: url = jar:file:/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/pyspark/jars/ivy-2.5.0.jar!/org/apache/ivy/core/settings/ivysettings.xml\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Ivy Default Cache set to: /home/ec2-user/.ivy2/cache\n",
      "The jars for the packages stored in: /home/ec2-user/.ivy2/jars\n",
      "com.amazon.deequ#deequ added as a dependency\n",
      ":: resolving dependencies :: org.apache.spark#spark-submit-parent-37bbf6aa-f0ee-4f60-8485-4571bfefeb8f;1.0\n",
      "\tconfs: [default]\n",
      "\tfound com.amazon.deequ#deequ;2.0.3-spark-3.3 in central\n",
      "\tfound org.scala-lang#scala-reflect;2.12.10 in central\n",
      "\tfound org.scalanlp#breeze_2.12;0.13.2 in central\n",
      "\tfound org.scalanlp#breeze-macros_2.12;0.13.2 in central\n",
      "\tfound com.github.fommil.netlib#core;1.1.2 in central\n",
      "\tfound net.sf.opencsv#opencsv;2.3 in central\n",
      "\tfound com.github.rwl#jtransforms;2.4.0 in central\n",
      "\tfound junit#junit;4.8.2 in central\n",
      "\tfound org.apache.commons#commons-math3;3.2 in central\n",
      "\tfound org.spire-math#spire_2.12;0.13.0 in central\n",
      "\tfound org.spire-math#spire-macros_2.12;0.13.0 in central\n",
      "\tfound org.typelevel#machinist_2.12;0.6.1 in central\n",
      "\tfound com.chuusai#shapeless_2.12;2.3.2 in central\n",
      "\tfound org.typelevel#macro-compat_2.12;1.1.1 in central\n",
      "\tfound org.slf4j#slf4j-api;1.7.5 in central\n",
      ":: resolution report :: resolve 385ms :: artifacts dl 12ms\n",
      "\t:: modules in use:\n",
      "\tcom.amazon.deequ#deequ;2.0.3-spark-3.3 from central in [default]\n",
      "\tcom.chuusai#shapeless_2.12;2.3.2 from central in [default]\n",
      "\tcom.github.fommil.netlib#core;1.1.2 from central in [default]\n",
      "\tcom.github.rwl#jtransforms;2.4.0 from central in [default]\n",
      "\tjunit#junit;4.8.2 from central in [default]\n",
      "\tnet.sf.opencsv#opencsv;2.3 from central in [default]\n",
      "\torg.apache.commons#commons-math3;3.2 from central in [default]\n",
      "\torg.scala-lang#scala-reflect;2.12.10 from central in [default]\n",
      "\torg.scalanlp#breeze-macros_2.12;0.13.2 from central in [default]\n",
      "\torg.scalanlp#breeze_2.12;0.13.2 from central in [default]\n",
      "\torg.slf4j#slf4j-api;1.7.5 from central in [default]\n",
      "\torg.spire-math#spire-macros_2.12;0.13.0 from central in [default]\n",
      "\torg.spire-math#spire_2.12;0.13.0 from central in [default]\n",
      "\torg.typelevel#machinist_2.12;0.6.1 from central in [default]\n",
      "\torg.typelevel#macro-compat_2.12;1.1.1 from central in [default]\n",
      "\t:: evicted modules:\n",
      "\torg.scala-lang#scala-reflect;2.12.1 by [org.scala-lang#scala-reflect;2.12.10] in [default]\n",
      "\torg.scala-lang#scala-reflect;2.12.0 by [org.scala-lang#scala-reflect;2.12.10] in [default]\n",
      "\t---------------------------------------------------------------------\n",
      "\t|                  |            modules            ||   artifacts   |\n",
      "\t|       conf       | number| search|dwnlded|evicted|| number|dwnlded|\n",
      "\t---------------------------------------------------------------------\n",
      "\t|      default     |   17  |   0   |   0   |   2   ||   15  |   0   |\n",
      "\t---------------------------------------------------------------------\n",
      ":: retrieving :: org.apache.spark#spark-submit-parent-37bbf6aa-f0ee-4f60-8485-4571bfefeb8f\n",
      "\tconfs: [default]\n",
      "\t0 artifacts copied, 15 already retrieved (0kB/10ms)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/06/14 23:33:34 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/06/14 23:33:36 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n",
      "24/06/14 23:33:36 WARN Utils: Service 'SparkUI' could not bind on port 4041. Attempting port 4042.\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession, Row, DataFrame\n",
    "import json\n",
    "import pandas as pd\n",
    "import sagemaker_pyspark\n",
    "\n",
    "import pydeequ\n",
    "\n",
    "classpath = \":\".join(sagemaker_pyspark.classpath_jars())\n",
    "\n",
    "spark = (SparkSession\n",
    "    .builder\n",
    "    .config(\"spark.driver.extraClassPath\", classpath)\n",
    "    .config(\"spark.jars.packages\", pydeequ.deequ_maven_coord)\n",
    "    .config(\"spark.jars.excludes\", pydeequ.f2j_maven_coord)\n",
    "    .config(\"spark.driver.memory\", \"15g\")\n",
    "    .config(\"spark.sql.parquet.int96RebaseModeInRead\", \"CORRECTED\")\n",
    "    .getOrCreate())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We will be using the synthetic reviews dataset for Electronics products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/06/14 23:33:38 WARN MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-s3a-file-system.properties,hadoop-metrics2.properties\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- marketplace: string (nullable = true)\n",
      " |-- customer_id: string (nullable = true)\n",
      " |-- review_id: string (nullable = true)\n",
      " |-- product_id: string (nullable = true)\n",
      " |-- product_title: string (nullable = true)\n",
      " |-- star_rating: long (nullable = true)\n",
      " |-- helpful_votes: long (nullable = true)\n",
      " |-- total_votes: long (nullable = true)\n",
      " |-- insight: string (nullable = true)\n",
      " |-- review_headline: string (nullable = true)\n",
      " |-- review_body: string (nullable = true)\n",
      " |-- review_date: timestamp (nullable = true)\n",
      " |-- review_year: long (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.read.parquet(\"s3a://aws-bigdata-blog/generated_synthetic_reviews/data/product_category=Electronics/\")\n",
    "\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/06/14 23:33:43 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "from pydeequ.profiles import *\n",
    "\n",
    "result = ColumnProfilerRunner(spark) \\\n",
    "    .onData(df) \\\n",
    "    .run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StandardProfiles for column: insight: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 2,\n",
      "    \"dataType\": \"String\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 0,\n",
      "        \"Unknown\": 0,\n",
      "        \"String\": 3010972\n",
      "    },\n",
      "    \"histogram\": [\n",
      "        [\n",
      "            \"N\",\n",
      "            1701683,\n",
      "            0.5651606856523408\n",
      "        ],\n",
      "        [\n",
      "            \"Y\",\n",
      "            1309289,\n",
      "            0.43483931434765916\n",
      "        ]\n",
      "    ]\n",
      "}\n",
      "StandardProfiles for column: review_id: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 3160409,\n",
      "    \"dataType\": \"String\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 0,\n",
      "        \"Unknown\": 0,\n",
      "        \"String\": 3010972\n",
      "    },\n",
      "    \"histogram\": null\n",
      "}\n",
      "NumericProfiles for column: customer_id: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 866021,\n",
      "    \"dataType\": \"Integral\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 3010972,\n",
      "        \"Unknown\": 0,\n",
      "        \"String\": 0\n",
      "    },\n",
      "    \"histogram\": null,\n",
      "    \"kll\": \"None\",\n",
      "    \"mean\": 514989.4746510429,\n",
      "    \"maximum\": 929121.0,\n",
      "    \"minimum\": 100000.0,\n",
      "    \"sum\": 1550618888469.0,\n",
      "    \"stdDev\": 239465.84713597817,\n",
      "    \"approxPercentiles\": []\n",
      "}\n",
      "StandardProfiles for column: review_date: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 7916,\n",
      "    \"dataType\": \"String\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {},\n",
      "    \"histogram\": null\n",
      "}\n",
      "NumericProfiles for column: helpful_votes: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 24,\n",
      "    \"dataType\": \"Integral\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {},\n",
      "    \"histogram\": [\n",
      "        [\n",
      "            \"12\",\n",
      "            287924,\n",
      "            0.09562493440656372\n",
      "        ],\n",
      "        [\n",
      "            \"8\",\n",
      "            8337,\n",
      "            0.0027688733073572254\n",
      "        ],\n",
      "        [\n",
      "            \"19\",\n",
      "            72602,\n",
      "            0.02411247929240126\n",
      "        ],\n",
      "        [\n",
      "            \"23\",\n",
      "            434,\n",
      "            0.00014413950046695883\n",
      "        ],\n",
      "        [\n",
      "            \"4\",\n",
      "            13,\n",
      "            4.317542640715357e-06\n",
      "        ],\n",
      "        [\n",
      "            \"15\",\n",
      "            520006,\n",
      "            0.1727036983406023\n",
      "        ],\n",
      "        [\n",
      "            \"11\",\n",
      "            159423,\n",
      "            0.052947353877751104\n",
      "        ],\n",
      "        [\n",
      "            \"9\",\n",
      "            27286,\n",
      "            0.009062189884196863\n",
      "        ],\n",
      "        [\n",
      "            \"22\",\n",
      "            2218,\n",
      "            0.000736639198238974\n",
      "        ],\n",
      "        [\n",
      "            \"26\",\n",
      "            2,\n",
      "            6.642373293408242e-07\n",
      "        ],\n",
      "        [\n",
      "            \"13\",\n",
      "            427526,\n",
      "            0.1419893642318826\n",
      "        ],\n",
      "        [\n",
      "            \"24\",\n",
      "            70,\n",
      "            2.3248306526928844e-05\n",
      "        ],\n",
      "        [\n",
      "            \"16\",\n",
      "            426940,\n",
      "            0.14179474269438574\n",
      "        ],\n",
      "        [\n",
      "            \"5\",\n",
      "            87,\n",
      "            2.889432382632585e-05\n",
      "        ],\n",
      "        [\n",
      "            \"10\",\n",
      "            72476,\n",
      "            0.024070632340652785\n",
      "        ],\n",
      "        [\n",
      "            \"21\",\n",
      "            8287,\n",
      "            0.0027522673741237048\n",
      "        ],\n",
      "        [\n",
      "            \"6\",\n",
      "            450,\n",
      "            0.00014945339910168542\n",
      "        ],\n",
      "        [\n",
      "            \"17\",\n",
      "            288496,\n",
      "            0.0958149062827552\n",
      "        ],\n",
      "        [\n",
      "            \"25\",\n",
      "            12,\n",
      "            3.985423976044945e-06\n",
      "        ],\n",
      "        [\n",
      "            \"14\",\n",
      "            519410,\n",
      "            0.17250575561645873\n",
      "        ],\n",
      "        [\n",
      "            \"20\",\n",
      "            27100,\n",
      "            0.009000415812568167\n",
      "        ],\n",
      "        [\n",
      "            \"18\",\n",
      "            159692,\n",
      "            0.053036693798547446\n",
      "        ],\n",
      "        [\n",
      "            \"7\",\n",
      "            2180,\n",
      "            0.0007240186889814984\n",
      "        ],\n",
      "        [\n",
      "            \"3\",\n",
      "            1,\n",
      "            3.321186646704121e-07\n",
      "        ]\n",
      "    ],\n",
      "    \"kll\": \"None\",\n",
      "    \"mean\": 14.500331454427341,\n",
      "    \"maximum\": 26.0,\n",
      "    \"minimum\": 3.0,\n",
      "    \"sum\": 43660092.0,\n",
      "    \"stdDev\": 2.2547987149482704,\n",
      "    \"approxPercentiles\": []\n",
      "}\n",
      "NumericProfiles for column: star_rating: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 5,\n",
      "    \"dataType\": \"Integral\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {},\n",
      "    \"histogram\": [\n",
      "        [\n",
      "            \"4\",\n",
      "            1053843,\n",
      "            0.35000092993226106\n",
      "        ],\n",
      "        [\n",
      "            \"5\",\n",
      "            1204384,\n",
      "            0.3999984058304096\n",
      "        ],\n",
      "        [\n",
      "            \"1\",\n",
      "            150549,\n",
      "            0.05000013284746587\n",
      "        ],\n",
      "        [\n",
      "            \"2\",\n",
      "            150549,\n",
      "            0.05000013284746587\n",
      "        ],\n",
      "        [\n",
      "            \"3\",\n",
      "            451647,\n",
      "            0.1500003985423976\n",
      "        ]\n",
      "    ],\n",
      "    \"kll\": \"None\",\n",
      "    \"mean\": 3.9999973430506826,\n",
      "    \"maximum\": 5.0,\n",
      "    \"minimum\": 1.0,\n",
      "    \"sum\": 12043880.0,\n",
      "    \"stdDev\": 1.0954453575522467,\n",
      "    \"approxPercentiles\": []\n",
      "}\n",
      "StandardProfiles for column: product_title: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 5847,\n",
      "    \"dataType\": \"String\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 0,\n",
      "        \"Unknown\": 0,\n",
      "        \"String\": 3010972\n",
      "    },\n",
      "    \"histogram\": null\n",
      "}\n",
      "StandardProfiles for column: review_headline: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 3263448,\n",
      "    \"dataType\": \"String\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 0,\n",
      "        \"Unknown\": 0,\n",
      "        \"String\": 3010972\n",
      "    },\n",
      "    \"histogram\": null\n",
      "}\n",
      "NumericProfiles for column: review_year: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 25,\n",
      "    \"dataType\": \"Integral\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {},\n",
      "    \"histogram\": [\n",
      "        [\n",
      "            \"2014\",\n",
      "            321464,\n",
      "            0.10676419441960935\n",
      "        ],\n",
      "        [\n",
      "            \"2003\",\n",
      "            52307,\n",
      "            0.017372130992915246\n",
      "        ],\n",
      "        [\n",
      "            \"1997\",\n",
      "            10723,\n",
      "            0.0035613084412608287\n",
      "        ],\n",
      "        [\n",
      "            \"2101\",\n",
      "            2,\n",
      "            6.642373293408242e-07\n",
      "        ],\n",
      "        [\n",
      "            \"2013\",\n",
      "            316199,\n",
      "            0.10501558965011963\n",
      "        ],\n",
      "        [\n",
      "            \"2002\",\n",
      "            42388,\n",
      "            0.014077845958049427\n",
      "        ],\n",
      "        [\n",
      "            \"2202\",\n",
      "            1,\n",
      "            3.321186646704121e-07\n",
      "        ],\n",
      "        [\n",
      "            \"2007\",\n",
      "            147466,\n",
      "            0.048976211004286986\n",
      "        ],\n",
      "        [\n",
      "            \"2004\",\n",
      "            55873,\n",
      "            0.018556466151129936\n",
      "        ],\n",
      "        [\n",
      "            \"1996\",\n",
      "            9521,\n",
      "            0.0031621018063269935\n",
      "        ],\n",
      "        [\n",
      "            \"2015\",\n",
      "            325725,\n",
      "            0.10817935204976997\n",
      "        ],\n",
      "        [\n",
      "            \"2011\",\n",
      "            197133,\n",
      "            0.06547154872247235\n",
      "        ],\n",
      "        [\n",
      "            \"2001\",\n",
      "            38141,\n",
      "            0.012667337989194187\n",
      "        ],\n",
      "        [\n",
      "            \"2008\",\n",
      "            166847,\n",
      "            0.055413002844264245\n",
      "        ],\n",
      "        [\n",
      "            \"1696\",\n",
      "            2,\n",
      "            6.642373293408242e-07\n",
      "        ],\n",
      "        [\n",
      "            \"2012\",\n",
      "            234672,\n",
      "            0.07793895127553495\n",
      "        ],\n",
      "        [\n",
      "            \"1999\",\n",
      "            12318,\n",
      "            0.004091037711410136\n",
      "        ],\n",
      "        [\n",
      "            \"2005\",\n",
      "            122344,\n",
      "            0.040632725910436894\n",
      "        ],\n",
      "        [\n",
      "            \"2010\",\n",
      "            190961,\n",
      "            0.06342171232412656\n",
      "        ],\n",
      "        [\n",
      "            \"2009\",\n",
      "            169434,\n",
      "            0.0562721938297666\n",
      "        ],\n",
      "        [\n",
      "            \"2000\",\n",
      "            20756,\n",
      "            0.0068934550038990735\n",
      "        ],\n",
      "        [\n",
      "            \"2016\",\n",
      "            423293,\n",
      "            0.14058350592433275\n",
      "        ],\n",
      "        [\n",
      "            \"1998\",\n",
      "            11633,\n",
      "            0.003863536426110904\n",
      "        ],\n",
      "        [\n",
      "            \"2006\",\n",
      "            141769,\n",
      "            0.04708413097165965\n",
      "        ]\n",
      "    ],\n",
      "    \"kll\": \"None\",\n",
      "    \"mean\": 2010.9318020891592,\n",
      "    \"maximum\": 2202.0,\n",
      "    \"minimum\": 1696.0,\n",
      "    \"sum\": 6054859350.0,\n",
      "    \"stdDev\": 4.348285411927376,\n",
      "    \"approxPercentiles\": []\n",
      "}\n",
      "NumericProfiles for column: product_id: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 9375,\n",
      "    \"dataType\": \"Integral\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 3010972,\n",
      "        \"Unknown\": 0,\n",
      "        \"String\": 0\n",
      "    },\n",
      "    \"histogram\": null,\n",
      "    \"kll\": \"None\",\n",
      "    \"mean\": 54624.17527828223,\n",
      "    \"maximum\": 99991.0,\n",
      "    \"minimum\": 10000.0,\n",
      "    \"sum\": 164471862286.0,\n",
      "    \"stdDev\": 26182.48676113594,\n",
      "    \"approxPercentiles\": []\n",
      "}\n",
      "NumericProfiles for column: total_votes: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 41,\n",
      "    \"dataType\": \"Integral\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {},\n",
      "    \"histogram\": [\n",
      "        [\n",
      "            \"34\",\n",
      "            292,\n",
      "            9.697865008376033e-05\n",
      "        ],\n",
      "        [\n",
      "            \"12\",\n",
      "            47972,\n",
      "            0.01593239658156901\n",
      "        ],\n",
      "        [\n",
      "            \"8\",\n",
      "            3891,\n",
      "            0.0012922737242325735\n",
      "        ],\n",
      "        [\n",
      "            \"19\",\n",
      "            306985,\n",
      "            0.10195544827384645\n",
      "        ],\n",
      "        [\n",
      "            \"23\",\n",
      "            206056,\n",
      "            0.06843504356732644\n",
      "        ],\n",
      "        [\n",
      "            \"4\",\n",
      "            125,\n",
      "            4.151483308380151e-05\n",
      "        ],\n",
      "        [\n",
      "            \"15\",\n",
      "            158682,\n",
      "            0.05270125394723033\n",
      "        ],\n",
      "        [\n",
      "            \"11\",\n",
      "            28004,\n",
      "            0.00930065108543022\n",
      "        ],\n",
      "        [\n",
      "            \"9\",\n",
      "            7913,\n",
      "            0.002628054993536971\n",
      "        ],\n",
      "        [\n",
      "            \"33\",\n",
      "            773,\n",
      "            0.0002567277277902285\n",
      "        ],\n",
      "        [\n",
      "            \"22\",\n",
      "            251211,\n",
      "            0.08343186187051889\n",
      "        ],\n",
      "        [\n",
      "            \"26\",\n",
      "            76151,\n",
      "            0.02529116843331655\n",
      "        ],\n",
      "        [\n",
      "            \"37\",\n",
      "            13,\n",
      "            4.317542640715357e-06\n",
      "        ],\n",
      "        [\n",
      "            \"13\",\n",
      "            75950,\n",
      "            0.025224412581717797\n",
      "        ],\n",
      "        [\n",
      "            \"24\",\n",
      "            158507,\n",
      "            0.05264313318091301\n",
      "        ],\n",
      "        [\n",
      "            \"35\",\n",
      "            100,\n",
      "            3.321186646704121e-05\n",
      "        ],\n",
      "        [\n",
      "            \"16\",\n",
      "            205878,\n",
      "            0.0683759264450151\n",
      "        ],\n",
      "        [\n",
      "            \"5\",\n",
      "            300,\n",
      "            9.963559940112363e-05\n",
      "        ],\n",
      "        [\n",
      "            \"10\",\n",
      "            15691,\n",
      "            0.005211273967343436\n",
      "        ],\n",
      "        [\n",
      "            \"21\",\n",
      "            287397,\n",
      "            0.09544990787028242\n",
      "        ],\n",
      "        [\n",
      "            \"32\",\n",
      "            1790,\n",
      "            0.0005944924097600376\n",
      "        ],\n",
      "        [\n",
      "            \"6\",\n",
      "            748,\n",
      "            0.00024842476117346824\n",
      "        ],\n",
      "        [\n",
      "            \"36\",\n",
      "            38,\n",
      "            1.262050925747566e-05\n",
      "        ],\n",
      "        [\n",
      "            \"1\",\n",
      "            2,\n",
      "            6.642373293408242e-07\n",
      "        ],\n",
      "        [\n",
      "            \"39\",\n",
      "            2,\n",
      "            6.642373293408242e-07\n",
      "        ],\n",
      "        [\n",
      "            \"17\",\n",
      "            251277,\n",
      "            0.08345378170238714\n",
      "        ],\n",
      "        [\n",
      "            \"25\",\n",
      "            113552,\n",
      "            0.037712738610654634\n",
      "        ],\n",
      "        [\n",
      "            \"14\",\n",
      "            113205,\n",
      "            0.037597493434014\n",
      "        ],\n",
      "        [\n",
      "            \"31\",\n",
      "            3833,\n",
      "            0.0012730108416816896\n",
      "        ],\n",
      "        [\n",
      "            \"0\",\n",
      "            1,\n",
      "            3.321186646704121e-07\n",
      "        ],\n",
      "        [\n",
      "            \"20\",\n",
      "            306210,\n",
      "            0.10169805630872689\n",
      "        ],\n",
      "        [\n",
      "            \"27\",\n",
      "            47680,\n",
      "            0.015835417931485248\n",
      "        ],\n",
      "        [\n",
      "            \"2\",\n",
      "            13,\n",
      "            4.317542640715357e-06\n",
      "        ],\n",
      "        [\n",
      "            \"38\",\n",
      "            4,\n",
      "            1.3284746586816483e-06\n",
      "        ],\n",
      "        [\n",
      "            \"18\",\n",
      "            286997,\n",
      "            0.09531706040441426\n",
      "        ],\n",
      "        [\n",
      "            \"30\",\n",
      "            7922,\n",
      "            0.0026310440615190047\n",
      "        ],\n",
      "        [\n",
      "            \"7\",\n",
      "            1782,\n",
      "            0.0005918354604426743\n",
      "        ],\n",
      "        [\n",
      "            \"29\",\n",
      "            15535,\n",
      "            0.005159463455654852\n",
      "        ],\n",
      "        [\n",
      "            \"3\",\n",
      "            49,\n",
      "            1.6273814568850192e-05\n",
      "        ],\n",
      "        [\n",
      "            \"28\",\n",
      "            28441,\n",
      "            0.00944578694189119\n",
      "        ]\n",
      "    ],\n",
      "    \"kll\": \"None\",\n",
      "    \"mean\": 19.500789446065923,\n",
      "    \"maximum\": 39.0,\n",
      "    \"minimum\": 0.0,\n",
      "    \"sum\": 58716331.0,\n",
      "    \"stdDev\": 3.884513343386856,\n",
      "    \"approxPercentiles\": []\n",
      "}\n",
      "StandardProfiles for column: review_body: {\n",
      "    \"completeness\": 1.0,\n",
      "    \"approximateNumDistinctValues\": 3111148,\n",
      "    \"dataType\": \"String\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 0,\n",
      "        \"Unknown\": 0,\n",
      "        \"String\": 3010972\n",
      "    },\n",
      "    \"histogram\": null\n",
      "}\n",
      "StandardProfiles for column: marketplace: {\n",
      "    \"completeness\": 0.8570391886739565,\n",
      "    \"approximateNumDistinctValues\": 6,\n",
      "    \"dataType\": \"String\",\n",
      "    \"isDataTypeInferred\": false,\n",
      "    \"typeCounts\": {\n",
      "        \"Boolean\": 0,\n",
      "        \"Fractional\": 0,\n",
      "        \"Integral\": 0,\n",
      "        \"Unknown\": 430451,\n",
      "        \"String\": 2580521\n",
      "    },\n",
      "    \"histogram\": [\n",
      "        [\n",
      "            \"\",\n",
      "            430828,\n",
      "            0.1430860200626243\n",
      "        ],\n",
      "        [\n",
      "            \"NullValue\",\n",
      "            430451,\n",
      "            0.14296081132604355\n",
      "        ],\n",
      "        [\n",
      "            \"US\",\n",
      "            429002,\n",
      "            0.14247957138093612\n",
      "        ],\n",
      "        [\n",
      "            \"FR\",\n",
      "            430382,\n",
      "            0.14293789513818128\n",
      "        ],\n",
      "        [\n",
      "            \"UK\",\n",
      "            430142,\n",
      "            0.1428581866586604\n",
      "        ],\n",
      "        [\n",
      "            \"JP\",\n",
      "            430345,\n",
      "            0.14292560674758847\n",
      "        ],\n",
      "        [\n",
      "            \"DE\",\n",
      "            429822,\n",
      "            0.14275190868596585\n",
      "        ]\n",
      "    ]\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "for col, profile in result.profiles.items():\n",
    "    print(profile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For more info ... look at full list of profiles in `docs/profiles.md` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}