diff --git a/README.md b/README.md index e0d3067..f388884 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ Set the following spark conf: ```python .conf("spark.jars.packages", - "com.rovio.ingest:rovio-ingest:1.0.6_spark_3.0.1") \ + "com.rovio.ingest:rovio-ingest:1.0.7_spark_3.4.1") \ ``` #### PySpark job example @@ -207,14 +207,14 @@ A `Dataset[Row]` extension is provided to repartition the dataset for the `Druid For an interactive spark session you can set the following spark conf: ```scala -("spark.jars.packages", "com.rovio.ingest:rovio-ingest:1.0.6_spark_3.0.1") +("spark.jars.packages", "com.rovio.ingest:rovio-ingest:1.0.7_spark_3.4.1") ``` To use a snapshot version: ```scala ("spark.jars.repositories", "https://s01.oss.sonatype.org/content/repositories/snapshots"), -("spark.jars.packages", "com.rovio.ingest:rovio-ingest:1.0.7_spark_3.4.1-SNAPSHOT") +("spark.jars.packages", "com.rovio.ingest:rovio-ingest:1.0.8_spark_3.4.1-SNAPSHOT") ``` ```scala @@ -250,7 +250,7 @@ Maven (for a full example, see [examples/rovio-ingest-maven-example](examples/ro com.rovio.ingest rovio-ingest - 1.0.6_spark_3.0.1 + 1.0.7_spark_3.4.1 org.apache.logging.log4j diff --git a/examples/rovio-ingest-maven-example/pom.xml b/examples/rovio-ingest-maven-example/pom.xml index 8285e6d..f28113f 100644 --- a/examples/rovio-ingest-maven-example/pom.xml +++ b/examples/rovio-ingest-maven-example/pom.xml @@ -38,9 +38,9 @@ com.rovio.ingest rovio-ingest - 1.0.6_spark_3.0.1 + 1.0.7_spark_3.4.1 diff --git a/pom.xml b/pom.xml index 374a226..c5f2e81 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.rovio.ingest rovio-ingest - 1.0.7_spark_3.4.1-SNAPSHOT + 1.0.8_spark_3.4.1-SNAPSHOT jar rovio-ingest An implementation of the DatasourceV2 interface of Apache Spark™ for writing Spark Datasets to Apache Druid™ diff --git a/python/notebooks/druid_ingestion_test.ipynb b/python/notebooks/druid_ingestion_test.ipynb index 2ca6084..f040c29 100644 --- a/python/notebooks/druid_ingestion_test.ipynb +++ b/python/notebooks/druid_ingestion_test.ipynb @@ -66,14 +66,14 @@ "2. A) Copy the shaded jar to s3:\n", "\n", " aws s3 --profile $AWS_PROFILE cp \\\n", - " target/rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar \\\n", - " s3://$JAR_BUCKET/tmp/juho/druid/jars/rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar\n", + " target/rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar \\\n", + " s3://$JAR_BUCKET/tmp/juho/druid/jars/rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar\n", "\n", "2. B) Copy the plain jar to s3: \n", "\n", " aws s3 --profile $AWS_PROFILE cp \\\n", - " target/original-rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar \\\n", - " s3://$JAR_BUCKET/tmp/juho/druid/jars/original-rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar\n", + " target/original-rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar \\\n", + " s3://$JAR_BUCKET/tmp/juho/druid/jars/original-rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar\n", "\n", "Then invert the boolean in the cell below to use it in spark_conf." ] @@ -200,8 +200,8 @@ " \"spark.sql.session.timeZone\": \"UTC\",\n", " # alternative if using a snapshot version\n", "# \"spark.jars.repositories\": \"https://s01.oss.sonatype.org/content/repositories/snapshots\",\n", - "# \"spark.jars.packages\": \"com.rovio.ingest:rovio-ingest:1.0.7_spark_3.4.1-SNAPSHOT\"\n", - " \"spark.jars.packages\": \"com.rovio.ingest:rovio-ingest:1.0.6_spark_3.0.1\"\n", + "# \"spark.jars.packages\": \"com.rovio.ingest:rovio-ingest:1.0.8_spark_3.4.1-SNAPSHOT\"\n", + " \"spark.jars.packages\": \"com.rovio.ingest:rovio-ingest:1.0.7_spark_3.4.1\"\n", " }\n", "}\n", "\n", @@ -220,7 +220,7 @@ "# Enable this to test with a manually built & copied jar instead of published package from maven\n", "if False:\n", " spark_conf[\"conf\"][\"spark.jars\"] = \\\n", - " f\"s3://{packages_bucket}/{PREFIX}druid/jars/rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar\"\n", + " f\"s3://{packages_bucket}/{PREFIX}druid/jars/rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar\"\n", " del spark_conf[\"conf\"][\"spark.jars.packages\"]\n", "\n", "set_spark_config(spark_conf)\n", diff --git a/python/notebooks/druid_sketch_ingestion_test.ipynb b/python/notebooks/druid_sketch_ingestion_test.ipynb index 8087c34..ffa85f0 100644 --- a/python/notebooks/druid_sketch_ingestion_test.ipynb +++ b/python/notebooks/druid_sketch_ingestion_test.ipynb @@ -66,14 +66,14 @@ "2. A) Copy the shaded jar to s3:\n", "\n", " aws s3 --profile $AWS_PROFILE cp \\\n", - " target/rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar \\\n", - " s3://$JAR_BUCKET/tmp/vivek/druid/jars/rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar\n", + " target/rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar \\\n", + " s3://$JAR_BUCKET/tmp/vivek/druid/jars/rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar\n", "\n", "2. B) Copy the plain jar to s3: \n", "\n", " aws s3 --profile $AWS_PROFILE cp \\\n", - " target/original-rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar \\\n", - " s3://$JAR_BUCKET/tmp/vivek/druid/jars/original-rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar\n", + " target/original-rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar \\\n", + " s3://$JAR_BUCKET/tmp/vivek/druid/jars/original-rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar\n", "\n", "Then invert the boolean in the cell below to use it in spark_conf." ] @@ -166,7 +166,7 @@ "if True:\n", " jars_base_path = \"s3://{packages_bucket}/{PREFIX}druid/jars\"\n", " jars = (\n", - " f\"{jars_base_path}/rovio-ingest-1.0.7_spark_3.4.1-SNAPSHOT.jar,\"\n", + " f\"{jars_base_path}/rovio-ingest-1.0.8_spark_3.4.1-SNAPSHOT.jar,\"\n", " f\"{jars_base_path}/datasketches-hive-1.2.0.jar,\"\n", " f\"{jars_base_path}/datasketches-java-4.1.0.jar,\"\n", " f\"{jars_base_path}/datasketches-memory-2.0.0.jar\"\n",