introduce util package

timvw · Nov 8, 2023 · bba670a · bba670a
1 parent a4b6700
commit bba670a
Show file tree

Hide file tree

Showing 18 changed files with 20 additions and 14 deletions.
diff --git a/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedPartition.scala b/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedPartition.scala
@@ -1,5 +1,6 @@
 package be.icteam.adobe.analytics.datafeed
 
+import be.icteam.adobe.analytics.datafeed.util.{DataFile, ManifestFile}
 import org.apache.spark.sql.connector.read.InputPartition
 
 case class DatafeedPartition(dataFile: DataFile, manifestFile: ManifestFile, options: DatafeedOptions) extends InputPartition
diff --git a/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedPartitionReader.scala b/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedPartitionReader.scala
@@ -1,6 +1,7 @@
 package be.icteam.adobe.analytics.datafeed
 
 import be.icteam.adobe.analytics.datafeed.contributor.ValuesContributor
+import be.icteam.adobe.analytics.datafeed.util.LookupFile
 import com.univocity.parsers.tsv.{TsvParser, TsvParserSettings}
 import org.apache.hadoop.conf.Configuration
 import org.apache.spark.sql.catalyst.InternalRow

diff --git a/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedScan.scala b/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedScan.scala
@@ -1,5 +1,6 @@
 package be.icteam.adobe.analytics.datafeed
 
+import be.icteam.adobe.analytics.datafeed.util.ManifestFile
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.connector.read._
 import org.apache.spark.sql.types.StructType

diff --git a/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedScanBuilder.scala b/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedScanBuilder.scala
@@ -1,5 +1,6 @@
 package be.icteam.adobe.analytics.datafeed
 
+import be.icteam.adobe.analytics.datafeed.util.ManifestFile
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.connector.read.{Scan, ScanBuilder, SupportsPushDownRequiredColumns}
 import org.apache.spark.sql.types.StructType

diff --git a/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedTable.scala b/src/main/scala/be/icteam/adobe/analytics/datafeed/DatafeedTable.scala
@@ -1,6 +1,7 @@
 package be.icteam.adobe.analytics.datafeed
 
 import be.icteam.adobe.analytics.datafeed.contributor.ValuesContributor
+import be.icteam.adobe.analytics.datafeed.util.{LookupFile, ManifestFile}
 import org.apache.hadoop.fs.Path
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability}

diff --git a/...ain/scala/be/icteam/adobe/analytics/datafeed/contributor/EventListValuesContributor.scala b/...ain/scala/be/icteam/adobe/analytics/datafeed/contributor/EventListValuesContributor.scala
@@ -1,6 +1,6 @@
 package be.icteam.adobe.analytics.datafeed.contributor
 
-import be.icteam.adobe.analytics.datafeed.{LookupDatabase, LookupFile}
+import be.icteam.adobe.analytics.datafeed.util.{LookupDatabase, LookupFile}
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
 import org.apache.spark.sql.catalyst.util.ArrayData
 import org.apache.spark.sql.types.{ArrayType, StringType, StructField, StructType}

diff --git a/...n/scala/be/icteam/adobe/analytics/datafeed/contributor/ProductListValuesContributor.scala b/...n/scala/be/icteam/adobe/analytics/datafeed/contributor/ProductListValuesContributor.scala
@@ -1,6 +1,6 @@
 package be.icteam.adobe.analytics.datafeed.contributor
 
-import be.icteam.adobe.analytics.datafeed.Product
+import be.icteam.adobe.analytics.datafeed.util.Product
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
 import org.apache.spark.sql.catalyst.util.ArrayData
 import org.apache.spark.sql.types.{ArrayType, StringType, StructField, StructType}

diff --git a/.../scala/be/icteam/adobe/analytics/datafeed/contributor/SimpleLookupValuesContributor.scala b/.../scala/be/icteam/adobe/analytics/datafeed/contributor/SimpleLookupValuesContributor.scala
@@ -1,6 +1,6 @@
 package be.icteam.adobe.analytics.datafeed.contributor
 
-import be.icteam.adobe.analytics.datafeed.LookupFile
+import be.icteam.adobe.analytics.datafeed.util.LookupFile
 import com.univocity.parsers.tsv.{TsvParser, TsvParserSettings}
 import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
 import org.apache.spark.sql.types.{StringType, StructField, StructType}

diff --git a/...m/adobe/analytics/datafeed/DataFile.scala → ...be/analytics/datafeed/util/DataFile.scala b/...m/adobe/analytics/datafeed/DataFile.scala → ...be/analytics/datafeed/util/DataFile.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import org.apache.hadoop.fs.Path
 

diff --git a/...e/analytics/datafeed/LookupDatabase.scala → ...lytics/datafeed/util/LookupDatabase.scala b/...e/analytics/datafeed/LookupDatabase.scala → ...lytics/datafeed/util/LookupDatabase.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import com.univocity.parsers.tsv.{TsvParser, TsvParserSettings}
 import org.rocksdb.{Options, RocksDB}

diff --git a/...adobe/analytics/datafeed/LookupFile.scala → .../analytics/datafeed/util/LookupFile.scala b/...adobe/analytics/datafeed/LookupFile.scala → .../analytics/datafeed/util/LookupFile.scala
@@ -1,5 +1,6 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
+import be.icteam.adobe.analytics.datafeed.DatafeedOptions
 import com.univocity.parsers.tsv.{TsvParser, TsvParserSettings}
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream

diff --git a/...obe/analytics/datafeed/ManifestFile.scala → ...nalytics/datafeed/util/ManifestFile.scala b/...obe/analytics/datafeed/ManifestFile.scala → ...nalytics/datafeed/util/ManifestFile.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, Path}

diff --git a/...am/adobe/analytics/datafeed/Product.scala → ...obe/analytics/datafeed/util/Product.scala b/...am/adobe/analytics/datafeed/Product.scala → ...obe/analytics/datafeed/util/Product.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import java.util.regex.Pattern
 

diff --git a/...cteam/adobe/analytics/datafeed/Util.scala → .../adobe/analytics/datafeed/util/Util.scala b/...cteam/adobe/analytics/datafeed/Util.scala → .../adobe/analytics/datafeed/util/Util.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import org.apache.hadoop.fs.{FileSystem, Path}
 

diff --git a/...be/analytics/datafeed/DataFileTests.scala → ...alytics/datafeed/util/DataFileTests.scala b/...be/analytics/datafeed/DataFileTests.scala → ...alytics/datafeed/util/DataFileTests.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import org.apache.hadoop.fs.Path
 import org.scalatest.funsuite.AnyFunSuite

diff --git a/.../analytics/datafeed/LookupFileTests.scala → ...ytics/datafeed/util/LookupFileTests.scala b/.../analytics/datafeed/LookupFileTests.scala → ...ytics/datafeed/util/LookupFileTests.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path

diff --git a/...nalytics/datafeed/ManifestFileTests.scala → ...ics/datafeed/util/ManifestFileTests.scala b/...nalytics/datafeed/ManifestFileTests.scala → ...ics/datafeed/util/ManifestFileTests.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path

diff --git a/...alytics/datafeed/ProductParserTests.scala → ...cs/datafeed/util/ProductParserTests.scala b/...alytics/datafeed/ProductParserTests.scala → ...cs/datafeed/util/ProductParserTests.scala
@@ -1,4 +1,4 @@
-package be.icteam.adobe.analytics.datafeed
+package be.icteam.adobe.analytics.datafeed.util
 
 import org.scalatest.funsuite.AnyFunSuite
 
@@ -9,7 +9,7 @@ class ProductParserTests extends AnyFunSuite {
   /* examples from: https://experienceleague.adobe.com/docs/analytics/implementation/vars/page-vars/products.html?lang=en */
   test("parse product") {
 
-    import Product.Parser.parseProducts
+    import be.icteam.adobe.analytics.datafeed.util.Product.Parser.parseProducts
 
     // Include only product and category. Common on individual product pages// Include only product and category. Common on individual product pages
     assert(parseProducts("Example category;Example product") == List(