From 98f9ff47091162cfcca872ffd66d5ecce6a7324e Mon Sep 17 00:00:00 2001 From: Tim Van Wassenhove Date: Thu, 9 Nov 2023 08:41:21 +0100 Subject: [PATCH] hardcode mobile_attributes schema --- .../resources/mobile_attributes_headers.tsv | 1 - .../MobileAttributeValuesContributor.scala | 3 +- .../datafeed/util/MobileAttributes.scala | 63 +++++++++++++++---- .../datafeed/util/MobileAttributesTests.scala | 17 ----- 4 files changed, 53 insertions(+), 31 deletions(-) delete mode 100644 src/main/resources/mobile_attributes_headers.tsv delete mode 100644 src/test/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributesTests.scala diff --git a/src/main/resources/mobile_attributes_headers.tsv b/src/main/resources/mobile_attributes_headers.tsv deleted file mode 100644 index 8446abd..0000000 --- a/src/main/resources/mobile_attributes_headers.tsv +++ /dev/null @@ -1 +0,0 @@ -mobile_id Manufacturer Device Device_Type Operating_System Diagonal_Screen_Size Screen_Height Screen_Width Cookie_Support Color_Depth MP3_Audio_Support AAC_Audio_Support AMR_Audio_Support Midi_Monophonic_Audio_Support Midi_Polyphonic_Audio_Support QCELP_Audio_Support GIF87_Image_Support GIF89a_Image_Support PNG_Image_Support JPG_Image_Support 3GPP_Video_Support MPEG4_Video_Support 3GPP2_Video_Support WMV_Video_Support MPEG4_Part_2_Video_Support Stream_MP4_AAC_LC_Video_Support Stream_3GP_H264_Level_10b_Video_Support Stream_3GP_AAC_LC_Video_Support 3GP_AAC_LC_Video_Support Stream_MP4_H264_Level_11_Video_Support Stream_MP4_H264_Level_13_Video_Support Stream_3GP_H264_Level_12_Video_Support Stream_3GP_H264_Level_11_Video_Support Stream_3GP_H264_Level_10_Video_Support Stream_3GP_H264_Level_13_Video_Support 3GP_AMR_NB_Video_Support 3GP_AMR_WB_Video_Support MP4_H264_Level_11_Video_Support 3GP_H263_Video_Support MP4_H264_Level_13_Video_Support Stream_3GP_H263_Video_Support Stream_3GP_AMR_WB_Video_Support 3GP_H264_Level_10b_Video_Support MP4_ACC_LC_Video_Support Stream_3GP_AMR_NB_Video_Support 3GP_H264_Level_10_Video_Support 3GP_H264_Level_13_Video_Support 3GP_H264_Level_11_Video_Support 3GP_H264_Level_12_Video_Support Stream_HTTP_Live_Streaming_Video_Support \ No newline at end of file diff --git a/src/main/scala/be/icteam/adobe/analytics/datafeed/contributor/MobileAttributeValuesContributor.scala b/src/main/scala/be/icteam/adobe/analytics/datafeed/contributor/MobileAttributeValuesContributor.scala index 115cb35..9baf243 100644 --- a/src/main/scala/be/icteam/adobe/analytics/datafeed/contributor/MobileAttributeValuesContributor.scala +++ b/src/main/scala/be/icteam/adobe/analytics/datafeed/contributor/MobileAttributeValuesContributor.scala @@ -64,8 +64,7 @@ case class MobileAttributeValuesContributor(lookupFilesByName: Map[String, File] private case class SimpleLookupRule(lookupfileName: String, phyiscalColumnName: String, resultSchemaField: StructField) private lazy val mobileAttributesSchema = { - val fullSchema = MobileAttributes.getSchema() - StructType(fullSchema.fields.filter(x => x.name != "mobile_id")) + StructType(MobileAttributes.schema.fields.filter(x => x.name != "mobile_id")) } private val simpleLookupRules = List( diff --git a/src/main/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributes.scala b/src/main/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributes.scala index aeb62fb..4ace065 100644 --- a/src/main/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributes.scala +++ b/src/main/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributes.scala @@ -2,18 +2,59 @@ package be.icteam.adobe.analytics.datafeed.util import org.apache.spark.sql.types.{StringType, StructField, StructType} -import scala.io.Source - case object MobileAttributes { - def getHeaders(): List[String] = { - val in = getClass.getResourceAsStream("/mobile_attributes_headers.tsv") - Source.fromInputStream(in).getLines().flatMap(_.split('\t')).toList - } - - def getSchema(): StructType = { - val headerFields = getHeaders().map(x => StructField(x, StringType)) - StructType(headerFields) - } + val schema: StructType = StructType(Array( + StructField("mobile_id", StringType), + StructField("Manufacturer", StringType), + StructField("Device", StringType), + StructField("Device_Type", StringType), + StructField("Operating_System", StringType), + StructField("Diagonal_Screen_Size", StringType), + StructField("Screen_Height", StringType), + StructField("Screen_Width", StringType), + StructField("Cookie_Support", StringType), + StructField("Color_Depth", StringType), + StructField("MP3_Audio_Support", StringType), + StructField("AAC_Audio_Support", StringType), + StructField("AMR_Audio_Support", StringType), + StructField("Midi_Monophonic_Audio_Support", StringType), + StructField("Midi_Polyphonic_Audio_Support", StringType), + StructField("QCELP_Audio_Support", StringType), + StructField("GIF87_Image_Support", StringType), + StructField("GIF89a_Image_Support", StringType), + StructField("PNG_Image_Support", StringType), + StructField("JPG_Image_Support", StringType), + StructField("3GPP_Video_Support", StringType), + StructField("MPEG4_Video_Support", StringType), + StructField("3GPP2_Video_Support", StringType), + StructField("WMV_Video_Support", StringType), + StructField("MPEG4_Part_2_Video_Support", StringType), + StructField("Stream_MP4_AAC_LC_Video_Support", StringType), + StructField("Stream_3GP_H264_Level_10b_Video_Support", StringType), + StructField("Stream_3GP_AAC_LC_Video_Support", StringType), + StructField("3GP_AAC_LC_Video_Support", StringType), + StructField("Stream_MP4_H264_Level_11_Video_Support", StringType), + StructField("Stream_MP4_H264_Level_13_Video_Support", StringType), + StructField("Stream_3GP_H264_Level_12_Video_Support", StringType), + StructField("Stream_3GP_H264_Level_11_Video_Support", StringType), + StructField("Stream_3GP_H264_Level_10_Video_Support", StringType), + StructField("Stream_3GP_H264_Level_13_Video_Support", StringType), + StructField("3GP_AMR_NB_Video_Support", StringType), + StructField("3GP_AMR_WB_Video_Support", StringType), + StructField("MP4_H264_Level_11_Video_Support", StringType), + StructField("3GP_H263_Video_Support", StringType), + StructField("MP4_H264_Level_13_Video_Support", StringType), + StructField("Stream_3GP_H263_Video_Support", StringType), + StructField("Stream_3GP_AMR_WB_Video_Support", StringType), + StructField("3GP_H264_Level_10b_Video_Support", StringType), + StructField("MP4_ACC_LC_Video_Support", StringType), + StructField("Stream_3GP_AMR_NB_Video_Support", StringType), + StructField("3GP_H264_Level_10_Video_Support", StringType), + StructField("3GP_H264_Level_13_Video_Support", StringType), + StructField("3GP_H264_Level_11_Video_Support", StringType), + StructField("3GP_H264_Level_12_Video_Support", StringType), + StructField("Stream_HTTP_Live_Streaming_Video_Support", StringType) + )) } diff --git a/src/test/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributesTests.scala b/src/test/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributesTests.scala deleted file mode 100644 index ea84f4b..0000000 --- a/src/test/scala/be/icteam/adobe/analytics/datafeed/util/MobileAttributesTests.scala +++ /dev/null @@ -1,17 +0,0 @@ -package be.icteam.adobe.analytics.datafeed.util - -import org.scalatest.funsuite.AnyFunSuite - -class MobileAttributesTests extends AnyFunSuite { - - test("can read the headers") { - val headers = MobileAttributes.getHeaders() - assert(headers.length > 0) - } - - test("can get schema") { - val schema = MobileAttributes.getSchema() - assert(schema.fieldNames.contains("Manufacturer")) - assert(schema.fieldNames.contains("Device")) - } -}