diff --git a/asset-enrichment/pom.xml b/asset-enrichment/pom.xml deleted file mode 100644 index 463d59fc0..000000000 --- a/asset-enrichment/pom.xml +++ /dev/null @@ -1,243 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - asset-enrichment - 1.0.0 - jar - asset-enrichment - - Asset Enrichment Flink Job - - - UTF-8 - 1.4.0 - - - - - com.google.guava - guava - 21.0 - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - org.bytedeco - javacv-platform - 1.5 - - - org.im4java - im4java - 1.4.0 - - - com.google.apis - google-api-services-youtube - v3-rev182-1.22.0 - - - com.google.guava - guava-jdk5 - - - - - org.imgscalr - imgscalr-lib - 4.2 - jar - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - org.sunbird.job.assetenricment.task.AssetEnrichmentStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - asset-enrichment-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - diff --git a/asset-enrichment/src/main/resources/asset-enrichment.conf b/asset-enrichment/src/main/resources/asset-enrichment.conf deleted file mode 100644 index 2f16eabf7..000000000 --- a/asset-enrichment/src/main/resources/asset-enrichment.conf +++ /dev/null @@ -1,48 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.learning.job.request" - groupId = "sunbirddev-asset-enrichment-group" - video_stream.topic = "sunbirddev.content.postpublish.request" -} - -task { - consumer.parallelism = 1 - router.parallelism = 1 - videoEnrichment.parallelism = 1 - imageEnrichment.parallelism = 1 -} - -content { - stream { - enabled = true - mimeType = ["video/mp4", "video/webm"] - } - youtube { - applicationName = "fetch-youtube-license" - regexPattern = ["\\?vi?=([^&]*)", "watch\\?.*v=([^&]*)", "(?:embed|vi?)/([^/?]*)", "^([A-Za-z0-9\\-\\_]*)"] - } - upload.context.driven = true - max.iteration.count = 2 -} - -thumbnail.max { - sample = 5 - size.pixel = 150 -} - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "CONTENT_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"] -cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] - -cloud_storage_type="" -cloud_storage_key="" -cloud_storage_secret="" -cloud_storage_container="" -cloud_storage_endpoint="" \ No newline at end of file diff --git a/asset-enrichment/src/main/resources/log4j.properties b/asset-enrichment/src/main/resources/log4j.properties deleted file mode 100644 index 13e7d31a9..000000000 --- a/asset-enrichment/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=asset-enrichment.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/domain/Event.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/domain/Event.scala deleted file mode 100644 index 9f077cc85..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/domain/Event.scala +++ /dev/null @@ -1,34 +0,0 @@ -package org.sunbird.job.assetenricment.domain - -import org.sunbird.job.domain.reader.JobRequest - -import java.util - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "AssetEnrichment" - - val data: util.Map[String, Any] = getMap() - - def eData: Map[String, AnyRef] = eventMap.getOrDefault("edata", new java.util.HashMap[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - - def objectData: Map[String, AnyRef] = eventMap.getOrDefault("object", new java.util.HashMap[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - - def id: String = objectData.getOrElse("id", "").asInstanceOf[String] - - def objectType: String = eData.getOrElse("objectType", "").asInstanceOf[String] - - def mediaType: String = eData.getOrElse("mediaType", "").asInstanceOf[String] - - def status: String = eData.getOrElse("status", "").asInstanceOf[String] - - def validate(maxIterationCount: Int): String = { - val iteration = eData.getOrElse("iteration", 0).asInstanceOf[Int] - if (id.isEmpty) s"Invalid ID present in the Event for the Object Data: ${objectData}." - else if (!objectType.equalsIgnoreCase("asset")) s"Ignoring Event due to ObjectType : ${objectType} for ID : ${id}." - else if (!mediaType.equalsIgnoreCase("image") && !mediaType.equalsIgnoreCase("video")) s"Ignoring Event due to MediaType: ${mediaType} for ID : ${id}." - else if (iteration == 1 && status.equalsIgnoreCase("processing")) "" - else if (iteration > 1 && iteration <= maxIterationCount && status.equalsIgnoreCase("failed")) "" - else s"Ignoring Event due to Iteration Limit Exceed. Iteration Count : ${iteration} for ID : ${id}." - } -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/AssetEnrichmentEventRouter.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/AssetEnrichmentEventRouter.scala deleted file mode 100644 index 7ea16145c..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/AssetEnrichmentEventRouter.scala +++ /dev/null @@ -1,47 +0,0 @@ -package org.sunbird.job.assetenricment.functions - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.assetenricment.domain.Event -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.{BaseProcessFunction, Metrics} - -class AssetEnrichmentEventRouter(config: AssetEnrichmentConfig) - extends BaseProcessFunction[Event, String](config) { - - private[this] val logger = LoggerFactory.getLogger(classOf[AssetEnrichmentEventRouter]) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - logger.info(s"Processing event for AssetEnrichment for identifier : ${event.id}") - metrics.incCounter(config.totalEventsCount) - val message = event.validate(config.maxIterationCount) - if (message.isEmpty) { - event.mediaType.toLowerCase match { - case "image" => - context.output(config.imageEnrichmentDataOutTag, event) - case "video" => - context.output(config.videoEnrichmentDataOutTag, event) - case _ => - logSkippedEvent(s"Media Type UNKNOWN. Identifier: ${event.id} & mediaType: ${event.mediaType}.")(metrics) - } - } else logSkippedEvent(message)(metrics) - } - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.skippedEventCount) - } - - def logSkippedEvent(message: String)(metrics: Metrics): Unit = { - logger.info(message) - metrics.incCounter(config.skippedEventCount) - } -} \ No newline at end of file diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/ImageEnrichmentFunction.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/ImageEnrichmentFunction.scala deleted file mode 100644 index 9ad7418f3..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/ImageEnrichmentFunction.scala +++ /dev/null @@ -1,69 +0,0 @@ -package org.sunbird.job.assetenricment.functions - -import org.apache.commons.lang3.StringUtils -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.assetenricment.domain.Event -import org.sunbird.job.assetenricment.helpers.{ImageEnrichmentHelper, OptimizerHelper} -import org.sunbird.job.assetenricment.models.Asset -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.util.{CloudStorageUtil, Neo4JUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import scala.collection.JavaConverters._ - -class ImageEnrichmentFunction(config: AssetEnrichmentConfig, - @transient var neo4JUtil: Neo4JUtil = null) - extends BaseProcessFunction[Event, String](config) - with ImageEnrichmentHelper with OptimizerHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[ImageEnrichmentFunction]) - lazy val definitionCache: DefinitionCache = new DefinitionCache - lazy val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(config) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - } - - override def close(): Unit = { - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - logger.info(s"Received message for Image Enrichment for identifier : ${event.id}.") - metrics.incCounter(config.imageEnrichmentEventCount) - val asset = Asset(event.data) - try { - if (asset.validate(config.contentUploadContextDriven)) replaceArtifactUrl(asset)(cloudStorageUtil) - asset.putAll(getMetadata(event.id)(neo4JUtil)) - val mimeType = asset.get("mimeType", "").asInstanceOf[String] - if (config.unsupportedMimeTypes.contains(mimeType) || !StringUtils.startsWithIgnoreCase(mimeType, "image")) { - saveImageVariants(Map(), asset)(neo4JUtil) - metrics.incCounter(config.ignoredImageEnrichmentEventCount) - } else { - enrichImage(asset)(config, definitionCache, cloudStorageUtil, neo4JUtil) - metrics.incCounter(config.successImageEnrichmentEventCount) - } - } catch { - case ex: Exception => - logger.error(s"Error while processing message for Image Enrichment for identifier : ${asset.identifier}.", ex) - metrics.incCounter(config.failedImageEnrichmentEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - override def metricsList(): List[String] = { - List(config.successImageEnrichmentEventCount, config.failedImageEnrichmentEventCount, config.imageEnrichmentEventCount, config.ignoredImageEnrichmentEventCount) - } - - def getMetadata(identifier: String)(neo4JUtil: Neo4JUtil): Map[String, AnyRef] = { - val metadata = neo4JUtil.getNodeProperties(identifier).asScala.toMap - if (metadata != null && metadata.nonEmpty) metadata else throw new Exception(s"Received null or Empty metadata for identifier: $identifier.") - } - -} \ No newline at end of file diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/VideoEnrichmentFunction.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/VideoEnrichmentFunction.scala deleted file mode 100644 index f94a6cfbd..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/functions/VideoEnrichmentFunction.scala +++ /dev/null @@ -1,63 +0,0 @@ -package org.sunbird.job.assetenricment.functions - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.assetenricment.domain.Event -import org.sunbird.job.assetenricment.helpers.{OptimizerHelper, VideoEnrichmentHelper} -import org.sunbird.job.assetenricment.models.Asset -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.assetenricment.util.YouTubeUtil -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.util.{CloudStorageUtil, Neo4JUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import scala.collection.JavaConverters._ - -class VideoEnrichmentFunction(config: AssetEnrichmentConfig, - @transient var neo4JUtil: Neo4JUtil = null) - extends BaseProcessFunction[Event, String](config) - with VideoEnrichmentHelper with OptimizerHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[VideoEnrichmentFunction]) - lazy val youTubeUtil = new YouTubeUtil(config) - lazy val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(config) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - } - - override def close(): Unit = { - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - logger.info(s"Received message for Video Enrichment for identifier : ${event.id}.") - metrics.incCounter(config.videoEnrichmentEventCount) - val asset = Asset(event.data) - try { - if (asset.validate(config.contentUploadContextDriven)) replaceArtifactUrl(asset)(cloudStorageUtil) - asset.putAll(getMetadata(event.id)(neo4JUtil)) - val enrichedAsset = enrichVideo(asset)(config, youTubeUtil, cloudStorageUtil, neo4JUtil) - pushStreamingUrlEvent(enrichedAsset, context)(metrics, config) - metrics.incCounter(config.successVideoEnrichmentEventCount) - } catch { - case ex: Exception => - logger.error(s"Error while processing message for Video Enrichment for identifier : ${asset.identifier}.", ex) - metrics.incCounter(config.failedVideoEnrichmentEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - override def metricsList(): List[String] = { - List(config.successVideoEnrichmentEventCount, config.failedVideoEnrichmentEventCount, config.videoEnrichmentEventCount, config.videoStreamingGeneratorEventCount) - } - - def getMetadata(identifier: String)(neo4JUtil: Neo4JUtil): Map[String, AnyRef] = { - val metadata = neo4JUtil.getNodeProperties(identifier).asScala.toMap - if (metadata != null && metadata.nonEmpty) metadata else throw new Exception(s"Received null or Empty metadata for identifier: $identifier.") - } - -} \ No newline at end of file diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/ImageEnrichmentHelper.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/ImageEnrichmentHelper.scala deleted file mode 100644 index 4710f1638..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/ImageEnrichmentHelper.scala +++ /dev/null @@ -1,118 +0,0 @@ -package org.sunbird.job.assetenricment.helpers - -import org.im4java.core.Info -import org.slf4j.LoggerFactory -import org.sunbird.job.assetenricment.models.Asset -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.assetenricment.util.{AssetFileUtils, ImageResizerUtil} -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.util.{CloudStorageUtil, FileUtils, Neo4JUtil, ScalaJsonUtil, Slug} - -import java.io.File -import scala.collection.mutable - -trait ImageEnrichmentHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[ImageEnrichmentHelper]) - private val CONTENT_FOLDER = "content" - private val ARTIFACT_FOLDER = "artifact" - - def enrichImage(asset: Asset)(implicit config: AssetEnrichmentConfig, definitionCache: DefinitionCache, cloudStorageUtil: CloudStorageUtil, neo4JUtil: Neo4JUtil): Unit = { - val downloadUrl = asset.get("artifactUrl", "").asInstanceOf[String] - try { - val variantsMap = optimizeImage(asset.identifier, downloadUrl)(config, definitionCache, cloudStorageUtil) - saveImageVariants(variantsMap, asset)(neo4JUtil) - } catch { - case e: Exception => - logger.error(s"Something Went Wrong While Performing Asset Enrichment operation.Content Id: ${asset.identifier}", e) - asset.put("processingError", e.getMessage) - asset.put("status", "Failed") - neo4JUtil.updateNode(asset.identifier, asset.getMetadata) - throw e - } - } - - def optimizeImage(contentId: String, originalURL: String)(implicit config: AssetEnrichmentConfig, definitionCache: DefinitionCache, cloudStorageUtil: CloudStorageUtil): Map[String, String] = { - val variantsMap = mutable.Map[String, String]() - val variants = getVariant()(definitionCache, config) - val originalFile = FileUtils.copyURLToFile(contentId, originalURL, originalURL.substring(originalURL.lastIndexOf("/") + 1, originalURL.length)) - try { - originalFile match { - case Some(file: File) => variants.foreach(variant => { - val resolution = variant._1 - val variantValueMap = variant._2.asInstanceOf[Map[String, AnyRef]] - val dimension = variantValueMap.getOrElse("dimensions", List[Int]()).asInstanceOf[List[Int]] - val dpi = variantValueMap.getOrElse("dpi", 0).asInstanceOf[Int] - if (dimension == null || dimension.size != 2) throw new Exception("Content Optimizer Error. Image Resolution/variants is not configured for content optimization.") - if (isImageOptimizable(file, dimension(0), dimension(1))) { - val targetResolution = getOptimalDPI(file, dpi) - val optimisedFile = optimizeImage(file, targetResolution, dimension(0), dimension(1), resolution) - if (null != optimisedFile && optimisedFile.exists) { - val optimisedURLArray = upload(optimisedFile, contentId)(cloudStorageUtil) - variantsMap.put(resolution, optimisedURLArray(1)) - } - } else variantsMap.put(resolution, originalURL) - }) - case _ => logger.error("ERR_INVALID_FILE_URL", s"Please Provide Valid File Url for identifier: $contentId!") - throw new Exception(s"Please Provide Valid File Url for identifier : $contentId and URL : $originalURL.") - } - } finally { - FileUtils.deleteDirectory(new File(s"/tmp/$contentId")) - } - if (variantsMap.getOrElse("medium", "").isEmpty && originalURL.nonEmpty) variantsMap.put("medium", originalURL) - variantsMap.toMap - } - - private def getVariant()(implicit definitionCache: DefinitionCache, config: AssetEnrichmentConfig): Map[String, AnyRef] = { - val version = config.schemaSupportVersionMap.getOrElse("asset", "1.0") - val definition = definitionCache.getDefinition("Asset", version, config.definitionBasePath) - val variants = definition.config.getOrElse("variants", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - variants - } - - private def optimizeImage(file: File, dpi: Double, width: Int, height: Int, resolution: String): File = { - val fileType = AssetFileUtils.getFileType(file) - val proc = new ImageResizerUtil - if (proc.isApplicable(fileType)) proc.process(file, dpi, width, height, resolution) else null - } - - def isImageOptimizable(file: File, dimensionX: Int, dimensionY: Int): Boolean = { - val inputFileName = file.getAbsolutePath - val imageInfo = new Info(inputFileName, true) - val width = imageInfo.getImageWidth - val height = imageInfo.getImageHeight - (dimensionX < width && dimensionY < height) - } - - def getOptimalDPI(file: File, dpi: Int): Double = { - val inputFileName = file.getAbsolutePath - val imageInfo = new Info(inputFileName, false) - val resString = imageInfo.getProperty("Resolution") - if (resString != null) { - val res = resString.split("x") - if (res.nonEmpty) { - val xresd = res(0).toDouble - if (xresd < dpi.toDouble) xresd else dpi.toDouble - } else 0.toDouble - } else 0.toDouble - } - - def saveImageVariants(variantsMap: Map[String, String], asset: Asset)(implicit neo4JUtil: Neo4JUtil): Unit = { - if (variantsMap.nonEmpty) asset.put("variants", ScalaJsonUtil.serialize(variantsMap)) - asset.put("status", "Live") - logger.info(s"Processed Image for identifier: ${asset.identifier}. Updating metadata.") - neo4JUtil.updateNode(asset.identifier, asset.getMetadata) - } - - def upload(file: File, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil): Array[String] = { - try { - val slug = Slug.makeSlug(identifier, isTransliterate = true) - val folder = s"$CONTENT_FOLDER/$slug/$ARTIFACT_FOLDER" - cloudStorageUtil.uploadFile(folder, file, Some(true)) - } catch { - case e: Exception => - throw new Exception(s"Error while uploading the File for identifier : $identifier.", e) - } - } - -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/OptimizerHelper.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/OptimizerHelper.scala deleted file mode 100644 index 370dd7698..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/OptimizerHelper.scala +++ /dev/null @@ -1,40 +0,0 @@ -package org.sunbird.job.assetenricment.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.assetenricment.models.Asset -import org.sunbird.job.util.CloudStorageUtil - -import java.io.File - -trait OptimizerHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[OptimizerHelper]) - - def replaceArtifactUrl(asset: Asset)(cloudStorageUtil: CloudStorageUtil): Unit = { - logger.info(s"Replacing ArtifactUrl for Asset Id : ${asset.identifier}") - val sourcePath = asset.artifactUrl.substring(asset.artifactUrl.indexOf(asset.artifactBasePath)) - val destinationPath = sourcePath.replace(asset.artifactBasePath + File.separator, "") - try { - cloudStorageUtil.copyObjectsByPrefix(sourcePath, destinationPath, false) - logger.info(s"Copying Objects...DONE | Under: ${destinationPath} for identifier : ${asset.identifier}") - val newArtifactUrl = asset.artifactUrl.replace(sourcePath, destinationPath) - asset.put("artifactUrl", newArtifactUrl) - asset.put("downloadUrl", newArtifactUrl) - val storageKey = asset.get("cloudStorageKey", "").asInstanceOf[String] - if (storageKey.nonEmpty) { - val cloudStorageKey = storageKey.replace(asset.artifactBasePath + File.separator, "") - asset.put("cloudStorageKey", cloudStorageKey) - } - val s3Key = asset.get("s3Key", "").asInstanceOf[String] - if (s3Key.nonEmpty) { - val s3KeyNew = s3Key.replace(asset.artifactBasePath + File.separator, "") - asset.put("s3Key", s3KeyNew) - } - } catch { - case e: Exception => - logger.error(s"Error while copying object by prefix for identifier : ${asset.identifier}", e) - throw e - } - } - -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/VideoEnrichmentHelper.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/VideoEnrichmentHelper.scala deleted file mode 100644 index 055649bac..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/helpers/VideoEnrichmentHelper.scala +++ /dev/null @@ -1,158 +0,0 @@ -package org.sunbird.job.assetenricment.helpers - -import org.apache.commons.lang3.StringUtils -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.bytedeco.javacv.{FFmpegFrameGrabber, Java2DFrameConverter} -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.assetenricment.domain.Event -import org.sunbird.job.assetenricment.models.Asset -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.assetenricment.util._ -import org.sunbird.job.util._ - -import java.io.File -import java.util.UUID -import java.util.concurrent.TimeUnit -import javax.imageio.ImageIO -import scala.collection.mutable - -trait VideoEnrichmentHelper extends ThumbnailUtil { - - private[this] val logger = LoggerFactory.getLogger(classOf[VideoEnrichmentHelper]) - private val CONTENT_FOLDER = "content" - private val ARTIFACT_FOLDER = "artifact" - - def enrichVideo(asset: Asset)(config: AssetEnrichmentConfig, youTubeUtil: YouTubeUtil, cloudStorageUtil: CloudStorageUtil, neo4JUtil: Neo4JUtil): Asset = { - try { - val videoUrl = asset.artifactUrl - enrichVideo(asset, videoUrl)(youTubeUtil, cloudStorageUtil, neo4JUtil, config) - } catch { - case e: Exception => - logger.error(s"Something Went Wrong While Performing Asset Enrichment operation. Content Id: ${asset.identifier}. ", e) - asset.put("processingError", e.getMessage) - asset.put("status", "Failed") - neo4JUtil.updateNode(asset.identifier, asset.getMetadata) - throw e - } - } - - private def enrichVideo(asset: Asset, videoUrl: String)(youTubeUtil: YouTubeUtil, cloudStorageUtil: CloudStorageUtil, neo4JUtil: Neo4JUtil, config: AssetEnrichmentConfig): Asset = { - if (StringUtils.equalsIgnoreCase("video/x-youtube", asset.mimeType)) processYoutubeVideo(asset, videoUrl)(youTubeUtil) - else processOtherVideo(asset, videoUrl)(cloudStorageUtil, config) - asset.put("status", "Live") - neo4JUtil.updateNode(asset.identifier, asset.getMetadata) - asset - } - - def processYoutubeVideo(asset: Asset, videoUrl: String)(youTubeUtil: YouTubeUtil): Unit = { - val data = youTubeUtil.getVideoInfo(videoUrl, "snippet,contentDetails", List[String]("thumbnail", "duration")) - if (data.nonEmpty) { - if (data.contains("thumbnail")) asset.put("thumbnail", data("thumbnail").asInstanceOf[String]) - if (data.contains("duration")) asset.put("duration", data("duration").toString) - } else throw new Exception(s"Failed to get data for Youtube Video Url : $videoUrl and identifier: ${asset.identifier}.") - } - - def processOtherVideo(asset: Asset, videoUrl: String)(implicit cloudStorageUtil: CloudStorageUtil, config: AssetEnrichmentConfig): Unit = { - val videoFile = FileUtils.copyURLToFile(asset.identifier, videoUrl, videoUrl.substring(videoUrl.lastIndexOf("/") + 1)) - try { - videoFile match { - case Some(file: File) => enrichVideo(asset, file)(cloudStorageUtil, config) - case _ => logger.error("ERR_INVALID_FILE_URL", s"Please Provide Valid File Url for identifier: ${asset.identifier}!") - throw new Exception(s"Invalid Artifact Url for identifier: ${asset.identifier}!") - } - } finally { - FileUtils.deleteDirectory(new File(s"/tmp/${asset.identifier}")) - } - } - - private def enrichVideo(asset: Asset, videoFile: File)(implicit cloudStorageUtil: CloudStorageUtil, config: AssetEnrichmentConfig): Unit = { - val frameGrabber = new FFmpegFrameGrabber(videoFile) - frameGrabber.start() - val videoDuration = frameGrabber.getLengthInTime - if (videoDuration != 0) asset.put("duration", TimeUnit.MICROSECONDS.toSeconds(videoDuration).toString) - val numberOfFrames = frameGrabber.getLengthInFrames - val thumbNail = fetchThumbNail(asset.identifier, config.sampleThumbnailCount, config.thumbnailSize, numberOfFrames, frameGrabber) - frameGrabber.stop() - if (null != thumbNail && thumbNail.exists) { - logger.info("Thumbnail created for Content Id: " + asset.identifier) - val urlArray = upload(thumbNail, asset.identifier)(cloudStorageUtil) - val thumbUrl = urlArray(1) - asset.put("thumbnail", thumbUrl) - } else logger.error(s"Thumbnail could not be generated for identifier : ${asset.identifier}.") - } - - private def fetchThumbNail(identifier: String, sampleThumbnailCount: Int, thumbnailSize: Int, numberOfFrames: Long, frameGrabber: FFmpegFrameGrabber): File = { - val converter = new Java2DFrameConverter - var thumbnail: File = null - var colorCount = 0 - for (i <- 1 to sampleThumbnailCount) { - val inFile = FileUtils.createFile(s"${FileUtils.getBasePath(identifier)}${File.separator}${System.currentTimeMillis}.png") - frameGrabber.setFrameNumber((numberOfFrames / sampleThumbnailCount).toInt * i) - try { - val bufferedImage = converter.convert(frameGrabber.grabImage()) - if (null != bufferedImage) { - ImageIO.write(bufferedImage, "png", inFile) - val outFile = generateOutFile(inFile, thumbnailSize) - outFile match { - case Some(file: File) => - val tmpColorCount = getImageColor(file) - if (colorCount < tmpColorCount) { - colorCount = tmpColorCount - thumbnail = file - } - case _ => logger.error(s"Thumbnail Could Not Be Generated For : $identifier.") - } - } - } catch { - case e: Throwable => - logger.error(s"Exception while generating thumbnail for identifier: $identifier.", e) - throw new Exception(s"Exception while generating thumbnail for identifier: $identifier. Error : ${e.getMessage}", e) - } - } - thumbnail - } - - private def getImageColor(imagePath: File): Int = { - val image = ImageIO.read(imagePath) - val colorSet = mutable.Set[Int]() - for (r <- 0 until image.getHeight) { - for (c <- 0 until image.getWidth) { - val colorCode = image.getRGB(c, r) - colorSet.add(colorCode) - } - } - colorSet.size - } - - def upload(uploadedFile: File, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil): Array[String] = { - try { - val slug = Slug.makeSlug(identifier, isTransliterate = true) - val folder = s"$CONTENT_FOLDER/$slug/$ARTIFACT_FOLDER" - cloudStorageUtil.uploadFile(folder, uploadedFile, Some(true)) - } catch { - case e: Exception => - throw new Exception("Error while uploading File to cloud for identifier : ${identifier}.", e) - } - } - - def pushStreamingUrlEvent(asset: Asset, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics, config: AssetEnrichmentConfig): Unit = { - if (config.isStreamingEnabled && config.streamableMimeType.contains(asset.get("mimeType", "").asInstanceOf[String])) { - val event = getStreamingEvent(asset) - context.output(config.generateVideoStreamingOutTag, event) - metrics.incCounter(config.videoStreamingGeneratorEventCount) - } - } - - def getStreamingEvent(asset: Asset)(implicit config: AssetEnrichmentConfig): String = { - val ets = System.currentTimeMillis - val mid = s"""LP.$ets.${UUID.randomUUID}""" - val metadata = asset.getMetadata - val channelId = metadata.getOrElse("channel", "").asInstanceOf[String] - val ver = metadata.getOrElse("versionKey", "").asInstanceOf[String] - val event = s"""{"eid":"BE_JOB_REQUEST", "ets": $ets, "mid": "$mid", "actor": {"id": "Post Publish Processor", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"}, "channel":"$channelId","env":"${config.jobEnv}"},"object":{"ver":"$ver","id":"${asset.identifier}"},"edata": {"action":"post-publish-process","iteration":1,"identifier":"${asset.identifier}","channel":"$channelId","artifactUrl":"${asset.artifactUrl}","mimeType":"video/mp4","contentType":"Resource","pkgVersion":1,"status":"Live"}}""".stripMargin - logger.info(s"Video Streaming Event for identifier ${asset.identifier} is : $event") - event - } - -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/models/Asset.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/models/Asset.scala deleted file mode 100644 index 2bd7ee8ef..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/models/Asset.scala +++ /dev/null @@ -1,29 +0,0 @@ -package org.sunbird.job.assetenricment.models - -import java.util -import scala.collection.mutable - -case class Asset(eventMap: util.Map[String, Any]) { - - val metadata: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]() - - def putAll(data: Map[String, AnyRef]): Unit = metadata ++= data - - def put(key: String, value: AnyRef): Unit = metadata.put(key, value.asInstanceOf[AnyRef]) - - def get(key: String, defaultValue: AnyRef): AnyRef = metadata.getOrElse(key, defaultValue) - - def getMetadata: Map[String, AnyRef] = metadata.toMap - - def artifactBasePath: String = metadata.getOrElse("artifactBasePath", "").asInstanceOf[String] - - def artifactUrl: String = metadata.getOrElse("artifactUrl", "").asInstanceOf[String] - - def identifier: String = metadata.getOrElse("IL_UNIQUE_ID", "").asInstanceOf[String] - - def mimeType: String = metadata.getOrElse("mimeType", "").asInstanceOf[String] - - def validate(contentUploadContextDriven: Boolean): Boolean = { - contentUploadContextDriven && artifactBasePath.nonEmpty && artifactUrl.nonEmpty && artifactUrl.contains(artifactBasePath) - } -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/task/AssetEnrichmentConfig.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/task/AssetEnrichmentConfig.scala deleted file mode 100644 index 6bf5e1aa2..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/task/AssetEnrichmentConfig.scala +++ /dev/null @@ -1,72 +0,0 @@ -package org.sunbird.job.assetenricment.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.assetenricment.domain.Event - -import java.util -import scala.collection.JavaConverters._ - -class AssetEnrichmentConfig(override val config: Config) extends BaseJobConfig(config, "asset-enrichment") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Job Configuration - val jobEnv: String = config.getString("job.env") - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val videoStreamingTopic: String = config.getString("kafka.video_stream.topic") - - // Parallelism - val eventRouterParallelism: Int = config.getInt("task.router.parallelism") - val videoEnrichmentIndexerParallelism: Int = config.getInt("task.videoEnrichment.parallelism") - val imageEnrichmentIndexerParallelism: Int = config.getInt("task.imageEnrichment.parallelism") - - // Consumers - val assetEnrichmentConsumer = "asset-enrichment-consumer" - val assetEnrichmentRouter = "asset-enrichment-router" - - // Metric List - val totalEventsCount = "total-events-count" - val skippedEventCount = "skipped-event-count" - val imageEnrichmentEventCount = "image-enrichment-event-count" - val successImageEnrichmentEventCount = "success-image-enrichment-event-count" - val ignoredImageEnrichmentEventCount = "skipped-image-enrichment-event-count" - val failedImageEnrichmentEventCount = "failed-image-enrichment-event-count" - val successVideoEnrichmentEventCount = "success-video-enrichment-event-count" - val failedVideoEnrichmentEventCount = "failed-video-enrichment-event-count" - val videoEnrichmentEventCount = "video-enrichment-event-count" - val videoStreamingGeneratorEventCount = "video-streaming-event-count" - - // Neo4J Configurations - val graphRoutePath: String = config.getString("neo4j.routePath") - val graphName: String = config.getString("neo4j.graph") - - // Tags - val imageEnrichmentDataOutTag: OutputTag[Event] = OutputTag[Event]("image-enrichment-data") - val videoEnrichmentDataOutTag: OutputTag[Event] = OutputTag[Event]("video-enrichment-data") - val generateVideoStreamingOutTag: OutputTag[String] = OutputTag[String]("video-streaming-generator-request") - - // Asset Variables - val contentUploadContextDriven: Boolean = if (config.hasPath("content.upload.context.driven")) config.getBoolean("content.upload.context.driven") else true - val maxIterationCount: Int = if (config.hasPath("content.max.iteration.count")) config.getInt("content.max.iteration.count") else 2 - - // Video Enrichment - val youtubeAppName: String = if (config.hasPath("content.youtube.applicationName")) config.getString("content.youtube.applicationName") else "fetch-youtube-license" - val videoIdRegex: util.List[String] = if (config.hasPath("content.youtube.regexPattern")) config.getStringList("content.youtube.regexPattern") else util.Arrays.asList[String]("\\?vi?=([^&]*)", "watch\\?.*v=([^&]*)", "(?:embed|vi?)/([^/?]*)", "^([A-Za-z0-9\\-\\_]*)") - val streamableMimeType: util.List[String] = if (config.hasPath("content.stream.mimeType")) config.getStringList("content.stream.mimeType") else util.Arrays.asList[String]("video/mp4") - val isStreamingEnabled: Boolean = if (config.hasPath("content.stream.enabled")) config.getBoolean("content.stream.enabled") else false - val sampleThumbnailCount: Int = if (config.hasPath("thumbnail.max.sample")) config.getInt("thumbnail.max.sample") else 5 - val thumbnailSize: Int = if (config.hasPath("thumbnail.max.size.pixel")) config.getInt("thumbnail.max.size.pixel") else 150 - - // Schema Definition Util for Image Enrichment - val definitionBasePath: String = if (config.hasPath("schema.base_path")) config.getString("schema.base_path") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - val schemaSupportVersionMap: Map[String, String] = if (config.hasPath("schema.supported_version")) config.getAnyRef("schema.supported_version").asInstanceOf[util.Map[String, String]].asScala.toMap else Map[String, String]() - - val unsupportedMimeTypes: util.List[String] = if (config.hasPath("unsupported.mimetypes")) config.getStringList("unsupported.mimetypes") else util.Arrays.asList[String]("image/svg+xml") -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/task/AssetEnrichmentStreamTask.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/task/AssetEnrichmentStreamTask.scala deleted file mode 100644 index f640dd976..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/task/AssetEnrichmentStreamTask.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.assetenricment.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.assetenricment.domain.Event -import org.sunbird.job.assetenricment.functions.{AssetEnrichmentEventRouter, ImageEnrichmentFunction, VideoEnrichmentFunction} -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.util.FlinkUtil - -import java.io.File - -class AssetEnrichmentStreamTask(config: AssetEnrichmentConfig, kafkaConnector: FlinkKafkaConnector) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - val processStreamTask = env.addSource(source).name(config.assetEnrichmentConsumer) - .uid(config.assetEnrichmentConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new AssetEnrichmentEventRouter(config)) - .name("asset-enrichment-router").uid("asset-enrichment-router") - .setParallelism(config.eventRouterParallelism) - - processStreamTask.getSideOutput(config.imageEnrichmentDataOutTag).process(new ImageEnrichmentFunction(config)) - .name("image-enrichment-process").uid("image-enrichment-process").setParallelism(config.imageEnrichmentIndexerParallelism) - - val videoStream = processStreamTask.getSideOutput(config.videoEnrichmentDataOutTag).process(new VideoEnrichmentFunction(config)) - .name("video-enrichment-process").uid("video-enrichment-process").setParallelism(config.videoEnrichmentIndexerParallelism) - - videoStream.getSideOutput(config.generateVideoStreamingOutTag).addSink(kafkaConnector.kafkaStringSink(config.videoStreamingTopic)) - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object AssetEnrichmentStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("asset-enrichment.conf").withFallback(ConfigFactory.systemEnvironment())) - val assetEnrichmentConfig = new AssetEnrichmentConfig(config) - val kafkaUtil = new FlinkKafkaConnector(assetEnrichmentConfig) - val task = new AssetEnrichmentStreamTask(assetEnrichmentConfig, kafkaUtil) - task.process() - } - -} -// $COVERAGE-ON$ \ No newline at end of file diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/AssetFileUtils.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/AssetFileUtils.scala deleted file mode 100644 index 11eb360e3..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/AssetFileUtils.scala +++ /dev/null @@ -1,33 +0,0 @@ -package org.sunbird.job.assetenricment.util - -import org.slf4j.LoggerFactory - -import java.io.File -import javax.activation.MimetypesFileTypeMap - -object AssetFileUtils { - - private[this] val logger = LoggerFactory.getLogger("org.sunbird.job.util.AssetFileUtils") - val mimeTypesMap: MimetypesFileTypeMap = initializeMimeTypes() - - def initializeMimeTypes(): MimetypesFileTypeMap = { - val mimeTypes = new MimetypesFileTypeMap - mimeTypes.addMimeTypes("image png jpg jpeg") - mimeTypes.addMimeTypes("audio mp3 ogg wav") - mimeTypes.addMimeTypes("video mp4") - mimeTypes - } - - def getFileType(file: File): String = { - if (file.isDirectory) "Directory" else { - val mimeType = mimeTypesMap.getContentType(file) - mimeType.split("/")(0) match { - case "image" => "Image" - case "audio" => "Audio" - case "video" => "Video" - case _ => "Other" - } - } - } - -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/ImageResizerUtil.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/ImageResizerUtil.scala deleted file mode 100644 index 87e6d8145..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/ImageResizerUtil.scala +++ /dev/null @@ -1,38 +0,0 @@ -package org.sunbird.job.assetenricment.util - -import org.im4java.core.{ConvertCmd, IMOperation} - -import java.io.File - -class ImageResizerUtil { - - def process(file: File, targetResolution: Double, width: Int, height: Int, outputFileNameSuffix: String): File = { - val inputFileName = file.getAbsolutePath - val outputFileName = file.getAbsolutePath.replaceAll("\\.", s"\\.${outputFileNameSuffix}\\.") - // set optimize width and height - val ow = width - val oh = height - // create command - val cmd = new ConvertCmd - // create the operation, add images and operators/options - val op = new IMOperation - op.addImage(inputFileName) - op.resize(ow, oh) - if (targetResolution.toInt > 0) op.resample(targetResolution.toInt) - op.addImage(outputFileName) - // execute the operation - cmd.run(op) - // replace the file - if (outputFileNameSuffix.equalsIgnoreCase("out")) replace(file, new File(outputFileName)) else new File(outputFileName) - } - - def replace(input: File, output: File): File = { - val inputFile = input.getAbsolutePath - input.delete - output.renameTo(new File(inputFile)) - output - } - - def isApplicable(fileType: String): Boolean = fileType.equalsIgnoreCase("image") - -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/ThumbnailUtil.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/ThumbnailUtil.scala deleted file mode 100644 index 4ef7faee6..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/ThumbnailUtil.scala +++ /dev/null @@ -1,40 +0,0 @@ -package org.sunbird.job.assetenricment.util - -import org.imgscalr.Scalr -import org.slf4j.LoggerFactory -import org.sunbird.job.util.FileUtils - -import java.awt.image.BufferedImage -import java.io.File -import javax.imageio.ImageIO - -trait ThumbnailUtil { - - private[this] val logger = LoggerFactory.getLogger(classOf[ThumbnailUtil]) - - def generateOutFile(inFile: File, thumbnailSize: Int): Option[File] = { - if (inFile != null) { - try { - val srcImage = ImageIO.read(inFile) - if ((srcImage.getHeight > thumbnailSize) || (srcImage.getWidth > thumbnailSize)) { - val scaledImage: BufferedImage = Scalr.resize(srcImage, thumbnailSize) - val thumbFile = getThumbnailFileName(inFile) - val outFile = FileUtils.createFile(thumbFile) - ImageIO.write(scaledImage, "png", outFile) - Some(outFile) - } else None - } catch { - case ex: Exception => - logger.error("Please Provide Valid File Url!", ex) - None - } - } else None - } - - def getThumbnailFileName(input: File): String = { - val outputFileName = input.getName.replaceAll("\\.", "\\.thumb\\.") - val outputFolder = input.getParent - s"$outputFolder/$outputFileName" - } - -} diff --git a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/YouTubeUtil.scala b/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/YouTubeUtil.scala deleted file mode 100644 index 597dacc69..000000000 --- a/asset-enrichment/src/main/scala/org/sunbird/job/assetenricment/util/YouTubeUtil.scala +++ /dev/null @@ -1,109 +0,0 @@ -package org.sunbird.job.assetenricment.util - -import com.google.api.client.googleapis.json.GoogleJsonResponseException -import com.google.api.client.http.javanet.NetHttpTransport -import com.google.api.client.http.{HttpRequest, HttpRequestInitializer} -import com.google.api.client.json.jackson2.JacksonFactory -import com.google.api.services.youtube.YouTube -import com.google.api.services.youtube.model.Video -import org.slf4j.LoggerFactory -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig - -import java.util.regex.{Matcher, Pattern} -import scala.collection.JavaConverters._ - -class YouTubeUtil(config: AssetEnrichmentConfig) { - - private[this] val logger = LoggerFactory.getLogger(classOf[YouTubeUtil]) - private val APP_NAME = config.youtubeAppName - private val API_KEY = config.getString("content_youtube_apikey", "") - private val videoIdRegex = config.videoIdRegex.asScala.toList - private var limitExceeded = false - private val youtube: YouTube = initializeYoutube() - - def initializeYoutube(): YouTube = { - new YouTube.Builder(new NetHttpTransport, new JacksonFactory, new HttpRequestInitializer() { - override def initialize(request: HttpRequest): Unit = { - } - }).setApplicationName(APP_NAME).build() - } - - def getVideoInfo(videoUrl: String, apiParams: String, metadata: List[String]): Map[String, AnyRef] = { - val videoId = getIdFromUrl(videoUrl) - videoId match { - case Some(id: String) => - val video: Video = getVideo(id, apiParams) - if (null != video) { - metadata.flatMap(str => { - val res = getResult(str, video) - if (res.nonEmpty) Map[String, AnyRef](str -> res) else Map[String, AnyRef]() - }).toMap[String, AnyRef] - } else Map[String, AnyRef]() - case _ => - logger.error(s"Failed to get Id from the VideoUrl : ${videoUrl}") - Map[String, AnyRef]() - } - } - - def getIdFromUrl(url: String): Option[String] = { - val videoLink = getVideoLink(url) - val result = videoIdRegex.map(p => Pattern.compile(p)) - .map(compiledPattern => compiledPattern.matcher(videoLink)) - .find(f => f.find()) - .map((matcher: Matcher) => matcher.group(1)).orElse(null) - result - } - - private def getVideoLink(url: String): String = { - val youTubeUrlRegEx = "^(https?)?(://)?(www.)?(m.)?((youtube.com)|(youtu.be))/" - val compiledPattern = Pattern.compile(youTubeUrlRegEx) - val matcher = compiledPattern.matcher(url) - if (matcher.find) url.replace(matcher.group, "") else url - } - - def getVideo(videoId: String, params: String): Video = { - if (limitExceeded) throw new Exception(s"Unable to Check License for videoId = ${videoId}. Please Try Again After Sometime!") - try { - val videosListByIdRequest = youtube.videos.list(params) - videosListByIdRequest.setKey(API_KEY) - videosListByIdRequest.setId(videoId) - val response = videosListByIdRequest.execute - val items = response.getItems - if (items == null || items.isEmpty) null else items.get(0) - } catch { - case ex: GoogleJsonResponseException => - val error = ex.getDetails.getErrors.get(0) - val reason = error.get("reason").asInstanceOf[String] - limitExceeded = true - logger.error(s"Youtube API Limit Exceeded while processing for videoId = ${videoId}. Reason is: ${reason}", ex) - throw new Exception(s"Unable to Check License for videoId = ${videoId}. Please Try Again After Sometime!", ex) - case e: Exception => - logger.error("Error Occurred While Calling Youtube API. Error Details : ", e) - throw e - } - } - - private def getResult(str: String, video: Video): String = { - str.toLowerCase match { - case "license" => - video.getStatus.getLicense - case "thumbnail" => - video.getSnippet.getThumbnails.getMedium.getUrl - case "duration" => - computeVideoDuration(video.getContentDetails.getDuration) - case _ => "" - } - } - - def computeVideoDuration(videoDuration: String): String = { - val youtubeDuration = videoDuration.replaceAll("PT|S", "").replaceAll("H|M", ":") - val values = youtubeDuration.split(":") - values.length match { - case 1 => values(0) - case 2 => String.valueOf((values(0).toInt * 60) + (values(1).toInt * 1)) - case 3 => String.valueOf((values(0).toInt * 3600) + (values(1).toInt * 60) + (values(2).toInt * 1)) - case _ => "" - } - } - -} diff --git a/asset-enrichment/src/test/resources/logback-test.xml b/asset-enrichment/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/asset-enrichment/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/asset-enrichment/src/test/resources/test.conf b/asset-enrichment/src/test/resources/test.conf deleted file mode 100644 index 5d61f109f..000000000 --- a/asset-enrichment/src/test/resources/test.conf +++ /dev/null @@ -1,35 +0,0 @@ -include "base-test.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.learning.job.request" - groupId = "sunbirddev-asset-enrichment-group" - video_stream.topic = "sunbirddev.content.postpublish.request" -} - -task { - consumer.parallelism = 1 - router.parallelism = 1 - videoEnrichment.parallelism = 1 - imageEnrichment.parallelism = 1 -} - -content { - stream { - enabled = true - mimeType = ["video/mp4", "video/webm"] - } - youtube { - applicationName = "fetch-youtube-license" - regexPattern = ["\\?vi?=([^&]*)", "watch\\?.*v=([^&]*)", "(?:embed|vi?)/([^/?]*)", "^([A-Za-z0-9\\-\\_]*)"] - } - upload.context.driven = true -} - -thumbnail.max { - sample = 5 - size.pixel = 150 -} \ No newline at end of file diff --git a/asset-enrichment/src/test/resources/test_files/test_audio.mp3 b/asset-enrichment/src/test/resources/test_files/test_audio.mp3 deleted file mode 100644 index 6720cbfc0..000000000 Binary files a/asset-enrichment/src/test/resources/test_files/test_audio.mp3 and /dev/null differ diff --git a/asset-enrichment/src/test/resources/test_files/test_other.pdf b/asset-enrichment/src/test/resources/test_files/test_other.pdf deleted file mode 100644 index dbf091df9..000000000 Binary files a/asset-enrichment/src/test/resources/test_files/test_other.pdf and /dev/null differ diff --git a/asset-enrichment/src/test/resources/test_files/test_video.mp4 b/asset-enrichment/src/test/resources/test_files/test_video.mp4 deleted file mode 100644 index 1fc478842..000000000 Binary files a/asset-enrichment/src/test/resources/test_files/test_video.mp4 and /dev/null differ diff --git a/asset-enrichment/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/asset-enrichment/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index 441006a24..000000000 --- a/asset-enrichment/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,20 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val IMAGE_ASSET: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1615443994411,"mid":"LP.1615443994411.c94524f2-0d45-4da7-93d8-6d4059f6f999","actor":{"id":"Asset Enrichment Samza Job","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":"b00bc992ef25f1a9a8d63291e20efc8d","env":"dev"},"object":{"ver":"1615443994322","id":"do_113233716442578944194"},"edata":{"action":"assetenrichment","iteration":1,"mediaType":"image","status":"Processing","objectType":"Asset"}} - |""".stripMargin - - val VIDEO_MP4_ASSET: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1615444029534,"mid":"LP.1615444029534.a4588b0e-50e9-4af7-8a73-2efa8efc68aa","actor":{"id":"Asset Enrichment Samza Job","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":"b00bc992ef25f1a9a8d63291e20efc8d","env":"dev"},"object":{"ver":"1551877993917","id":"do_1127129845261680641588"},"edata":{"action":"assetenrichment","iteration":1,"mediaType":"video","status":"Processing","objectType":"Asset"}} - |""".stripMargin - - val VIDEO_YOUTUBE_ASSET: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1615444029534,"mid":"LP.1615444029534.a4588b0e-50e9-4af7-8a73-2efa8efc68aa","actor":{"id":"Asset Enrichment Samza Job","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":"b00bc992ef25f1a9a8d63291e20efc8d","env":"dev"},"object":{"ver":"1551877993917","id":"do_1127129845261680641599"},"edata":{"action":"assetenrichment","iteration":1,"mediaType":"video","status":"Processing","objectType":"Asset"}} - |""".stripMargin - -} diff --git a/asset-enrichment/src/test/scala/org/sunbird/job/spec/AssetEnrichmentTaskTestSpec.scala b/asset-enrichment/src/test/scala/org/sunbird/job/spec/AssetEnrichmentTaskTestSpec.scala deleted file mode 100644 index 973f48f31..000000000 --- a/asset-enrichment/src/test/scala/org/sunbird/job/spec/AssetEnrichmentTaskTestSpec.scala +++ /dev/null @@ -1,208 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.doNothing -import org.sunbird.job.assetenricment.domain.Event -import org.sunbird.job.assetenricment.functions.{ImageEnrichmentFunction, VideoEnrichmentFunction} -import org.sunbird.job.assetenricment.models.Asset -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.assetenricment.util.{AssetFileUtils, ImageResizerUtil, YouTubeUtil} -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.util.{CloudStorageUtil, FileUtils, JSONUtil, Neo4JUtil, ScalaJsonUtil} -import org.sunbird.spec.BaseTestSpec - -import java.io.File -import java.util - -class AssetEnrichmentTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig = new AssetEnrichmentConfig(config) - val definitionUtil = new DefinitionCache - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val cloudUtil: CloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val youTubeUtil: YouTubeUtil = new YouTubeUtil(jobConfig) - - override protected def beforeAll(): Unit = { - super.beforeAll() - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - super.afterAll() - flinkCluster.after() - } - - "enrichImage()" should " enrich the image for the asset " in { - doNothing().when(mockNeo4JUtil).updateNode(anyString(), any[Map[String, AnyRef]]()) - - val metadata = getMetadataForImageAsset - val asset = getAsset(EventFixture.IMAGE_ASSET, metadata) - new ImageEnrichmentFunction(jobConfig).enrichImage(asset)(jobConfig, definitionUtil, cloudUtil, mockNeo4JUtil) - val variants = ScalaJsonUtil.deserialize[Map[String, String]](asset.get("variants", "").asInstanceOf[String]) - variants.size should be(3) - variants.keys should contain allOf("high", "medium", "low") - variants.getOrElse("high", "") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113233717480390656195/artifact/bitcoin-4_1545114579639.jpg") - asset.get("status", "").asInstanceOf[String] should be("Live") - } - - "videoEnrichment" should " enrich the video for the mp4 asset " in { - doNothing().when(mockNeo4JUtil).updateNode(anyString(), any[Map[String, AnyRef]]()) - - val metadata = getMetadataForMp4VideoAsset - val asset = getAsset(EventFixture.VIDEO_MP4_ASSET, metadata) - new VideoEnrichmentFunction(jobConfig).enrichVideo(asset)(jobConfig, youTubeUtil, cloudUtil, mockNeo4JUtil) - asset.get("status", "").asInstanceOf[String] should be("Live") - } - - "videoEnrichment" should " enrich the video for the youtube asset " in { - doNothing().when(mockNeo4JUtil).updateNode(anyString(), any[Map[String, AnyRef]]()) - - val metadata = getMetadataForYouTubeVideoAsset - val asset = getAsset(EventFixture.VIDEO_YOUTUBE_ASSET, metadata) - new VideoEnrichmentFunction(jobConfig).enrichVideo(asset)(jobConfig, youTubeUtil, cloudUtil, mockNeo4JUtil) - asset.get("thumbnail", "").asInstanceOf[String] should be("https://i.ytimg.com/vi/-SgZ3Enpau8/mqdefault.jpg") - asset.get("status", "").asInstanceOf[String] should be("Live") - asset.get("duration", "0").asInstanceOf[String] should be("274") - } - - "validateForArtifactUrl" should "validate for content upload context driven" in { - val metadata = Map[String, AnyRef]("cloudStorageKey" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg", - "s3Key" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg", - "artifactBasePath" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev", - "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg") - val asset = getAsset(EventFixture.IMAGE_ASSET, metadata) - val validate = asset.validate(true) - validate should be(true) - } - - "ImageEnrichmentHelper" should "return Exception if the image url is wrong" in { - val metadata = getMetadataForImageAsset - val asset = getAsset(EventFixture.IMAGE_ASSET, metadata) - asset.put("artifactUrl", "https://unknownurl123.com") - assertThrows[Exception] { - new ImageEnrichmentFunction(jobConfig).enrichImage(asset)(jobConfig, definitionUtil, cloudUtil, mockNeo4JUtil) - } - } - - "VideoEnrichmentHelper" should "return Exception if the video url is wrong" in { - val metadata = getMetadataForMp4VideoAsset - val asset = getAsset(EventFixture.VIDEO_MP4_ASSET, metadata) - asset.put("artifactUrl", "https://unknownurl1234.com") - assertThrows[Exception] { - new VideoEnrichmentFunction(jobConfig).enrichVideo(asset)(jobConfig, youTubeUtil, cloudUtil, mockNeo4JUtil) - } - } - - "VideoEnrichmentHelper" should "return Exception if the video url is empty" in { - val metadata = getMetadataForMp4VideoAsset - val asset = getAsset(EventFixture.VIDEO_MP4_ASSET, metadata) - asset.put("artifactUrl", "") - assertThrows[Exception] { - new VideoEnrichmentFunction(jobConfig).enrichVideo(asset)(jobConfig, youTubeUtil, cloudUtil, mockNeo4JUtil) - } - } - - "VideoEnrichmentFunction" should "return Exception if the metadata is not found" in { - val metadata = getMetadataForMp4VideoAsset - val asset = getAsset(EventFixture.VIDEO_MP4_ASSET, metadata) - asset.put("artifactUrl", "") - assertThrows[Exception] { - new VideoEnrichmentFunction(jobConfig).getMetadata(asset.identifier)(mockNeo4JUtil) - } - } - - "ImageEnrichmentFunction" should "return Exception if the metadata is not found" in { - val metadata = getMetadataForImageAsset - val asset = getAsset(EventFixture.IMAGE_ASSET, metadata) - asset.put("artifactUrl", "") - assertThrows[Exception] { - new ImageEnrichmentFunction(jobConfig).getMetadata(asset.identifier)(mockNeo4JUtil) - } - } - - "getVideoInfo" should "throw exception if video Url is empty" in { - val metadata = getMetadataForYouTubeVideoAsset - val asset = getAsset(EventFixture.VIDEO_YOUTUBE_ASSET, metadata) - assertThrows[Exception] { - new VideoEnrichmentFunction(jobConfig).processYoutubeVideo(asset, "")(youTubeUtil) - } - } - - "event.validate" should " validate the event " in { - val event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.IMAGE_ASSET), 0, 10) - val message = event.validate(jobConfig.maxIterationCount) - message.isEmpty should be(true) - } - - "getStreamingEvent" should "return event string for streaming for video assset" in { - val metadata = getMetadataForMp4VideoAsset - val asset = getAsset(EventFixture.VIDEO_MP4_ASSET, metadata) - val eventMsg = new VideoEnrichmentFunction(jobConfig).getStreamingEvent(asset)(jobConfig) - eventMsg.isEmpty should be(false) - } - - "upload" should " throw exception for incorrect file in ImageEnrichment " in { - assertThrows[Exception] { - new ImageEnrichmentFunction(jobConfig).upload(null, "do_123")(cloudUtil) - } - } - - "upload" should " throw exception for incorrect file in VideoEnrichment " in { - assertThrows[Exception] { - new VideoEnrichmentFunction(jobConfig).upload(null, "do_123")(cloudUtil) - } - } - - "ImageResizerUtil" should " replace the provided files " in { - val imageUrl = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113233717480390656195/artifact/bitcoin-4_1545114579639.jpg" - try { - val file = FileUtils.copyURLToFile("do_113233717480390656195", imageUrl, imageUrl.substring(imageUrl.lastIndexOf("/") + 1)) - val newFile = new File("/tmp/do_113233717480390656195/1617194149349_temp/bitcoin-4_1545114579640.jpg") - val result = new ImageResizerUtil().replace(file.get, newFile) - result.getAbsolutePath should endWith("bitcoin-4_1545114579640.jpg") - } finally { - FileUtils.deleteDirectory(new File(s"/tmp/do_113233717480390656195")) - } - } - - def getAsset(event: String, metadata: Map[String, AnyRef]): Asset = { - val eventMap = JSONUtil.deserialize[util.Map[String, Any]](event) - val asset = Asset(eventMap) - asset.putAll(metadata) - asset - } - - def getMetadataForImageAsset: Map[String, AnyRef] = { - val metadata = """{"ownershipType": ["createdBy"], "code": "org.ekstep0.07321483804683715", "prevStatus": "Processing", "downloadUrl": "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113233717480390656195/artifact/bitcoin-4_1545114579639.jpg", "channel": "b00bc992ef25f1a9a8d63291e20efc8d", "language": ["English"], "mimeType": "image/jpeg","idealScreenSize": "normal", "createdOn": "2021-03-11T06:27:08.370+0000", "primaryCategory": "Certificate Template", "contentDisposition": "inline", "artifactUrl": "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113233717480390656195/artifact/bitcoin-4_1545114579639.jpg", "contentEncoding": "identity", "lastUpdatedOn": "2021-03-11T06:27:10.254+0000", "contentType": "Asset", "dialcodeRequired": "No", "audience": ["Student"], "creator": "Reviewer User", "lastStatusChangedOn": "2021-03-11T06:27:10.237+0000", "visibility": "Default", "os": ["All"], "IL_SYS_NODE_TYPE": "DATA_NODE", "cloudStorageKey": "content/do_113233717480390656195/artifact/bitcoin-4_1545114579639.jpg", "mediaType": "image", "osId": "org.ekstep.quiz.app", "version": 2, "versionKey": "1615444030254", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "s3Key": "content/do_113233717480390656195/artifact/bitcoin-4_1545114579639.jpg", "framework": "NCF", "size": 86402.0, "createdBy": "95e4942d-cbe8-477d-aebd-ad8e6de4bfc8", "compatibilityLevel": 1, "IL_FUNC_OBJECT_TYPE": "Asset", "name": "bitcoin-4 1545114579639", "IL_UNIQUE_ID": "do_113233717480390656195", "status": "Live"}""" - ScalaJsonUtil.deserialize[Map[String, AnyRef]](metadata) - } - - def getMetadataForMp4VideoAsset: Map[String, AnyRef] = { - val metadata = """{"artifactUrl": "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/kp_ft_1563562323128/artifact/sample_1563562323191.mp4", "ownershipType": ["createdBy"], "code": "Test_Asset_15_MB", "apoc_json": "{\"batch\": true}", "channel": "in.ekstep", "organisation": ["Sunbird", "QA ORG"], "language": ["English"], "mimeType": "video/mp4", "media": "video", "idealScreenSize": "normal", "createdOn": "2019-03-06T13:13:13.917+0000", "apoc_text": "APOC", "primaryCategory": "Asset", "appId": "local.sunbird.portal", "contentDisposition": "inline", "contentEncoding": "identity", "lastUpdatedOn": "2019-03-06T13:13:13.917+0000", "contentType": "Asset", "dialcodeRequired": "No", "apoc_num": 1, "audience": ["Learner"], "creator": "Creation", "createdFor": ["ORG_001", "0123653943740170242"], "visibility": "Default", "os": ["All"], "IL_SYS_NODE_TYPE": "DATA_NODE", "consumerId": "9393568c-3a56-47dd-a9a3-34da3c821638", "mediaType": "content", "osId": "org.ekstep.quiz.app", "versionKey": "1551877993917", "idealScreenDensity": "hdpi", "framework": "NCFCOPY", "createdBy": "874ed8a5-782e-4f6c-8f36-e0288455901e", "compatibilityLevel": 1.0, "IL_FUNC_OBJECT_TYPE": "Asset", "name": "Test_Asset_15_MB", "IL_UNIQUE_ID": "do_1127129845261680641588", "status": "Draft"}""" - ScalaJsonUtil.deserialize[Map[String, AnyRef]](metadata) - } - - def getMetadataForYouTubeVideoAsset: Map[String, AnyRef] = { - val metadata = """{"artifactUrl": "https://www.youtube.com/watch?v=-SgZ3Enpau8", "ownershipType": ["createdBy"], "code": "Test_Asset_15_MB", "apoc_json": "{\"batch\": true}", "channel": "in.ekstep", "organisation": ["Sunbird", "QA ORG"], "language": ["English"], "mimeType": "video/x-youtube", "media": "video", "idealScreenSize": "normal", "createdOn": "2019-03-06T13:13:13.917+0000", "apoc_text": "APOC", "primaryCategory": "Asset", "appId": "local.sunbird.portal", "contentDisposition": "inline", "contentEncoding": "identity", "lastUpdatedOn": "2019-03-06T13:13:13.917+0000", "contentType": "Asset", "dialcodeRequired": "No", "apoc_num": 1, "audience": ["Learner"], "creator": "Creation", "createdFor": ["ORG_001", "0123653943740170242"], "visibility": "Default", "os": ["All"], "IL_SYS_NODE_TYPE": "DATA_NODE", "consumerId": "9393568c-3a56-47dd-a9a3-34da3c821638", "mediaType": "content", "osId": "org.ekstep.quiz.app", "versionKey": "1551877993917", "idealScreenDensity": "hdpi", "framework": "NCFCOPY", "createdBy": "874ed8a5-782e-4f6c-8f36-e0288455901e", "compatibilityLevel": 1.0, "IL_FUNC_OBJECT_TYPE": "Asset", "name": "Test_Asset_15_MB", "IL_UNIQUE_ID": "do_1127129845261680641599", "status": "Draft"}""" - ScalaJsonUtil.deserialize[Map[String, AnyRef]](metadata) - } - -} \ No newline at end of file diff --git a/asset-enrichment/src/test/scala/org/sunbird/job/spec/AssetFileUtilsSpec.scala b/asset-enrichment/src/test/scala/org/sunbird/job/spec/AssetFileUtilsSpec.scala deleted file mode 100644 index 9813f0629..000000000 --- a/asset-enrichment/src/test/scala/org/sunbird/job/spec/AssetFileUtilsSpec.scala +++ /dev/null @@ -1,34 +0,0 @@ -package org.sunbird.job.spec - -import org.sunbird.job.assetenricment.util.AssetFileUtils -import org.sunbird.spec.BaseTestSpec - -import java.io.File - -class AssetFileUtilsSpec extends BaseTestSpec { - - "getFileType" should "return file type for the provided audio file" in { - val file = new File(getClass.getClassLoader.getResource("test_files/test_audio.mp3").getFile) - val fileType = AssetFileUtils.getFileType(file) - assert(fileType == "Audio") - } - - "getFileType" should "return file type for the provided video file" in { - val file = new File(getClass.getClassLoader.getResource("test_files/test_video.mp4").getFile) - val fileType = AssetFileUtils.getFileType(file) - assert(fileType == "Video") - } - - "getFileType" should "return file type for the provided pdf file" in { - val file = new File(getClass.getClassLoader.getResource("test_files/test_other.pdf").getFile) - val fileType = AssetFileUtils.getFileType(file) - assert(fileType == "Other") - } - - "getFileType" should "return file type for the provided directory" in { - val file = new File(getClass.getClassLoader.getResource("test_files").getFile) - val fileType = AssetFileUtils.getFileType(file) - assert(fileType == "Directory") - } - -} diff --git a/asset-enrichment/src/test/scala/org/sunbird/job/spec/OptimizerHelperSpec.scala b/asset-enrichment/src/test/scala/org/sunbird/job/spec/OptimizerHelperSpec.scala deleted file mode 100644 index 4968ba4ae..000000000 --- a/asset-enrichment/src/test/scala/org/sunbird/job/spec/OptimizerHelperSpec.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.mockito.ArgumentMatchers.{anyBoolean, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.{doNothing, when} -import org.sunbird.job.assetenricment.functions.ImageEnrichmentFunction -import org.sunbird.job.assetenricment.models.Asset -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.util.{CloudStorageUtil, JSONUtil} -import org.sunbird.spec.BaseTestSpec - -import java.util - -class OptimizerHelperSpec extends BaseTestSpec { - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig = new AssetEnrichmentConfig(config) - implicit val mockCloudUtil: CloudStorageUtil = mock[CloudStorageUtil](Mockito.withSettings().serializable()) - - "replaceArtifactUrl" should " update the provided asset properties " in { - doNothing().when(mockCloudUtil).copyObjectsByPrefix(anyString(), anyString(), anyBoolean()) - val asset = getAsset(EventFixture.IMAGE_ASSET, getMetadata) - new ImageEnrichmentFunction(jobConfig).replaceArtifactUrl(asset)(mockCloudUtil) - asset.get("artifactUrl", "") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg") - asset.get("downloadUrl", "") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg") - asset.get("cloudStorageKey", "") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg") - asset.get("s3Key", "") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg") - } - - "replaceArtifactUrl" should " throw exception for invalid asset " in { - when(mockCloudUtil.copyObjectsByPrefix(anyString(), anyString(), anyBoolean())).thenThrow(new IllegalArgumentException("Failed to copy the artifact.")) - val asset = getAsset(EventFixture.IMAGE_ASSET, getMetadata) - assertThrows[Exception] { - new ImageEnrichmentFunction(jobConfig).replaceArtifactUrl(asset)(mockCloudUtil) - } - } - - def getAsset(event: String, metadata: Map[String, AnyRef]): Asset = { - val eventMap = JSONUtil.deserialize[util.Map[String, Any]](event) - val asset = Asset(eventMap) - asset.putAll(metadata) - asset - } - - def getMetadata: Map[String, AnyRef] = { - val metadata = Map[String, AnyRef]("cloudStorageKey" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/tmp/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg", - "s3Key" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/tmp/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg", - "artifactBasePath" -> "tmp", - "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/tmp/content/do_1132316405761064961124/artifact/0_jmrpnxe-djmth37l_.jpg") - metadata - } -} diff --git a/asset-enrichment/src/test/scala/org/sunbird/job/spec/ThumbnailSpec.scala b/asset-enrichment/src/test/scala/org/sunbird/job/spec/ThumbnailSpec.scala deleted file mode 100644 index d7419ed64..000000000 --- a/asset-enrichment/src/test/scala/org/sunbird/job/spec/ThumbnailSpec.scala +++ /dev/null @@ -1,48 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.assetenricment.util.ThumbnailUtil -import org.sunbird.job.util.FileUtils -import org.sunbird.spec.BaseTestSpec - -import java.io.File - -class ThubnailUtilSpec extends BaseTestSpec { - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig = new AssetEnrichmentConfig(config) - - "ThumbnailUtil.generateOutFile" should " return null for no file" in { - val file = new ThumbnailUtilTest().generateOutFile(null, 150) - file should be(None) - } - - "ThumbnailUtil.generateOutFile" should " return None for the file" in { - val imageUrl = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113233717480390656195/artifact/bitcoin-4_1545114579639.jpg" - try { - val file = FileUtils.copyURLToFile("do_113233717480390656195", imageUrl, imageUrl.substring(imageUrl.lastIndexOf("/") + 1)) - val newFile = new ThumbnailUtilTest().generateOutFile(file.get, 15000) - newFile should be(None) - } finally { - FileUtils.deleteDirectory(new File(s"/tmp/do_113233717480390656195")) - } - } - - "ThumbnailUtil.generateOutFile" should " return None for file" in { - val contentId = "do_1127129845261680641588" - val originalURL = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/kp_ft_1563562323128/artifact/sample_1563562323191.mp4" - try { - val originalFile = FileUtils.copyURLToFile(contentId, originalURL, originalURL.substring(originalURL.lastIndexOf("/") + 1, originalURL.length)) - val result = new ThumbnailUtilTest().generateOutFile(originalFile.get, 150) - result should be(None) - } finally { - FileUtils.deleteDirectory(new File(s"/tmp/$contentId")) - } - } -} - - -class ThumbnailUtilTest extends ThumbnailUtil { - -} \ No newline at end of file diff --git a/asset-enrichment/src/test/scala/org/sunbird/job/spec/YouTubeUtilSpec.scala b/asset-enrichment/src/test/scala/org/sunbird/job/spec/YouTubeUtilSpec.scala deleted file mode 100644 index 8045e0c91..000000000 --- a/asset-enrichment/src/test/scala/org/sunbird/job/spec/YouTubeUtilSpec.scala +++ /dev/null @@ -1,60 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.sunbird.job.assetenricment.task.AssetEnrichmentConfig -import org.sunbird.job.assetenricment.util.YouTubeUtil -import org.sunbird.spec.BaseTestSpec - -class YouTubeUtilSpec extends BaseTestSpec { - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig = new AssetEnrichmentConfig(config) - - "getIdFromUrl" should "return the id of the video for the provided Youtube URL" in { - val result = new YouTubeUtil(jobConfig).getIdFromUrl("https://www.youtube.com/watch?v=-SgZ3Enpau8") - result.getOrElse("") should be("-SgZ3Enpau8") - } - - "computeVideoDuration" should "return the duration of the medium video for the " in { - val result = new YouTubeUtil(jobConfig).computeVideoDuration("PT4M33S") - result should be("273") - } - - "computeVideoDuration" should "return the duration of the long video for the " in { - val result = new YouTubeUtil(jobConfig).computeVideoDuration("PT1H4M33S") - result should be("3873") - } - - "computeVideoDuration" should "return the duration of the short video for the " in { - val result = new YouTubeUtil(jobConfig).computeVideoDuration("PT33S") - result should be("33") - } - - "computeVideoDuration" should "return the empty for wrong video length2 " in { - val result = new YouTubeUtil(jobConfig).computeVideoDuration("PT1H4M33SPT1H4M33SPT1H4M33S") - result should be("") - } - - "getVideoInfo" should "return empty map if no url is passed " in { - val result = new YouTubeUtil(jobConfig).getVideoInfo("", "snippet,contentDetails", List[String]("thumbnail", "duration", "license")) - result.isEmpty should be(true) - } - - "getVideoInfo" should "return the empty data for incorrect Youtube URL" in { - val result = new YouTubeUtil(jobConfig).getVideoInfo("https://www.youtube.com/watch?v=-SgZ3En23sd", "snippet,contentDetails", List[String]("thumbnail", "duration", "xyz")) - result.isEmpty should be(true) - } - - "getVideoInfo" should "throw Exception for null URL" in { - assertThrows[Exception] { - new YouTubeUtil(jobConfig).getVideoInfo(null, "snippet,contentDetails", List[String]("thumbnail", "duration", "xyz")) - } - } - - "getVideo" should " throw exception for null video Id " in { - assertThrows[Exception] { - new YouTubeUtil(jobConfig).getVideo(null, null) - } - } - -} diff --git a/audit-event-generator/README.md b/audit-event-generator/README.md deleted file mode 100644 index 1521e3611..000000000 --- a/audit-event-generator/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Audit Event Generator - -Audit Event Generator job generates an audit event for every transaction in Graph updation for content data modification - -## Getting Started - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. -### Prerequisites - -1. Download flink-1.13.6-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.13.6/flink-1.13.6-bin-scala_2.12.tgz). -2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. -3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. -4. Docker installed. -5. A running yarn cluster or a kubernetes cluster. - -### Build - -mvn clean install - -## Deployment - -### Yarn - -Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. - -``` -./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /audit-event-generator/target/audit-event-generator-0.0.1.jar -``` - -### Kubernetes - -``` -# Create a single node cluster -k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 -# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. -export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" - -# Only for Mac OSX -# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl -rm /usr/local/bin/kubectl -brew link --overwrite kubernetes-cli - -# Create a configmap using the flink-configuration-configmap.yaml -kubectl create -f knowledge-platform-job/kubernetes/flink-configuration-configmap.yaml - -# Create pods for jobmanager-service, job-manager and task-manager using the yaml files -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-service.yaml -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-deployment.yaml -kubectl create -f knowledge-platform-job/kubernetes/taskmanager-deployment.yaml - -# Create a port-forwarding for accessing the job-manager UI on localhost:8081 -kubectl port-forward deployment/flink-jobmanager 8081:8081 - -# Submit the job to the Kubernetes single node cluster flink-cluster -./bin/flink run -m localhost:8081 /audit-event-generator/target/audit-event-generator-0.0.1.jar - -# Commands to delete the pods created in the cluster -kubectl delete deployment/flink-jobmanager -kubectl delete deployment/flink-taskmanager -kubectl delete service/flink-jobmanager -kubectl delete configmaps/flink-config - -# Command to stop the single-node cluster -k3d stop --name="flink-cluster" -``` diff --git a/audit-event-generator/pom.xml b/audit-event-generator/pom.xml deleted file mode 100644 index be8ba4e72..000000000 --- a/audit-event-generator/pom.xml +++ /dev/null @@ -1,236 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - audit-event-generator - 1.0.0 - jar - audit-event-generator - - Audit Event Generator Flink Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.sunbird - platform-common - 1.0-beta - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-api - - - - - org.sunbird - platform-telemetry - 1.0-beta - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.auditevent.task.AuditEventGeneratorStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - audit-event-generator-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - diff --git a/audit-event-generator/src/main/resources/audit-event-generator.conf b/audit-event-generator/src/main/resources/audit-event-generator.conf deleted file mode 100644 index f7db59171..000000000 --- a/audit-event-generator/src/main/resources/audit-event-generator.conf +++ /dev/null @@ -1,20 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - output.topic = "sunbirddev.telemetry.raw" - groupId = "sunbirddev-audit-event-generator-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - producer.parallelism = 1 - window.time = 60 -} - -schema { - basePath = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" -} - -channel.default = "org.sunbird" diff --git a/audit-event-generator/src/main/resources/log4j.properties b/audit-event-generator/src/main/resources/log4j.properties deleted file mode 100644 index 775b16009..000000000 --- a/audit-event-generator/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=audit-event-generator.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/domain/Event.scala b/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/domain/Event.scala deleted file mode 100644 index ebba0b8f4..000000000 --- a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/domain/Event.scala +++ /dev/null @@ -1,33 +0,0 @@ -package org.sunbird.job.auditevent.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val IMAGE_SUFFIX = ".img" - private val jobName = "AuditEventGenerator" - - def id: String = readOrDefault("nodeUniqueId", "") - - def operationType: String = readOrDefault("operationType", "") - - def nodeUniqueId: String = readOrDefault("nodeUniqueId", null) - - def createdOn: String = read("createdOn").get - - def channelId(channel: String): String = readOrDefault("channel", channel) - - def objectId: String = if (null != nodeUniqueId) nodeUniqueId.replaceAll(IMAGE_SUFFIX, "") else nodeUniqueId - - def objectType: String = readOrDefault[String]("objectType", null) - - def userId: String = readOrDefault[String]("userId", "") - - def transactionData: Map[String, AnyRef] = readOrDefault("transactionData", Map[String, AnyRef]()) - - def isValid: Boolean = { - StringUtils.isNotBlank(objectType) - } - -} diff --git a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/functions/AuditEventGenerator.scala b/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/functions/AuditEventGenerator.scala deleted file mode 100644 index f6b591f68..000000000 --- a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/functions/AuditEventGenerator.scala +++ /dev/null @@ -1,49 +0,0 @@ -package org.sunbird.job.auditevent.functions - -import java.util -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.auditevent.domain.Event -import org.sunbird.job.auditevent.service.AuditEventGeneratorService -import org.sunbird.job.auditevent.task.AuditEventGeneratorConfig -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.{BaseProcessFunction, Metrics} - -class AuditEventGenerator(config: AuditEventGeneratorConfig) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with AuditEventGeneratorService{ - - private[this] lazy val logger = LoggerFactory.getLogger(classOf[AuditEventGenerator]) - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount, config.emptySchemaEventCount, config.emptyPropsEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - try { - metrics.incCounter(config.totalEventsCount) - if(event.isValid) { - logger.info("valid event::"+event.nodeUniqueId) - processEvent(event, context, metrics)(config) - } else metrics.incCounter(config.skippedEventCount) - } catch { - case ex: Exception => - metrics.incCounter(config.failedEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } -} diff --git a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/service/AuditEventGeneratorService.scala b/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/service/AuditEventGeneratorService.scala deleted file mode 100644 index aab6acaa7..000000000 --- a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/service/AuditEventGeneratorService.scala +++ /dev/null @@ -1,201 +0,0 @@ -package org.sunbird.job.auditevent.service - -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.auditevent.task.AuditEventGeneratorConfig -import org.sunbird.job.util.JSONUtil -import org.sunbird.telemetry.TelemetryGenerator -import org.sunbird.telemetry.TelemetryParams -import org.sunbird.job.auditevent.domain.Event -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import com.google.gson.Gson -import org.sunbird.job.exception.InvalidEventException - -import java.util -import java.text.SimpleDateFormat - -trait AuditEventGeneratorService { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[AuditEventGeneratorService]) - private val OBJECT_TYPE_IMAGE_SUFFIX = "Image" - private val SKIP_AUDIT = """{"object": {"type":null}}""" - private lazy val definitionCache = new DefinitionCache - private lazy val gson = new Gson - - private val systemPropsList = List("IL_SYS_NODE_TYPE", "IL_FUNC_OBJECT_TYPE", "IL_UNIQUE_ID", "IL_TAG_NAME", "IL_ATTRIBUTE_NAME", "IL_INDEXABLE_METADATA_KEY", "IL_NON_INDEXABLE_METADATA_KEY", - "IL_IN_RELATIONS_KEY", "IL_OUT_RELATIONS_KEY", "IL_REQUIRED_PROPERTIES", "IL_SYSTEM_TAGS_KEY", "IL_SEQUENCE_INDEX", "SYS_INTERNAL_LAST_UPDATED_ON", "lastUpdatedOn", "versionKey", "lastStatusChangedOn") - - private def getContext(channelId: String, env: String): Map[String, String] = { - val context = Map( - TelemetryParams.ACTOR.name -> "org.ekstep.learning.platform", - TelemetryParams.CHANNEL.name -> channelId, - TelemetryParams.ENV.name -> env - ) - context - } - - @throws(classOf[InvalidEventException]) - def processEvent(message: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics)(implicit config: AuditEventGeneratorConfig): Unit = { - logger.info("AUDIT Event::" + JSONUtil.serialize(message)) - logger.info("Input Message Received for : [" + message.nodeUniqueId + "], Txn Event createdOn:" + message.createdOn + ", Operation Type:" + message.operationType) - try { - val (auditEventStr, objectType) = getAuditMessage(message)(config, metrics) - if (StringUtils.isNotBlank(objectType)) { - context.output(config.auditOutputTag, auditEventStr) - logger.info("Telemetry Audit Message Successfully Sent for : " + message.objectId + " :: event ::" + auditEventStr) - metrics.incCounter(config.successEventCount) - } - else { - logger.info("Skipped event as the objectype is not available, event =" + auditEventStr) - metrics.incCounter(config.emptyPropsEventCount) - } - } catch { - case e: Exception => - logger.error("Failed to process message :: " + JSONUtil.serialize(message), e) - throw e - } - } - - def getDefinition(objectType: String)(implicit config: AuditEventGeneratorConfig, metrics: Metrics): ObjectDefinition = { - try { - definitionCache.getDefinition(objectType, config.configVersion, config.basePath) - } catch { - case ex: Exception => { - metrics.incCounter(config.emptySchemaEventCount) - new ObjectDefinition(objectType, config.configVersion, Map[String, AnyRef](), Map[String, AnyRef]()) - } - } - } - - - def getAuditMessage(message: Event)(implicit config: AuditEventGeneratorConfig, metrics: Metrics): (String, String) = { - var auditMap: String = null - var objectType = message.objectType - val env = if (null != objectType) objectType.toLowerCase.replace("image", "") else "system" - - val definitionNode: ObjectDefinition = getDefinition(objectType) - - val propertyMap = message.transactionData("properties").asInstanceOf[Map[String, AnyRef]] - val statusMap = propertyMap.getOrElse("status", null).asInstanceOf[Map[String, AnyRef]] - val lastStatusChangedOn = propertyMap.getOrElse("lastStatusChangedOn", null).asInstanceOf[Map[String, AnyRef]] - val addedRelations = message.transactionData.getOrElse("addedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - val removedRelations = message.transactionData.getOrElse("removedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - - var pkgVersion = "" - var prevStatus = "" - var currStatus = "" - var duration = "" - val pkgVerMap = propertyMap.getOrElse("pkgVersion", null).asInstanceOf[Map[String, AnyRef]] - if (null != pkgVerMap) pkgVersion = s"${pkgVerMap.get("nv")}" - - if (null != statusMap) { - prevStatus = statusMap.getOrElse("ov", null).asInstanceOf[String] - currStatus = statusMap.getOrElse("nv", null).asInstanceOf[String] - // Compute Duration for Status Change - if (StringUtils.isNotBlank(currStatus) && StringUtils.isNotBlank(prevStatus) && null != lastStatusChangedOn) { - var ov = lastStatusChangedOn.getOrElse("ov", null).asInstanceOf[String] - val nv = lastStatusChangedOn.getOrElse("nv", null).asInstanceOf[String] - if (null == ov) ov = propertyMap.getOrElse("lastUpdatedOn", null).asInstanceOf[Map[String, AnyRef]].getOrElse("ov", null).asInstanceOf[String] - if (null != ov && null != nv) duration = String.valueOf(computeDuration(ov, nv)) - } - } - - var props: List[String] = propertyMap.keys.toList - props ++= getRelationProps(addedRelations, definitionNode) - props ++= getRelationProps(removedRelations, definitionNode) - val propsExceptSystemProps = props.filter(prop => !systemPropsList.contains(prop)) - val cdata = getCData(addedRelations, removedRelations, propertyMap) - - var context: Map[String, String] = getContext(message.channelId(config.defaultChannel), env) - - objectType = if (null != objectType) objectType.replaceAll(OBJECT_TYPE_IMAGE_SUFFIX, "") else objectType - context ++= Map("objectId" -> message.objectId, "objectType" -> objectType) - - if (StringUtils.isNotBlank(duration)) context ++= Map("duration" -> duration) - if (StringUtils.isNotBlank(pkgVersion)) context ++= Map("pkgVersion" -> pkgVersion) - if (StringUtils.isNotBlank(message.userId)) context ++= Map(TelemetryParams.ACTOR.name -> message.userId) - - if (propsExceptSystemProps.nonEmpty) { - val cdataList = gson.fromJson(JSONUtil.serialize(cdata), classOf[java.util.List[java.util.Map[String, Object]]]) - - TelemetryGenerator.setComponent("audit-event-generator") - auditMap = TelemetryGenerator.audit( - JSONUtil.deserialize[util.Map[String, String]](JSONUtil.serialize(context)), - JSONUtil.deserialize[util.List[String]](JSONUtil.serialize(propsExceptSystemProps)), - currStatus, - prevStatus, - cdataList) - logger.info("Audit Message for Content Id [" + message.objectId + "] : " + auditMap); - - (auditMap, message.objectType) - } - else { - logger.info("Skipping Audit log as props is null or empty") - (SKIP_AUDIT, "") - } - } - - /** - * @param addedRelations - * @param removedRelations - * @param propertyMap - * @return - */ - private def getCData(addedRelations: List[Map[String, AnyRef]], removedRelations: List[Map[String, AnyRef]], propertyMap: Map[String, AnyRef]): List[Map[String, AnyRef]] = { - var cdata = List[Map[String, AnyRef]]() - if (null != propertyMap && propertyMap.nonEmpty && propertyMap.contains("dialcodes")) { - val dialcodeMap = propertyMap("dialcodes").asInstanceOf[Map[String, AnyRef]] - val dialcodes = dialcodeMap("nv").asInstanceOf[List[String]] - if (null != dialcodes) { - cdata :+= Map[String, AnyRef]("id" -> dialcodes, "type" -> "DialCode") - } - } - if (null != addedRelations && addedRelations.nonEmpty) cdata ++= prepareCMap(addedRelations) - if (null != removedRelations && removedRelations.nonEmpty) cdata ++= prepareCMap(removedRelations) - cdata - } - - /** - * @param relations - */ - private def prepareCMap(relations: List[Map[String, AnyRef]]): List[Map[String, AnyRef]] = { - relations.map(relation => { - Map[String, AnyRef]("id" -> relation("id"), "type" -> relation("type")) - }) - } - - /** - * - * @param relations - */ - private def getRelationProps(relations: List[Map[String, AnyRef]], objectDefinition: ObjectDefinition)(implicit config: AuditEventGeneratorConfig):List[String] = { - var relationProps = List[String]() - if (relations.nonEmpty) { - relations.foreach(rel => { - val direction = rel.getOrElse("dir", "").asInstanceOf[String] - val relationType = rel.getOrElse("rel", "").asInstanceOf[String] - val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] - val relationProp = objectDefinition.relationLabel(targetObjType, direction, relationType) - if (relationProp.nonEmpty) { - relationProps :+= relationProp.get - } - }) - } - relationProps - } - - /** - * @param oldDate - * @param newDate - * @return - */ - def computeDuration(oldDate: String, newDate: String): Long = { - val sdf:SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - val od = sdf.parse(oldDate) - val nd = sdf.parse(newDate) - val diff = nd.getTime - od.getTime - val diffSeconds = diff / 1000 - diffSeconds - } -} \ No newline at end of file diff --git a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/task/AuditEventGeneratorConfig.scala b/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/task/AuditEventGeneratorConfig.scala deleted file mode 100644 index 2ca0ac023..000000000 --- a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/task/AuditEventGeneratorConfig.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.sunbird.job.auditevent.task - -import java.util -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig - -class AuditEventGeneratorConfig(override val config: Config) extends BaseJobConfig(config, "audit-event-generator") { - - private val serialVersionUID = 2905979434303791379L - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val kafkaOutputTopic: String = config.getString("kafka.output.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - val kafkaProducerParallelism: Int = config.getInt("task.producer.parallelism") - - val auditOutputTag: OutputTag[String] = OutputTag[String]("audit-event-tag") - - val defaultChannel: String =config.getString("channel.default") - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val skippedEventCount = "skipped-events-count" - val emptySchemaEventCount = "empty-schema-events-count" - val emptyPropsEventCount = "empty-props-events-count" - - // Consumers - val auditEventConsumer = "audit-event-generator-consumer" - val auditEventGeneratorFunction = "audit-event-generator-function" - val auditEventProducer = "audit-event-generator-producer" - - val basePath = config.getString("schema.basePath") - val configVersion = "1.0" -} diff --git a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/task/AuditEventGeneratorStreamTask.scala b/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/task/AuditEventGeneratorStreamTask.scala deleted file mode 100644 index 0898de6b3..000000000 --- a/audit-event-generator/src/main/scala/org/sunbird/job/auditevent/task/AuditEventGeneratorStreamTask.scala +++ /dev/null @@ -1,59 +0,0 @@ -package org.sunbird.job.auditevent.task - -import java.io.File -import java.util -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.functions.KeySelector -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.auditevent.domain.Event -import org.sunbird.job.auditevent.functions.AuditEventGenerator -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - - -class AuditEventGeneratorStreamTask(config: AuditEventGeneratorConfig, kafkaConnector: FlinkKafkaConnector) { - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val processStreamTask = env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.auditEventConsumer) - .uid(config.auditEventConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new AuditEventGenerator(config)) - .name(config.auditEventGeneratorFunction) - .uid(config.auditEventGeneratorFunction) - .setParallelism(config.parallelism) - - processStreamTask.getSideOutput(config.auditOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaOutputTopic)) - .name(config.auditEventProducer).uid(config.auditEventProducer).setParallelism(config.kafkaProducerParallelism) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object AuditEventGeneratorStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("audit-event-generator.conf").withFallback(ConfigFactory.systemEnvironment())) - val auditEventConfig = new AuditEventGeneratorConfig(config) - val kafkaUtil = new FlinkKafkaConnector(auditEventConfig) - val httpUtil = new HttpUtil - val task = new AuditEventGeneratorStreamTask(auditEventConfig, kafkaUtil) - task.process() - } -} - -// $COVERAGE-ON$ - -class AuditEventKeySelector extends KeySelector[Event, String] { - override def getKey(in: Event): String = in.id.replace(".img", "") -} \ No newline at end of file diff --git a/audit-event-generator/src/test/resources/logback-test.xml b/audit-event-generator/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/audit-event-generator/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/audit-event-generator/src/test/resources/test.conf b/audit-event-generator/src/test/resources/test.conf deleted file mode 100644 index 24b5d5b8c..000000000 --- a/audit-event-generator/src/test/resources/test.conf +++ /dev/null @@ -1,20 +0,0 @@ -include "base-test.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - output.topic = "sunbirddev.telemetry.raw" - groupId = "sunbirddev-audit-event-generator-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - producer.parallelism = 1 - window.time = 60 -} - -schema { - basePath = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" -} - -channel.default = "org.sunbird" \ No newline at end of file diff --git a/audit-event-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/audit-event-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index 0385a90d9..000000000 --- a/audit-event-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,44 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"ets":1518517878987,"nodeUniqueId":"do_11243969846440755213","requestId":null,"transactionData":{"properties":{"code":{"ov":null,"nv":"test_code"},"keywords":{"ov":null,"nv":["colors","games"]},"channel":{"ov":null,"nv":"in.ekstep"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/pdf"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentDisposition":{"ov":null,"nv":"inline"},"contentEncoding":{"ov":null,"nv":"identity"},"lastUpdatedOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentType":{"ov":null,"nv":"Story"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"versionKey":{"ov":null,"nv":"1518517878947"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"framework":{"ov":null,"nv":"NCF"},"compatibilityLevel":{"ov":null,"nv":1.0},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"Untitled Resource"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_11243969846440755213"},"status":{"ov":null,"nv":"Draft"},"resourceType":{"ov":null,"nv":"Story"}}},"operationType":"CREATE","nodeGraphId":113603,"label":"Untitled Resource","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2018-02-13T16:01:18.987+0530","objectType":"Content"} - |""".stripMargin - - val EVENT_2: String = - """ - |{"ets":1552464504681,"channel":"in.ekstep","transactionData":{"properties":{"lastStatusChangedOn":{"ov":"2019-03-13T13:25:43.129+0530","nv":"2019-03-13T13:38:24.358+0530"},"lastSubmittedOn":{"ov":null,"nv":"2019-03-13T13:38:21.901+0530"},"lastUpdatedOn":{"ov":"2019-03-13T13:36:20.093+0530","nv":"2019-03-13T13:38:24.399+0530"},"status":{"ov":"Draft","nv":"Review"},"versionKey":{"ov":"1552464380093","nv":"1552464504399"}}},"label":"Resource Content 1","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2019-03-13T13:38:24.680+0530","objectType":"Content","nodeUniqueId":"do_11271778298376192013","requestId":null,"operationType":"UPDATE","nodeGraphId":590883,"graphId":"domain"} - |""".stripMargin - - val EVENT_3: String = - """ - |{"ets":1552464380225,"channel":"in.ekstep","transactionData":{"properties":{"s3Key":{"ov":null,"nv":"content/do_11271778298376192013/artifact/pdf_1552464372724.pdf"},"size":{"ov":null,"nv":433994.0},"artifactUrl":{"ov":null,"nv":"https://sunbirddevbbpublic.blob.core.windows.net/sunbird-content-staging-knowlg/content/assets/do_21385816669265920016/pdf_233.pdf"},"lastUpdatedOn":{"ov":"2019-03-13T13:25:43.129+0530","nv":"2019-03-13T13:36:20.093+0530"},"versionKey":{"ov":"1552463743129","nv":"1552464380093"}}},"label":"Resource Content 1","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2019-03-13T13:36:20.223+0530","objectType":"Content","nodeUniqueId":"do_11271778298376192013","requestId":null,"operationType":"UPDATE","nodeGraphId":590883,"graphId":"domain"} - |""".stripMargin - - val EVENT_4: String = - """ - |{"ets":1552645516180,"channel":"in.ekstep","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"test.res.1"},"channel":{"ov":null,"nv":"in.ekstep"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/pdf"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2019-03-15T15:55:16.071+0530"},"contentDisposition":{"ov":null,"nv":"inline"},"lastUpdatedOn":{"ov":null,"nv":"2019-03-15T15:55:16.071+0530"},"contentEncoding":{"ov":null,"nv":"identity"},"dialcodeRequired":{"ov":null,"nv":"No"},"contentType":{"ov":null,"nv":"Resource"},"lastStatusChangedOn":{"ov":null,"nv":"2019-03-15T15:55:16.071+0530"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"visibility":{"ov":null,"nv":"Default"},"os":{"ov":null,"nv":["All"]},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"versionKey":{"ov":null,"nv":"1552645516071"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"framework":{"ov":null,"nv":"NCF"},"compatibilityLevel":{"ov":null,"nv":1.0},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"Resource Content 1"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_11271927206783385611"},"status":{"ov":null,"nv":"Draft"}}},"mid":"9ea9ae7a-9cc1-493d-aac3-3c66cd9ff01b","label":"Resource Content 1","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2019-03-15T15:55:16.178+0530","objectType":"Content","nodeUniqueId":"do_11271927206783385611","requestId":null,"operationType":"CREATE","nodeGraphId":590921,"graphId":"domain"} - |""".stripMargin - - val EVENT_5: String = - """ - |{"ets":1.614227716965E12,"transactionData":{"removedTags":[],"addedRelations":[{"rel":"hasSequenceMember","id":"do_11322375329257062411","label":"event 4.1: 11-1","dir":"OUT","type":"Event","relMetadata":{"IL_SEQUENCE_INDEX":1.0}}],"removedRelations":[],"addedTags":[],"properties":{}},"mid":"fc0bb006-7269-4b10-96c3-10672fca53a0","label":"eventset 4","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-02-25T04:35:16.965+0000","objectType":null,"partition":1,"nodeUniqueId":"do_11322375344215654413","operationType":"UPDATE","nodeGraphId":509461.0,"graphId":"domain"} - |""".stripMargin - - val EVENT_6: String = - """ - |{"ets":1502102183388,"nodeUniqueId":"do_112276071067320320114","requestId":null,"transactionData":{"addedRelations":[{"rel":"hasSequenceMember","id":"do_1123032073439723521148","label":"Test unit 11","dir":"IN","type":"Content"}],"removedRelations":[],"properties":{"name":{"nv":"","ov":""}}},"operationType":"CREATE","nodeGraphId":105631,"label":"collaborator test","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-08-07T10:36:23.388+0000","objectType":"Content"} - |""".stripMargin - - val EVENT_7: String = - """ - |{"ets":1614776853781,"channel":"b00bc992ef25f1a9a8d63291e20efc8d","transactionData":{"properties":{"dialcodes":{"ov":null,"nv":["K1W6L6"]}}},"mid":"5b5633a2-3c18-49a6-8822-6d7b85338104","label":"Test Again","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-03-03T13:07:33.781+0000","objectType":"Collection","nodeUniqueId":"do_1132282511204024321262","requestId":null,"operationType":"UPDATE","nodeGraphId":510086,"graphId":"domain"} - |""".stripMargin - - val EVENT_8: String = - """ - |{"ets":1.614227716965E12,"transactionData":{"removedTags":[],"addedRelations":[{"rel":"hasSequenceMember","id":"do_11322375329257062411","label":"event 4.1: 11-1","dir":"OUT","type":"Event","relMetadata":{"IL_SEQUENCE_INDEX":1.0}}],"removedRelations":[],"addedTags":[],"properties":{}},"mid":"fc0bb006-7269-4b10-96c3-10672fca53a0","label":"eventset 4","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-02-25T04:35:16.965+0000","objectType":"RandomSchemaType","partition":1,"nodeUniqueId":"do_11322375344215654413","operationType":"UPDATE","nodeGraphId":509461.0,"graphId":"domain"} - |""".stripMargin -} \ No newline at end of file diff --git a/audit-event-generator/src/test/scala/org/sunbird/job/spec/AuditEventGeneratorTaskTestSpec.scala b/audit-event-generator/src/test/scala/org/sunbird/job/spec/AuditEventGeneratorTaskTestSpec.scala deleted file mode 100644 index 66e533da5..000000000 --- a/audit-event-generator/src/test/scala/org/sunbird/job/spec/AuditEventGeneratorTaskTestSpec.scala +++ /dev/null @@ -1,108 +0,0 @@ -package org.sunbird.job.spec - -import java.util -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.sink.SinkFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.sunbird.job.util.JSONUtil -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.auditevent.domain.Event -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.auditevent.task.{AuditEventGeneratorConfig, AuditEventGeneratorStreamTask} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -class AuditEventGeneratorTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig: AuditEventGeneratorConfig = new AuditEventGeneratorConfig(config) - var currentMilliSecond = 1605816926271L - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - flinkCluster.after() - super.afterAll() - } - - "AuditEventGeneratorStreamTask" should "generate audit event" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditEventGeneratorMapSource) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaOutputTopic)).thenReturn(new AuditEventSink) - - new AuditEventGeneratorStreamTask(jobConfig, mockKafkaUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(1) - - AuditEventSink.values.size() should be(1) - AuditEventSink.values.forEach(event => { - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](event) - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - }) - } - - "AuditEventGeneratorStreamTask" should "increase metric for unknown schema" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new RandomObjectTypeAuditEventGeneratorMapSource) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaOutputTopic)).thenReturn(new AuditEventSink) - - new AuditEventGeneratorStreamTask(jobConfig, mockKafkaUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.emptySchemaEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.emptyPropsEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(0) - } -} - -class AuditEventGeneratorMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1), 0, 10)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_5), 0, 11)) - } - - override def cancel(): Unit = {} -} - -class RandomObjectTypeAuditEventGeneratorMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_8), 0, 10)) - } - - override def cancel(): Unit = {} -} - -class AuditEventSink extends SinkFunction[String] { - - override def invoke(value: String): Unit = { - synchronized { - AuditEventSink.values.add(value) - } - } -} - -object AuditEventSink { - val values: util.List[String] = new util.ArrayList() -} \ No newline at end of file diff --git a/audit-event-generator/src/test/scala/org/sunbird/job/spec/service/AuditEventGeneratorServiceTestSpec.scala b/audit-event-generator/src/test/scala/org/sunbird/job/spec/service/AuditEventGeneratorServiceTestSpec.scala deleted file mode 100644 index 8eb00e3bf..000000000 --- a/audit-event-generator/src/test/scala/org/sunbird/job/spec/service/AuditEventGeneratorServiceTestSpec.scala +++ /dev/null @@ -1,125 +0,0 @@ -package org.sunbird.job.spec.service - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.mockito.Mockito -import org.sunbird.job.Metrics -import org.sunbird.job.auditevent.domain.Event -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.auditevent.functions.AuditEventGenerator -import org.sunbird.job.auditevent.task.AuditEventGeneratorConfig -import org.sunbird.job.util.JSONUtil -import org.sunbird.spec.BaseTestSpec - -import java.util - -class AuditEventGeneratorServiceTestSpec extends BaseTestSpec { - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val config: Config = ConfigFactory.load("test.conf") - lazy val jobConfig: AuditEventGeneratorConfig = new AuditEventGeneratorConfig(config) - lazy val auditEventGenerator:AuditEventGenerator = new AuditEventGenerator(jobConfig) - lazy val mockMetrics = mock[Metrics](Mockito.withSettings().serializable()) - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "AuditEventGeneratorService" should "generate audit event" in { - val inputEvent:util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - } - - "AuditEventGeneratorService" should "add duration of status change" in { - val inputEvent:util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]]("duration").asInstanceOf[Int] - duration should be(761) - } - - "AuditEventGeneratorService" should "add Duration as null" in { - val inputEvent:util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]].getOrElse("duration", null) - duration should be(null) - } - - "AuditEventGeneratorService" should "generate audit for content creation" in { - val inputEvent:util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_4) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]].getOrElse("duration", null) - duration should be(null) - } - - "AuditEventGeneratorService" should "skip audit for objectType is null" in { - val inputEvent:util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_5) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - - eventStr should be("{\"object\": {\"type\":null}}") - } - - "AuditEventGeneratorService" should "event for addedRelations" in { - val inputEvent:util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_6) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata").asInstanceOf[Map[String, AnyRef]]("props").asInstanceOf[List[String]] should contain ("name") - eventMap("edata").asInstanceOf[Map[String, AnyRef]]("props").asInstanceOf[List[String]] should contain ("collections") - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]].getOrElse("duration", null) - duration should be(null) - } - - "AuditEventGeneratorService" should "generate audit for update dialcode" in { - val inputEvent:util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_7) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("edata").asInstanceOf[Map[String, AnyRef]]("props").asInstanceOf[List[String]] should contain ("dialcodes") - val cdata = eventMap("cdata").asInstanceOf[List[Map[String, AnyRef]]] - cdata.head("id").asInstanceOf[List[String]] should contain ("K1W6L6") - cdata.head("type") should be("DialCode") - } - - "AuditEventGeneratorService" should "compute duration" in { - val ov = "2019-03-13T13:25:43.129+0530" - val nv = "2019-03-13T13:38:24.358+0530" - val duration = auditEventGenerator.computeDuration(ov, nv) - duration should be(761) - } -} diff --git a/audit-history-indexer/README.md b/audit-history-indexer/README.md deleted file mode 100644 index 319641780..000000000 --- a/audit-history-indexer/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Audit History Indexer - -Audit History Indexer job - -## Getting Started - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. -### Prerequisites - -1. Download flink-1.13.6-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.13.6/flink-1.13.6-bin-scala_2.12.tgz). -2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. -3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. -4. Docker installed. -5. A running yarn cluster or a kubernetes cluster. - -### Build - -mvn clean install - -## Deployment - -### Yarn - -Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. - -``` -./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /audit-history-indexer/target/audit-history-indexer-0.0.1.jar -``` - -### Kubernetes - -``` -# Create a single node cluster -k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 -# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. -export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" - -# Only for Mac OSX -# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl -rm /usr/local/bin/kubectl -brew link --overwrite kubernetes-cli - -# Create a configmap using the flink-configuration-configmap.yaml -kubectl create -f knowledge-platform-job/kubernetes/flink-configuration-configmap.yaml - -# Create pods for jobmanager-service, job-manager and task-manager using the yaml files -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-service.yaml -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-deployment.yaml -kubectl create -f knowledge-platform-job/kubernetes/taskmanager-deployment.yaml - -# Create a port-forwarding for accessing the job-manager UI on localhost:8081 -kubectl port-forward deployment/flink-jobmanager 8081:8081 - -# Submit the job to the Kubernetes single node cluster flink-cluster -./bin/flink run -m localhost:8081 /audit-history-indexer/target/audit-history-indexer-0.0.1.jar - -# Commands to delete the pods created in the cluster -kubectl delete deployment/flink-jobmanager -kubectl delete deployment/flink-taskmanager -kubectl delete service/flink-jobmanager -kubectl delete configmaps/flink-config - -# Command to stop the single-node cluster -k3d stop --name="flink-cluster" -``` diff --git a/audit-history-indexer/pom.xml b/audit-history-indexer/pom.xml deleted file mode 100644 index 45e056768..000000000 --- a/audit-history-indexer/pom.xml +++ /dev/null @@ -1,214 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - audit-history-indexer - 1.0.0 - jar - audit-history-indexer - - Audit history indexer Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - com.squareup.okhttp3 - mockwebserver - 4.9.1 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.audithistory.task.AuditHistoryIndexerStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - audit-history-indexer-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/audit-history-indexer/src/main/resources/audit-history-indexer.conf b/audit-history-indexer/src/main/resources/audit-history-indexer.conf deleted file mode 100644 index c1a80e404..000000000 --- a/audit-history-indexer/src/main/resources/audit-history-indexer.conf +++ /dev/null @@ -1,14 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - groupId = "sunbirddev-audit-history-indexer-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - window.time = 60 -} - -timezone = "IST" \ No newline at end of file diff --git a/audit-history-indexer/src/main/resources/log4j.properties b/audit-history-indexer/src/main/resources/log4j.properties deleted file mode 100644 index faca05f3e..000000000 --- a/audit-history-indexer/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=audit-history-indexer.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/domain/CaseClasses.scala b/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/domain/CaseClasses.scala deleted file mode 100644 index 55f80c4a1..000000000 --- a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/domain/CaseClasses.scala +++ /dev/null @@ -1,7 +0,0 @@ -package org.sunbird.job.audithistory.domain - -import java.io.Serializable -import java.util.Date - -@SerialVersionUID(-5779950964487302125L) -case class AuditHistoryRecord(var objectId: String, objectType: String, label: String, graphId: String, var userId: String, requestId: String, logRecord: String, operation: String, createdOn: Date) extends Serializable \ No newline at end of file diff --git a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/domain/Event.scala b/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/domain/Event.scala deleted file mode 100644 index 6ab78a624..000000000 --- a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/domain/Event.scala +++ /dev/null @@ -1,53 +0,0 @@ -package org.sunbird.job.audithistory.domain - -import org.sunbird.job.domain.reader.JobRequest - -import java.text.{DateFormat, SimpleDateFormat} -import java.util -import java.util.Date - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val df:DateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - - private val jobName = "AuditHistoryIndexer" - - def id: String = readOrDefault("nodeUniqueId", "") - - def nodeType: String = readOrDefault("nodeType", "") - - def ets: Long = readOrDefault("ets", 0L) - - def userId: String = readOrDefault("userId", "") - - def nodeUniqueId: String = readOrDefault("nodeUniqueId", "") - - def objectType: String = readOrDefault("objectType", "") - - def label: String = readOrDefault("label", "") - - def graphId: String = readOrDefault("graphId", "") - - def requestId: String = readOrDefault("requestId", "") - - def transactionData: Map[String, AnyRef] = { - readOrDefault("transactionData", new util.HashMap[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - } - - def operationType: String = readOrDefault("operationType", null) - - def syncMessage: String = readOrDefault("syncMessage", null) - - def createdOn: String = readOrDefault("createdOn", "") - - def createdOnDate: Date = if (createdOn.isEmpty) new Date else df.parse(createdOn) - - def audit: Boolean = readOrDefault("audit", true) - - def isValid: Boolean = { - operationType != null && null == syncMessage && audit - } - - - -} diff --git a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/functions/AuditHistoryIndexer.scala b/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/functions/AuditHistoryIndexer.scala deleted file mode 100644 index 2368eff42..000000000 --- a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/functions/AuditHistoryIndexer.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.sunbird.job.audithistory.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.sunbird.job.audithistory.domain.Event -import org.sunbird.job.audithistory.service.AuditHistoryIndexerService -import org.sunbird.job.audithistory.task.AuditHistoryIndexerConfig -import org.sunbird.job.util.ElasticSearchUtil -import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} - -import java.util - -class AuditHistoryIndexer(config: AuditHistoryIndexerConfig, var esUtil: ElasticSearchUtil) - (implicit mapTypeInfo: TypeInformation[util.Map[String, Any]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessKeyedFunction[String, Event, String](config) with AuditHistoryIndexerService{ - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.esFailedEventCount, config.skippedEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - if (esUtil == null) { - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.auditHistoryIndex, config.auditHistoryIndexType) - } - } - - override def close(): Unit = { - esUtil.close() - super.close() - } - - override def processElement(event: Event, - context: KeyedProcessFunction[String, Event, String]#Context, - metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - if(event.isValid) { - processEvent(event, metrics)(esUtil, config) - } else metrics.incCounter(config.skippedEventCount) - } -} diff --git a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/service/AuditHistoryIndexerService.scala b/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/service/AuditHistoryIndexerService.scala deleted file mode 100644 index 5558c787e..000000000 --- a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/service/AuditHistoryIndexerService.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.sunbird.job.audithistory.service - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.audithistory.domain.{AuditHistoryRecord, Event} -import org.sunbird.job.audithistory.task.AuditHistoryIndexerConfig -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.util.{ElasticSearchUtil, JSONUtil} - -import java.io.IOException -import java.util.{Calendar, Date, TimeZone} - -trait AuditHistoryIndexerService { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[AuditHistoryIndexerService]) - - @throws(classOf[InvalidEventException]) - def processEvent(event: Event, metrics: Metrics)(implicit esUtil: ElasticSearchUtil, config: AuditHistoryIndexerConfig): Unit = { - if (event.isValid) { - val identifier = event.nodeUniqueId - logger.info("Audit learning event received :: " + identifier) - try { - val record = getAuditHistory(event) - val document = JSONUtil.serialize(record) - val indexName = getIndexName(event.ets) - esUtil.addDocumentWithIndex(document, indexName) - logger.info("Audit record created for " + identifier) - metrics.incCounter(config.successEventCount) - } catch { - case ex: IOException => - logger.error("Error while indexing message :: " + event.getJson + " :: " + ex.getMessage) - ex.printStackTrace() - metrics.incCounter(config.esFailedEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - case ex: Exception => - logger.error("Error while processing message :: " + event.getJson + " :: ", ex) - metrics.incCounter(config.failedEventCount) - } - } - else logger.info("Learning event not qualified for audit") - } - - private def getIndexName(ets: Long)(implicit config: AuditHistoryIndexerConfig):String = { - val cal = Calendar.getInstance(TimeZone.getTimeZone(config.timeZone)) - cal.setTime(new Date(ets)) - config.auditHistoryIndex + "_" + cal.get(Calendar.YEAR) + "_" + cal.get(Calendar.WEEK_OF_YEAR) - } - - def getAuditHistory(transactionDataMap: Event): AuditHistoryRecord = { - val nodeUniqueId = StringUtils.replace(transactionDataMap.nodeUniqueId, ".img", "") - AuditHistoryRecord(nodeUniqueId, transactionDataMap.objectType, transactionDataMap.label, transactionDataMap.graphId, transactionDataMap.userId, transactionDataMap.requestId, JSONUtil.serialize(transactionDataMap.transactionData), transactionDataMap.operationType, transactionDataMap.createdOnDate) - } - -} \ No newline at end of file diff --git a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/task/AuditHistoryIndexerConfig.scala b/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/task/AuditHistoryIndexerConfig.scala deleted file mode 100644 index 49a9480e7..000000000 --- a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/task/AuditHistoryIndexerConfig.scala +++ /dev/null @@ -1,42 +0,0 @@ -package org.sunbird.job.audithistory.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.sunbird.job.BaseJobConfig - -import java.util - -class AuditHistoryIndexerConfig(override val config: Config) extends BaseJobConfig(config, "audit-history-indexer") { - - private val serialVersionUID = 2905979434303791379L - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val esFailedEventCount = "elasticsearch-error-events-count" - val skippedEventCount = "skipped-events-count" - - // Consumers - val eventConsumer = "audit-history-indexer-consumer" - val auditHistoryIndexerFunction = "audit-history-indexer-function" - - val configVersion = "1.0" - - // ES Configs - val esConnectionInfo = config.getString("es.basePath") - - val timeZone = if(config.hasPath("timezone")) config.getString("timezone") else "IST" - val auditHistoryIndex = "kp_audit_log" - val operationCreate = "CREATE" - val auditHistoryIndexType = "ah" -} diff --git a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/task/AuditHistoryIndexerStreamTask.scala b/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/task/AuditHistoryIndexerStreamTask.scala deleted file mode 100644 index 8833666b0..000000000 --- a/audit-history-indexer/src/main/scala/org/sunbird/job/audithistory/task/AuditHistoryIndexerStreamTask.scala +++ /dev/null @@ -1,58 +0,0 @@ -package org.sunbird.job.audithistory.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.functions.KeySelector -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.audithistory.domain.Event -import org.sunbird.job.audithistory.functions.AuditHistoryIndexer -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.util.{ElasticSearchUtil, FlinkUtil} - -import java.io.File -import java.util - - -class AuditHistoryIndexerStreamTask(config: AuditHistoryIndexerConfig, kafkaConnector: FlinkKafkaConnector, esUtil: ElasticSearchUtil) { - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, Any]] = TypeExtractor.getForClass(classOf[util.Map[String, Any]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.eventConsumer) - .uid(config.eventConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .keyBy(new AuditHistoryIndexerKeySelector) - .process(new AuditHistoryIndexer(config, esUtil)) - .name(config.auditHistoryIndexerFunction) - .uid(config.auditHistoryIndexerFunction) - .setParallelism(config.parallelism) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object AuditHistoryIndexerStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("audit-history-indexer.conf").withFallback(ConfigFactory.systemEnvironment())) - val auditHistoryIndexerConfig = new AuditHistoryIndexerConfig(config) - val kafkaUtil = new FlinkKafkaConnector(auditHistoryIndexerConfig) - val esUtil:ElasticSearchUtil = null - val task = new AuditHistoryIndexerStreamTask(auditHistoryIndexerConfig, kafkaUtil, esUtil) - task.process() - } -} - -// $COVERAGE-ON$ - -class AuditHistoryIndexerKeySelector extends KeySelector[Event, String] { - override def getKey(in: Event): String = in.id.replace(".img", "") -} diff --git a/audit-history-indexer/src/test/resources/logback-test.xml b/audit-history-indexer/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/audit-history-indexer/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/audit-history-indexer/src/test/resources/test.conf b/audit-history-indexer/src/test/resources/test.conf deleted file mode 100644 index cbc419173..000000000 --- a/audit-history-indexer/src/test/resources/test.conf +++ /dev/null @@ -1,14 +0,0 @@ -include "base-test.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - groupId = "sunbirddev-audit-history-indexer-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - window.time = 60 -} - -timezone = "IST" \ No newline at end of file diff --git a/audit-history-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/audit-history-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index e9f1c4681..000000000 --- a/audit-history-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,25 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"ets":1518517878987,"nodeUniqueId":"do_11243969846440755213","requestId":null,"transactionData":{"properties":{"code":{"ov":null,"nv":"test_code"},"keywords":{"ov":null,"nv":["colors","games"]},"channel":{"ov":null,"nv":"in.ekstep"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/pdf"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentDisposition":{"ov":null,"nv":"inline"},"contentEncoding":{"ov":null,"nv":"identity"},"lastUpdatedOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentType":{"ov":null,"nv":"Story"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"versionKey":{"ov":null,"nv":"1518517878947"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"framework":{"ov":null,"nv":"NCF"},"compatibilityLevel":{"ov":null,"nv":1.0},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"Untitled Resource"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_11243969846440755213"},"status":{"ov":null,"nv":"Draft"},"resourceType":{"ov":null,"nv":"Story"}}},"operationType":"CREATE","nodeGraphId":113603,"label":"Untitled Resource","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2018-02-13T16:01:18.987+0530","objectType":"Content"} - |""".stripMargin - - val EVENT_2: String = - """ - |{"ets":1502102183388,"nodeUniqueId":"do_112276071067320320114","requestId":null,"transactionData":{"addedRelations":[{"rel":"hasSequenceMember","id":"do_1123032073439723521148","label":"Test unit 11","dir":"IN","type":"Content"}],"removedRelations":[],"properties":{"name":{"nv":"","ov":""}}},"operationType":"CREATE","nodeGraphId":105631,"label":"collaborator test","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-08-07T10:36:23.388+0000","objectType":"Content"} - |""".stripMargin - - val EVENT_3: String = - """ - |{"ets":1615191835547,"channel":"01309282781705830427","transactionData":{"removedTags":[],"addedRelations":[],"removedRelations":[{"rel":"associatedTo","id":"do_113198273083662336127","label":"qq\n","dir":"OUT","type":"AssessmentItem","relMetadata":{}}],"addedTags":[],"properties":{}},"mid":"98145983-63dc-4d55-866c-248d49306ad8","label":"ECML_CHANGES","nodeType":"DATA_NODE","userId":"5a587cc1-e018-4859-a0a8-e842650b9d64","createdOn":"2021-03-08T08:23:55.547+0000","objectType":"Content","nodeUniqueId":"do_1132316371218268161118","requestId":null,"operationType":"UPDATE","nodeGraphId":510477,"graphId":"domain"} - |""".stripMargin - - val EVENT_4: String = - """ - |{"ets":1500888709490,"requestId":null,"transactionData":{"properties":{"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"morphology":{"ov":null,"nv":false},"consumerId":{"ov":null,"nv":"a6654129-b58d-4dd8-9cf2-f8f3c2f458bc"},"channel":{"ov":null,"nv":"in.ekstep"},"lemma":{"ov":null,"nv":"ವಿಶ್ಲೇಷಣೆ"},"createdOn":{"ov":null,"nv":"2017-07-24T09:32:18.130+0000"},"versionKey":{"ov":null,"nv":"1500888738130"},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Word"},"ekstepWordnet":{"ov":null,"nv":false},"lastUpdatedOn":{"ov":null,"nv":"2017-07-24T09:32:18.130+0000"},"isPhrase":{"ov":null,"nv":false},"IL_UNIQUE_ID":{"ov":null,"nv":"ka_11229528054276096015"},"status":{"ov":null,"nv":"Draft"}}},"nodeGraphId":433342,"label":"ವಿಶ್ಲೇಷಣೆ","graphId":"ka","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-07-24T09:31:49.490+0000","objectType":"Word"} - |""".stripMargin - -} \ No newline at end of file diff --git a/audit-history-indexer/src/test/scala/org/sunbird/job/spec/AuditHistoryIndexerTaskTestSpec.scala b/audit-history-indexer/src/test/scala/org/sunbird/job/spec/AuditHistoryIndexerTaskTestSpec.scala deleted file mode 100644 index efe3b4eb1..000000000 --- a/audit-history-indexer/src/test/scala/org/sunbird/job/spec/AuditHistoryIndexerTaskTestSpec.scala +++ /dev/null @@ -1,90 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import okhttp3.mockwebserver.{MockResponse, MockWebServer} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.client.JobExecutionException -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.audithistory.domain.Event -import org.sunbird.job.audithistory.task.{AuditHistoryIndexerConfig, AuditHistoryIndexerStreamTask} -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.util.{ElasticSearchUtil, JSONUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.util - -class AuditHistoryIndexerTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig: AuditHistoryIndexerConfig = new AuditHistoryIndexerConfig(config) - val esUtil: ElasticSearchUtil = null - val server = new MockWebServer() - - var currentMilliSecond = 1605816926271L - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - flinkCluster.before() - super.beforeAll() - } - - "AuditHistoryIndexerStreamTask" should "generate event" in { - server.start(9200) - server.enqueue(new MockResponse().setHeader( - "Content-Type", "application/json" - ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryIndexerMapSource) - - new AuditHistoryIndexerStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.esFailedEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - server.close() - } - - "AuditHistoryIndexerStreamTask" should "throw exception and increase es error count" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryIndexerMapSource) - - try { - new AuditHistoryIndexerStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.esFailedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(0) - } - } -} - -class AuditHistoryIndexerMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - // Valid event - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1), 0, 10)) - // Invalid event - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_4), 0, 11)) - } - - override def cancel(): Unit = {} -} \ No newline at end of file diff --git a/audit-history-indexer/src/test/scala/org/sunbird/job/spec/service/AuditHistoryIndexerServiceTestSpec.scala b/audit-history-indexer/src/test/scala/org/sunbird/job/spec/service/AuditHistoryIndexerServiceTestSpec.scala deleted file mode 100644 index 1af909777..000000000 --- a/audit-history-indexer/src/test/scala/org/sunbird/job/spec/service/AuditHistoryIndexerServiceTestSpec.scala +++ /dev/null @@ -1,62 +0,0 @@ -package org.sunbird.job.spec.service - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.mockito.Mockito -import org.sunbird.job.audithistory.domain.{AuditHistoryRecord, Event} -import org.sunbird.job.audithistory.functions.AuditHistoryIndexer -import org.sunbird.job.audithistory.task.AuditHistoryIndexerConfig -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.util.{ElasticSearchUtil, JSONUtil} -import org.sunbird.spec.BaseTestSpec - -import java.util - -class AuditHistoryIndexerServiceTestSpec extends BaseTestSpec { - implicit val mapTypeInfo: TypeInformation[util.Map[String, Any]] = TypeExtractor.getForClass(classOf[util.Map[String, Any]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val config: Config = ConfigFactory.load("test.conf") - lazy val jobConfig: AuditHistoryIndexerConfig = new AuditHistoryIndexerConfig(config) - val mockElasticUtil:ElasticSearchUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - lazy val auditHistoryIndexer: AuditHistoryIndexer = new AuditHistoryIndexer(jobConfig, mockElasticUtil) - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "AuditHistoryIndexerService" should "generate es log" in { - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - - val auditHistoryRec:AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent); - - auditHistoryRec.objectId should be(inputEvent.nodeUniqueId) - auditHistoryRec.objectType should be(inputEvent.objectType) - auditHistoryRec.logRecord should be("""{"properties":{"mediaType":{"nv":"content"},"name":{"nv":"Untitled Resource"},"createdOn":{"nv":"2018-02-13T16:01:18.947+0530"},"channel":{"nv":"in.ekstep"},"lastUpdatedOn":{"nv":"2018-02-13T16:01:18.947+0530"},"IL_FUNC_OBJECT_TYPE":{"nv":"Content"},"resourceType":{"nv":"Story"},"compatibilityLevel":{"nv":1.0},"audience":{"nv":["Learner"]},"os":{"nv":["All"]},"IL_SYS_NODE_TYPE":{"nv":"DATA_NODE"},"framework":{"nv":"NCF"},"versionKey":{"nv":"1518517878947"},"mimeType":{"nv":"application/pdf"},"code":{"nv":"test_code"},"contentType":{"nv":"Story"},"language":{"nv":["English"]},"status":{"nv":"Draft"},"keywords":{"nv":["colors","games"]},"idealScreenSize":{"nv":"normal"},"contentEncoding":{"nv":"identity"},"osId":{"nv":"org.ekstep.quiz.app"},"IL_UNIQUE_ID":{"nv":"do_11243969846440755213"},"contentDisposition":{"nv":"inline"},"visibility":{"nv":"Default"},"idealScreenDensity":{"nv":"hdpi"}}}""") - } - - "AuditHistoryIndexerService" should "generate with added relations" in { - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2), 0, 11) - - val auditHistoryRec:AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent); - - auditHistoryRec.objectId should be(inputEvent.nodeUniqueId) - auditHistoryRec.objectType should be(inputEvent.objectType) - auditHistoryRec.logRecord should be("""{"addedRelations":[{"label":"Test unit 11","rel":"hasSequenceMember","dir":"IN","id":"do_1123032073439723521148","type":"Content"}],"removedRelations":[],"properties":{"name":{"nv":"","ov":""}}}""") - } - - "AuditHistoryIndexerService" should "generate with removed relations" in { - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3),0, 12) - - val auditHistoryRec:AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent); - - auditHistoryRec.objectId should be(inputEvent.nodeUniqueId) - auditHistoryRec.objectType should be(inputEvent.objectType) - auditHistoryRec.logRecord should be("""{"addedTags":[],"addedRelations":[],"properties":{},"removedRelations":[{"label":"qq\n","rel":"associatedTo","dir":"OUT","id":"do_113198273083662336127","relMetadata":{},"type":"AssessmentItem"}],"removedTags":[]}""") - } -} diff --git a/cassandra-data-migration/pom.xml b/cassandra-data-migration/pom.xml deleted file mode 100644 index 4d0cc6db5..000000000 --- a/cassandra-data-migration/pom.xml +++ /dev/null @@ -1,215 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - cassandra-data-migration - 1.0.0 - jar - cassandra-data-migration - - Cassandra Data Migration Flink Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.sunbird.job.task.AutoCreatorV2StreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - cassandra-data-migration-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/cassandra-data-migration/src/main/resources/cassandra-data-migration.conf b/cassandra-data-migration/src/main/resources/cassandra-data-migration.conf deleted file mode 100644 index e096edc04..000000000 --- a/cassandra-data-migration/src/main/resources/cassandra-data-migration.conf +++ /dev/null @@ -1,18 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.cassandra.data.migration.request" - failed.topic = "sunbirddev.cassandra.data.migration.job.request.failed" - groupId = "sunbirddev-cassandra-data-migration-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 -} - -migrate { - key_value_strings_to_migrate = { - "https://sunbirddev.blob.core.windows.net": "https://ekstep-public-dev.s3-ap-south-1.amazonaws.com" - } -} diff --git a/cassandra-data-migration/src/main/resources/log4j.properties b/cassandra-data-migration/src/main/resources/log4j.properties deleted file mode 100644 index 6217bab54..000000000 --- a/cassandra-data-migration/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=auto-creator-v2.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/domain/Event.scala b/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/domain/Event.scala deleted file mode 100644 index 6474a0a35..000000000 --- a/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/domain/Event.scala +++ /dev/null @@ -1,34 +0,0 @@ -package org.sunbird.job.migration.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest -import org.sunbird.job.task.CassandraDataMigrationConfig - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "cassandra-data-migration" - - def eData: Map[String, AnyRef] = readOrDefault("edata", Map()).asInstanceOf[Map[String, AnyRef]] - - def action: String = readOrDefault[String]("edata.action", "") - - def keyspace: String = readOrDefault[String]("edata.keyspace", "") - - def table: String = readOrDefault[String]("edata.table", "") - - def column: String = readOrDefault[String]("edata.column", "") - - def columnType: String = readOrDefault[String]("edata.columnType", "") - - def primaryKeyColumn: String = readOrDefault[String]("edata.primaryKeyColumn", "") - - def primaryKeyColumnType: String = readOrDefault[String]("edata.primaryKeyColumnType", "") - - - def isValid(): Boolean = { - (StringUtils.equals("migrate-cassandra", action) && StringUtils.isNotBlank(column) && StringUtils.isNotBlank(columnType) - && StringUtils.isNotBlank(table) && StringUtils.isNotBlank(keyspace) && StringUtils.isNotBlank(primaryKeyColumn) - && StringUtils.isNotBlank(primaryKeyColumnType)) - } - -} \ No newline at end of file diff --git a/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/functions/CassandraDataMigrationFunction.scala b/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/functions/CassandraDataMigrationFunction.scala deleted file mode 100644 index 5b14172ab..000000000 --- a/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/functions/CassandraDataMigrationFunction.scala +++ /dev/null @@ -1,45 +0,0 @@ -package org.sunbird.job.migration.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.task.CassandraDataMigrationConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.migration.domain.Event -import org.sunbird.job.migration.helpers.CassandraDataMigrator -import org.sunbird.job.util._ -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.util - -class CassandraDataMigrationFunction(config: CassandraDataMigrationConfig, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with CassandraDataMigrator { - - private[this] lazy val logger = LoggerFactory.getLogger(classOf[CassandraDataMigrationFunction]) - lazy val defCache: DefinitionCache = new DefinitionCache() - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - } - - override def close(): Unit = { - super.close() - cassandraUtil.close() - } - - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - logger.info("CassandraDataMigrationFunction:: processElement:: event:: " + event) - if (event.isValid()) - migrateData(event, config)(cassandraUtil) - else logger.info("CassandraDataMigrationFunction:: processElement:: event SKIPPED!! :: " + event) - } -} diff --git a/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/helpers/CassandraDataMigrator.scala b/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/helpers/CassandraDataMigrator.scala deleted file mode 100644 index e9f6e281d..000000000 --- a/cassandra-data-migration/src/main/scala/org/sunbird/job/migration/helpers/CassandraDataMigrator.scala +++ /dev/null @@ -1,78 +0,0 @@ -package org.sunbird.job.migration.helpers - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.{Clause, QueryBuilder} -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.migration.domain.Event -import org.sunbird.job.task.CassandraDataMigrationConfig -import org.sunbird.job.util._ - -trait CassandraDataMigrator { - - private[this] val logger = LoggerFactory.getLogger(classOf[CassandraDataMigrator]) - - def migrateData(event: Event, config: CassandraDataMigrationConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - - // select primary key Column rows from table to migrate - val primaryKeys = readPrimaryKeysFromCassandra(event) - logger.info(s"CassandraDataMigrator:: migrateData:: After fetching primary keys. Keys Count:: " + primaryKeys.size()) - primaryKeys.forEach(col => { - val primaryKey = event.primaryKeyColumnType.toLowerCase match { - case "uuid" => col.getUUID(event.primaryKeyColumn) - case _ => col.getString(event.primaryKeyColumn) - } - val row = readColumnDataFromCassandra(primaryKey, event)(cassandraUtil) - if(row != null) { - val fetchedData: String = row.getString(event.column) - logger.info(s"CassandraDataMigrator:: migrateData:: Fetched ${event.column} in Cassandra For $primaryKey :: $fetchedData") - - val migratedData = StringUtils.replaceEach(fetchedData, config.keyValueMigrateStrings.keySet().toArray().map(_.asInstanceOf[String]), config.keyValueMigrateStrings.values().toArray().map(_.asInstanceOf[String])) - - // Pass updated data to row using primaryKey field - updateMigratedDataToCassandra(migratedData, primaryKey, event) (cassandraUtil) - } - }) - - } - - def readPrimaryKeysFromCassandra(event: Event)(implicit cassandraUtil: CassandraUtil): java.util.List[Row] = { - val query = s"""select ${event.primaryKeyColumn} from ${event.keyspace}.${event.table} ALLOW FILTERING;""" - cassandraUtil.find(query) - } - - def readColumnDataFromCassandra(primaryKey: AnyRef, event: Event)(implicit cassandraUtil: CassandraUtil): Row = { - val query = event.primaryKeyColumnType.toLowerCase match { - case "uuid" => event.columnType.toLowerCase match { - case "blob" => s"""select blobAsText(${event.column}) as ${event.column} from ${event.keyspace}.${event.table} where ${event.primaryKeyColumn}=$primaryKey ALLOW FILTERING;""" - case _ => s"""select ${event.column} from ${event.keyspace}.${event.table} where ${event.primaryKeyColumn}=$primaryKey ALLOW FILTERING;""" - } - case _ => event.columnType.toLowerCase match { - case "blob" => s"""select blobAsText(${event.column}) as ${event.column} from ${event.keyspace}.${event.table} where ${event.primaryKeyColumn}='$primaryKey' ALLOW FILTERING;""" - case _ => s"""select ${event.column} from ${event.keyspace}.${event.table} where ${event.primaryKeyColumn}='$primaryKey' ALLOW FILTERING;""" - } - } - - cassandraUtil.findOne(query) - } - - def updateMigratedDataToCassandra(migratedData: String, primaryKey: AnyRef, event: Event)(implicit cassandraUtil: CassandraUtil): Unit = { - val update = QueryBuilder.update(event.keyspace, event.table) - val clause: Clause = QueryBuilder.eq(event.primaryKeyColumn, primaryKey) - update.where.and(clause) - event.columnType.toLowerCase match { - case "blob" => update.`with`(QueryBuilder.set(event.column, QueryBuilder.fcall("textAsBlob", migratedData))) - case _ => update.`with`(QueryBuilder.set(event.column, migratedData)) - } - - logger.info(s"CassandraDataMigrator:: updateMigratedDataToCassandra:: Updating ${event.column} in Cassandra For $primaryKey :: ${update}") - val result = cassandraUtil.update(update) - if (result) logger.info(s"CassandraDataMigrator:: updateMigratedDataToCassandra:: ${event.column} Updated Successfully For $primaryKey") - else { - logger.error(s"CassandraDataMigrator:: updateMigratedDataToCassandra:: ${event.column} Update Failed For $primaryKey") - throw new InvalidInputException(s"${event.column} Update Failed For $primaryKey") - } - } - -} diff --git a/cassandra-data-migration/src/main/scala/org/sunbird/job/task/CassandraDataMigrationConfig.scala b/cassandra-data-migration/src/main/scala/org/sunbird/job/task/CassandraDataMigrationConfig.scala deleted file mode 100644 index 2e8e93b8b..000000000 --- a/cassandra-data-migration/src/main/scala/org/sunbird/job/task/CassandraDataMigrationConfig.scala +++ /dev/null @@ -1,40 +0,0 @@ -package org.sunbird.job.task - -import java.util -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.sunbird.job.BaseJobConfig - - -class CassandraDataMigrationConfig(override val config: Config) extends BaseJobConfig(config, "cassandra-data-migraton") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val skippedEventCount = "skipped-events-count" - - // Consumers - val eventConsumer = "cassandra-data-migration-consumer" - val cassandraDataMigrationFunction = "cassandra-data-migration-process" - val cassandraDataMigrationEventProducer = "cassandra-data-migration-producer" - - val configVersion = "1.0" - - // DB Config - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - - val keyValueMigrateStrings: util.Map[String, String] = config.getAnyRef("migrate.key_value_strings_to_migrate").asInstanceOf[util.Map[String, String]] - - def getConfig() = config -} diff --git a/cassandra-data-migration/src/main/scala/org/sunbird/job/task/CassandraDataMigrationStreamTask.scala b/cassandra-data-migration/src/main/scala/org/sunbird/job/task/CassandraDataMigrationStreamTask.scala deleted file mode 100644 index e70cad4a3..000000000 --- a/cassandra-data-migration/src/main/scala/org/sunbird/job/task/CassandraDataMigrationStreamTask.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.sunbird.job.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.slf4j.LoggerFactory -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.migration.domain.Event -import org.sunbird.job.migration.functions.CassandraDataMigrationFunction -import org.sunbird.job.util.FlinkUtil - -import java.io.File -import java.util - - -class CassandraDataMigrationStreamTask(config: CassandraDataMigrationConfig, kafkaConnector: FlinkKafkaConnector) { - private[this] val logger = LoggerFactory.getLogger(classOf[CassandraDataMigrationStreamTask]) - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val cassandraDataMigratorStream = env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.eventConsumer) - .uid(config.eventConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new CassandraDataMigrationFunction(config)) - .name(config.cassandraDataMigrationFunction) - .uid(config.cassandraDataMigrationFunction) - .setParallelism(config.parallelism) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object CassandraDataMigrationStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("cassandra-data-migration.conf").withFallback(ConfigFactory.systemEnvironment())) - val cassandraDataMigratorConfig = new CassandraDataMigrationConfig(config) - val kafkaUtil = new FlinkKafkaConnector(cassandraDataMigratorConfig) - val task = new CassandraDataMigrationStreamTask(cassandraDataMigratorConfig, kafkaUtil) - task.process() - } -} - -// $COVERAGE-ON$ diff --git a/cassandra-data-migration/src/test/resources/test.conf b/cassandra-data-migration/src/test/resources/test.conf deleted file mode 100644 index 9746e85d4..000000000 --- a/cassandra-data-migration/src/test/resources/test.conf +++ /dev/null @@ -1,26 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.cassandra.data.migration.request" - failed.topic = "sunbirddev.cassandra.data.migration.job.request.failed" - groupId = "sunbirddev-cassandra-data-migration-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - content-auto-creator.parallelism = 1 -} - -lms-cassandra { - keyspace = "hierarchy_store" - table = "content_hierarchy" - host = "localhost" - port = "9142" -} - -migrate { - key_value_strings_to_migrate = { - "https://sunbirddev.blob.core.windows.net": "https://ekstep-public-dev.s3-ap-south-1.amazonaws.com" - } -} diff --git a/cassandra-data-migration/src/test/resources/test.cql b/cassandra-data-migration/src/test/resources/test.cql deleted file mode 100644 index c64b35f10..000000000 --- a/cassandra-data-migration/src/test/resources/test.cql +++ /dev/null @@ -1,31 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS hierarchy_store WITH replication = { - 'class': 'SimpleStrategy', - 'replication_factor': '1' -}; - -CREATE TABLE IF NOT EXISTS hierarchy_store.content_hierarchy ( - identifier text, - hierarchy text, - relational_metadata text, - PRIMARY KEY (identifier) -); - -CREATE KEYSPACE IF NOT EXISTS dialcodes WITH replication = { - 'class': 'SimpleStrategy', - 'replication_factor': '1' -}; - -CREATE TABLE IF NOT EXISTS dialcodes.dialcode_batch ( - processid uuid PRIMARY KEY, - channel text, - config map, - created_on timestamp, - dialcodes list, - publisher text, - status int, - url text -); - -INSERT INTO hierarchy_store.content_hierarchy(identifier, hierarchy) VALUES ('do_4567', '{"ownershipType":["createdBy"],"copyright":"Sunbird","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","channel":"b00bc992ef25f1a9a8d63291e20efc8d","organisation":["Sunbird"],"language":["English"],"variants":{"online":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar","size":7399.0},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","size":40995.0}},"mimeType":"application/vnd.ekstep.content-collection","leafNodes":["do_1130314841730334721104","do_1130314849898332161107","do_1130314857102131201110","do_1130314847650037761106","do_1130314851303178241108","do_1130314845426565121105"],"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11305855864948326411234/artifact/2a4b8abd789184932399d222d03d9b5c.thumb.jpg","collections":[],"children":[{"ownershipType":["createdBy"],"parent":"do_11305855864948326411234","copyright":"Sunbird","code":"do_11305855931318272011245","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:06:56.160+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305855931318272011245","copyright":"Sunbird","code":"b99980a2-63dd-4d55-9c35-5c5a70e0c505","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:06:56.153+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305855931312537611237","copyright":"Sunbird","code":"3e43357b-05ad-455e-bbb3-2911be50cc71","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:06:56.159+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305855931317452811243","copyright":"Sunbird","code":"82426726-038e-47d1-9403-813a083278d2","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:06:56.156+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305855931314995211239","copyright":"Sunbird","code":"9e2e3a6e-8a76-433d-85c7-d4ac3d732157","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:06:56.157+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314841730334721104/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314841730334721104/prad-pdf-content-1_1590758122228_do_1130314841730334721104_2.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314841730334721104/prad-pdf-content-1_1590758122675_do_1130314841730334721104_2.0_spine.ecar","size":943.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314841730334721104/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314841730334721104","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Live","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:18.492+0000","size":736557.0,"concepts":[],"name":"prad PDF Content-1","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314841730334721104/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:02:25.342+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:17.957+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:23.067+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:23.061+0000","os":["All"],"cloudStorageKey":"content/do_1130314841730334721104/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":2.0,"versionKey":"1590758117957","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314841730334721104/prad-pdf-content-1_1590758122228_do_1130314841730334721104_2.0.ecar","framework":"NCF","compatibilityLevel":4,"index":1,"depth":6,"parent":"do_11305855931315814411241"},{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314845426565121105/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314845426565121105/prad-pdf-content-2_1590758131556_do_1130314845426565121105_1.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314845426565121105/prad-pdf-content-2_1590758132007_do_1130314845426565121105_1.0_spine.ecar","size":859.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314845426565121105/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314845426565121105","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Draft","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:31.543+0000","size":736471.0,"concepts":[],"name":"prad PDF Content-2","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314845426565121105/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:03:10.463+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:31.107+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:32.306+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:32.302+0000","os":["All"],"cloudStorageKey":"content/do_1130314845426565121105/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":1.0,"versionKey":"1590758131107","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314845426565121105/prad-pdf-content-2_1590758131556_do_1130314845426565121105_1.0.ecar","framework":"NCF","compatibilityLevel":4,"index":2,"depth":6,"parent":"do_11305855931315814411241"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305855931315814411241","lastStatusChangedOn":"2020-07-06T19:06:56.157+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062416157","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":5,"compatibilityLevel":1,"name":"Unit - 1.1.1.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314841730334721104","do_1130314845426565121105"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305855931314995211239","lastStatusChangedOn":"2020-07-06T19:06:56.156+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062416156","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":4,"compatibilityLevel":1,"name":"Unit - 1.1.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314841730334721104","do_1130314845426565121105"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305855931317452811243","lastStatusChangedOn":"2020-07-06T19:06:56.159+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062416159","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":3,"compatibilityLevel":1,"name":"Unit - 1.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314841730334721104","do_1130314845426565121105"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305855931312537611237","lastStatusChangedOn":"2020-07-06T19:06:56.153+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062416153","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":2,"compatibilityLevel":1,"name":"Unit - 1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314841730334721104","do_1130314845426565121105"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305855931318272011245","lastStatusChangedOn":"2020-07-06T19:06:56.160+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062416160","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":1,"compatibilityLevel":1,"name":"Unit - 1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314841730334721104","do_1130314845426565121105"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"},{"ownershipType":["createdBy"],"parent":"do_11305855864948326411234","copyright":"Sunbird","code":"do_11305856007005798411326","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:08:28.552+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856007005798411326","copyright":"Sunbird","code":"6dd56b10-389a-49d1-a413-d11494856293","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:08:28.555+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856007008256011330","copyright":"Sunbird","code":"55c16590-ec0e-4c24-83c8-aed18e0b2d8d","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:08:28.554+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856007007436811328","copyright":"Sunbird","code":"70a1ac3b-e26c-4d92-9058-674286e597c5","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:08:28.550+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856007004160011324","copyright":"Sunbird","code":"ae34bd17-cac9-4198-b84e-244a1312bbd9","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:08:28.556+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314847650037761106/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314847650037761106/prad-pdf-content-3_1590758142406_do_1130314847650037761106_1.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314847650037761106/prad-pdf-content-3_1590758142803_do_1130314847650037761106_1.0_spine.ecar","size":859.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314847650037761106/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314847650037761106","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Draft","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:41.207+0000","size":736473.0,"concepts":[],"name":"prad PDF Content-3","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314847650037761106/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:03:37.605+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:40.816+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:43.114+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:43.110+0000","os":["All"],"cloudStorageKey":"content/do_1130314847650037761106/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":1.0,"versionKey":"1590758140816","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314847650037761106/prad-pdf-content-3_1590758142406_do_1130314847650037761106_1.0.ecar","framework":"NCF","compatibilityLevel":4,"index":1,"depth":6,"parent":"do_11305856007009075211332"},{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314849898332161107/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314849898332161107/prad-pdf-content-4_1590758137227_do_1130314849898332161107_1.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314849898332161107/prad-pdf-content-4_1590758137643_do_1130314849898332161107_1.0_spine.ecar","size":860.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314849898332161107/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314849898332161107","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Draft","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:32.747+0000","size":736472.0,"concepts":[],"name":"prad PDF Content-4","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314849898332161107/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:04:05.049+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:32.376+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:37.968+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:37.963+0000","os":["All"],"cloudStorageKey":"content/do_1130314849898332161107/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":1.0,"versionKey":"1590758132376","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314849898332161107/prad-pdf-content-4_1590758137227_do_1130314849898332161107_1.0.ecar","framework":"NCF","compatibilityLevel":4,"index":2,"depth":6,"parent":"do_11305856007009075211332"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856007009075211332","lastStatusChangedOn":"2020-07-06T19:08:28.556+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062508556","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":5,"compatibilityLevel":1,"name":"Unit - 2.1.1.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314849898332161107","do_1130314847650037761106"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856007004160011324","lastStatusChangedOn":"2020-07-06T19:08:28.550+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062508550","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":4,"compatibilityLevel":1,"name":"Unit - 2.1.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314849898332161107","do_1130314847650037761106"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856007007436811328","lastStatusChangedOn":"2020-07-06T19:08:28.554+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062508554","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":3,"compatibilityLevel":1,"name":"Unit - 2.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314849898332161107","do_1130314847650037761106"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856007008256011330","lastStatusChangedOn":"2020-07-06T19:08:28.555+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062508555","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":2,"compatibilityLevel":1,"name":"Unit - 2.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314849898332161107","do_1130314847650037761106"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856007005798411326","lastStatusChangedOn":"2020-07-06T19:08:28.552+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":2,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062508552","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":1,"compatibilityLevel":1,"name":"Unit - 2","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":2,"leafNodes":["do_1130314849898332161107","do_1130314847650037761106"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"},{"ownershipType":["createdBy"],"parent":"do_11305855864948326411234","copyright":"Sunbird","code":"0cc610ce-b54e-469c-87b5-8aadbe55ead2","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:09:35.086+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856061510451211340","copyright":"Sunbird","code":"73c4c338-5713-47c9-9d13-18b36952da43","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:09:35.098+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856061520281611348","copyright":"Sunbird","code":"ac62c605-e41a-43a0-82e8-ca6930a17442","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:09:35.091+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856061514547211344","copyright":"Sunbird","code":"beb2a366-c3ab-49aa-90e3-bac853837607","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:09:35.093+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"parent":"do_11305856061516185611346","copyright":"Sunbird","code":"dabe5332-cb83-45b5-9818-3b3b76b03299","channel":"b00bc992ef25f1a9a8d63291e20efc8d","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2020-07-06T19:09:35.090+0000","objectType":"Content","children":[{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314841730334721104/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314841730334721104/prad-pdf-content-1_1590758122228_do_1130314841730334721104_2.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314841730334721104/prad-pdf-content-1_1590758122675_do_1130314841730334721104_2.0_spine.ecar","size":943.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314841730334721104/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314841730334721104","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Live","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:18.492+0000","size":736557.0,"concepts":[],"name":"prad PDF Content-1","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314841730334721104/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:02:25.342+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:17.957+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:23.067+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:23.061+0000","os":["All"],"cloudStorageKey":"content/do_1130314841730334721104/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":2.0,"versionKey":"1590758117957","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314841730334721104/prad-pdf-content-1_1590758122228_do_1130314841730334721104_2.0.ecar","framework":"NCF","compatibilityLevel":4,"index":1,"depth":6,"parent":"do_11305856061513728011342"},{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314847650037761106/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314847650037761106/prad-pdf-content-3_1590758142406_do_1130314847650037761106_1.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314847650037761106/prad-pdf-content-3_1590758142803_do_1130314847650037761106_1.0_spine.ecar","size":859.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314847650037761106/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314847650037761106","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Draft","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:41.207+0000","size":736473.0,"concepts":[],"name":"prad PDF Content-3","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314847650037761106/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:03:37.605+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:40.816+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:43.114+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:43.110+0000","os":["All"],"cloudStorageKey":"content/do_1130314847650037761106/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":1.0,"versionKey":"1590758140816","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314847650037761106/prad-pdf-content-3_1590758142406_do_1130314847650037761106_1.0.ecar","framework":"NCF","compatibilityLevel":4,"index":2,"depth":6,"parent":"do_11305856061513728011342"},{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314851303178241108/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314851303178241108/prad-pdf-content-5_1590758139833_do_1130314851303178241108_1.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314851303178241108/prad-pdf-content-5_1590758140392_do_1130314851303178241108_1.0_spine.ecar","size":857.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314851303178241108/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314851303178241108","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Draft","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:39.820+0000","size":736469.0,"concepts":[],"name":"prad PDF Content-5","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314851303178241108/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:04:22.199+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:39.308+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:40.726+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:40.722+0000","os":["All"],"cloudStorageKey":"content/do_1130314851303178241108/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":1.0,"versionKey":"1590758139308","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314851303178241108/prad-pdf-content-5_1590758139833_do_1130314851303178241108_1.0.ecar","framework":"NCF","compatibilityLevel":4,"index":3,"depth":6,"parent":"do_11305856061513728011342"},{"ownershipType":["createdBy"],"previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314857102131201110/artifact/sftbr-04u.pdf","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314857102131201110/prad-pdf-content-6_1590758138471_do_1130314857102131201110_1.0.ecar","channel":"in.ekstep","questions":[],"language":["English"],"variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_1130314857102131201110/prad-pdf-content-6_1590758138908_do_1130314857102131201110_1.0_spine.ecar","size":857.0}},"mimeType":"application/pdf","usesContent":[],"artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314857102131201110/artifact/sftbr-04u.pdf","contentEncoding":"identity","contentType":"Resource","identifier":"do_1130314857102131201110","audience":["Learner"],"visibility":"Default","mediaType":"content","itemSets":[],"osId":"org.ekstep.quiz.app","lastPublishedBy":"System","version":2,"pragma":["external"],"prevState":"Draft","license":"CC BY 4.0","lastPublishedOn":"2020-05-29T13:15:38.461+0000","size":736468.0,"concepts":[],"name":"prad PDF Content-6","status":"Live","code":"test-Resourcce","prevStatus":"Processing","methods":[],"streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130314857102131201110/artifact/sftbr-04u.pdf","idealScreenSize":"normal","createdOn":"2020-05-29T13:05:32.987+0000","contentDisposition":"inline","lastUpdatedOn":"2020-05-29T13:15:38.046+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-05-29T13:15:39.231+0000","dialcodeRequired":"No","lastStatusChangedOn":"2020-05-29T13:15:39.226+0000","os":["All"],"cloudStorageKey":"content/do_1130314857102131201110/artifact/sftbr-04u.pdf","libraries":[],"pkgVersion":1.0,"versionKey":"1590758138046","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_1130314857102131201110/prad-pdf-content-6_1590758138471_do_1130314857102131201110_1.0.ecar","framework":"NCF","compatibilityLevel":4,"index":4,"depth":6,"parent":"do_11305856061513728011342"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856061513728011342","lastStatusChangedOn":"2020-07-06T19:09:35.090+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062575090","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":5,"compatibilityLevel":1,"name":"Unit - 3.1.1.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":4,"leafNodes":["do_1130314841730334721104","do_1130314857102131201110","do_1130314847650037761106","do_1130314851303178241108"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856061516185611346","lastStatusChangedOn":"2020-07-06T19:09:35.093+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062575093","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":4,"compatibilityLevel":1,"name":"Unit - 3.1.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":4,"leafNodes":["do_1130314841730334721104","do_1130314857102131201110","do_1130314847650037761106","do_1130314851303178241108"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856061514547211344","lastStatusChangedOn":"2020-07-06T19:09:35.091+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062575091","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":3,"compatibilityLevel":1,"name":"Unit - 3.1.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":4,"leafNodes":["do_1130314841730334721104","do_1130314857102131201110","do_1130314847650037761106","do_1130314851303178241108"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856061520281611348","lastStatusChangedOn":"2020-07-06T19:09:35.098+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062575098","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":2,"compatibilityLevel":1,"name":"Unit - 3.1","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":4,"leafNodes":["do_1130314841730334721104","do_1130314857102131201110","do_1130314847650037761106","do_1130314851303178241108"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","contentEncoding":"gzip","contentType":"CourseUnit","dialcodeRequired":"No","identifier":"do_11305856061510451211340","lastStatusChangedOn":"2020-07-06T19:09:35.086+0000","audience":["Learner"],"os":["All"],"visibility":"Parent","index":3,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"versionKey":"1594062575086","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCFCOPY","depth":1,"compatibilityLevel":1,"name":"Unit - 3","status":"Live","lastPublishedOn":"2020-07-06T19:10:42.996+0000","pkgVersion":1.0,"leafNodesCount":4,"leafNodes":["do_1130314841730334721104","do_1130314857102131201110","do_1130314847650037761106","do_1130314851303178241108"],"downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","variants":"{\"online\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643394_do_11305855864948326411234_1.0_online.ecar\",\"size\":7399.0},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar\",\"size\":40995.0}}"}],"contentEncoding":"gzip","lockKey":"98909e87-3361-4c6c-9b68-136fbead2bef","mimeTypesCount":"{\"application/pdf\":8,\"application/vnd.ekstep.content-collection\":15}","totalCompressedSize":5891940.0,"contentType":"Course","identifier":"do_11305855864948326411234","lastUpdatedBy":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","audience":["Learner"],"toc_url":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11305855864948326411234/artifact/do_11305855864948326411234_toc.json","visibility":"Default","contentTypesCount":"{\"CourseUnit\":15,\"Resource\":8}","childNodes":["do_11305856007009075211332","do_11305856061513728011342","do_11305855931312537611237","do_1130314841730334721104","do_11305856061520281611348","do_11305856007005798411326","do_11305856007008256011330","do_1130314847650037761106","do_1130314851303178241108","do_11305855931317452811243","do_11305855931314995211239","do_11305856061514547211344","do_1130314849898332161107","do_11305855931315814411241","do_11305856007007436811328","do_1130314857102131201110","do_11305855931318272011245","do_11305856061510451211340","do_11305856061516185611346","do_1130314845426565121105","do_11305856007004160011324"],"consumerId":"273f3b18-5dda-4a27-984a-060c7cd398d3","mediaType":"content","osId":"org.ekstep.quiz.app","lastPublishedBy":"Ekstep","version":2,"prevState":"Draft","license":"CC BY 4.0","size":40995.0,"lastPublishedOn":"2020-07-06T19:10:42.996+0000","name":"Course Hierarchy Test - 1","status":"Live","code":"org.sunbird.pZaKxV","prevStatus":"Processing","description":"Enter description for Course","posterImage":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11299104587967692816/artifact/2a4b8abd789184932399d222d03d9b5c.jpg","idealScreenSize":"normal","createdOn":"2020-07-06T19:05:35.144+0000","copyrightYear":2020,"contentDisposition":"inline","lastUpdatedOn":"2020-07-06T19:10:42.581+0000","SYS_INTERNAL_LAST_UPDATED_ON":"2020-07-06T19:10:43.861+0000","dialcodeRequired":"No","creator":"Reviewer User","lastStatusChangedOn":"2020-07-06T19:10:43.856+0000","createdFor":["ORG_001"],"os":["All"],"pkgVersion":1.0,"versionKey":"1594062642581","idealScreenDensity":"hdpi","s3Key":"ecar_files/do_11305855864948326411234/course-hierarchy-test-1_1594062643205_do_11305855864948326411234_1.0_spine.ecar","depth":0,"framework":"NCFCOPY","createdBy":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","leafNodesCount":6,"compatibilityLevel":4,"usedByContent":[],"board":"NCERT","resourceType":"Course","reservedDialcodes":{"K2D9J9":0},"c_sunbird_dev_private_batch_count":0,"c_sunbird_dev_open_batch_count":1,"batches":[{"createdFor":["ORG_001"],"endDate":null,"name":"Course Hierarchy Test - 1","batchId":"0130585642325770243","enrollmentType":"open","enrollmentEndDate":null,"startDate":"2020-07-06","status":0}]}'); - -INSERT INTO dialcodes.dialcode_batch(processid, channel, url) VALUES (uuid(), '123456', 'https://preprodall.blob.core.windows.net/dial/01272777697873100812/do_21312279009695334411163_english_class_1_mathematics_1601903174498.zip'); diff --git a/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/fixture/EventFixture.scala b/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/fixture/EventFixture.scala deleted file mode 100644 index 49378130d..000000000 --- a/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/fixture/EventFixture.scala +++ /dev/null @@ -1,26 +0,0 @@ -package org.sunbird.job.migration.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1619527882745,"mid":"LP.1619527882745.32dc378a-430f-49f6-83b5-bd73b767ad36","actor":{"id":"cassandra-migration","type":"System"},"context":{"channel":"ORG_001","pdata":{"id":"org.sunbird.platform","ver":"1.0"},"env":"dev"},"object":{"id":"","ver":""},"edata":{"keyspace":"hierarchy_store", "table":"content_hierarchy", "column": "hierarchy", "columnType":"String", "primaryKeyColumn": "identifier", "primaryKeyColumnType": "String", "action":"migrate-cassandra","iteration":1}} - |""".stripMargin - - val EVENT_2: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1619527882745,"mid":"LP.1619527882745.32dc378a-430f-49f6-83b5-bd73b767ad36","actor":{"id":"cassandra-migration","type":"System"},"context":{"channel":"ORG_001","pdata":{"id":"org.sunbird.platform","ver":"1.0"},"env":"dev"},"object":{"id":"","ver":""},"edata":{"keyspace":"dummy","table":"dummy", "column": "dummy", "action":"dummy","iteration":1}} - |""".stripMargin - - val EVENT_3: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1619527882745,"mid":"LP.1619527882745.32dc378a-430f-49f6-83b5-bd73b767ad36","actor":{"id":"cassandra-data-migration","type":"System"},"context":{"channel":"ORG_001","pdata":{"id":"org.sunbird.platform","ver":"1.0"},"env":"dev"},"edata":{"column":"url", "columnType":"String", "table": "dialcode_images", "keyspace": "dialcodes", "primaryKeyColumn": "filename", "primaryKeyColumnType": "String", "action":"migrate-cassandra","iteration":1}} - |""".stripMargin - - val EVENT_4: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1619527882745,"mid":"LP.1619527882745.32dc378a-430f-49f6-83b5-bd73b767ad36","actor":{"id":"cassandra-data-migration","type":"System"},"context":{"channel":"ORG_001","pdata":{"id":"org.sunbird.platform","ver":"1.0"},"env":"dev"},"edata":{"column":"url", "columnType":"String", "table": "dialcode_batch", "keyspace": "dialcodes", "primaryKeyColumn": "processid", "primaryKeyColumnType": "UUID", "action":"migrate-cassandra","iteration":1}} - |""".stripMargin - - -} \ No newline at end of file diff --git a/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/helpers/CassandraDataMigratorSpec.scala b/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/helpers/CassandraDataMigratorSpec.scala deleted file mode 100644 index 998218eff..000000000 --- a/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/helpers/CassandraDataMigratorSpec.scala +++ /dev/null @@ -1,58 +0,0 @@ -package org.sunbird.job.migration.helpers - -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.sunbird.job.migration.domain.Event -import org.sunbird.job.migration.fixture.EventFixture -import org.sunbird.job.task.CassandraDataMigrationConfig -import org.sunbird.job.util.{CassandraUtil, JSONUtil} -import org.sunbird.spec.BaseTestSpec - -import java.util - -class CassandraDataMigratorSpec extends BaseTestSpec { - - val config: Config = ConfigFactory.load("test.conf") - var cassandraUtil: CassandraUtil = _ - val jobConfig: CassandraDataMigrationConfig = new CassandraDataMigrationConfig(config) - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - // Clear the metrics - testCassandraUtil(cassandraUtil) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - } - - - "CassandraDataMigrator " should "update the string " in { - val content_id = "do_4567" - val event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1), 0, 10) - new TestCassandraDataMigrator().migrateData(event, jobConfig)(cassandraUtil) - val row = new TestCassandraDataMigrator().readColumnDataFromCassandra(content_id, event)(cassandraUtil) - val migratedData: String = row.getString(event.column) - assert(migratedData.contains("\""+jobConfig.keyValueMigrateStrings.values().toArray().head)) - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - } - - class TestCassandraDataMigrator extends CassandraDataMigrator {} -} diff --git a/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/task/CassandraDataMigrationTaskTestSpec.scala b/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/task/CassandraDataMigrationTaskTestSpec.scala deleted file mode 100644 index 74b3aaf67..000000000 --- a/cassandra-data-migration/src/test/scala/org/sunbird/job/migration/task/CassandraDataMigrationTaskTestSpec.scala +++ /dev/null @@ -1,73 +0,0 @@ -package org.sunbird.job.migration.task - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.migration.domain.Event -import org.sunbird.job.migration.fixture.EventFixture -import org.sunbird.job.task.{CassandraDataMigrationConfig, CassandraDataMigrationStreamTask} -import org.sunbird.job.util.{CassandraUtil, JSONUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.util - -class CassandraDataMigrationTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: CassandraDataMigrationConfig = new CassandraDataMigrationConfig(config) - var cassandraUtils: CassandraUtil = _ - - var currentMilliSecond = 1605816926271L - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtils = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtils.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - flinkCluster.after() - super.afterAll() - } - - "CassandraDataMigrationTask" should "generate event" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CassandraDataMigrationMapSource) - new CassandraDataMigrationStreamTask(jobConfig, mockKafkaUtil).process() - } -} - -class CassandraDataMigrationMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]): Unit = { - // Valid event - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1), 0, 10)) - - // Invalid event - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2), 0, 10)) - } - - override def cancel(): Unit = {} -} \ No newline at end of file diff --git a/csp-migrator/pom.xml b/csp-migrator/pom.xml deleted file mode 100644 index e24493acc..000000000 --- a/csp-migrator/pom.xml +++ /dev/null @@ -1,248 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - csp-migrator - 1.0.0 - jar - csp-migrator - - CSP Migrator Flink Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - com.google.apis - google-api-services-youtube - v3-rev182-1.22.0 - - - com.google.guava - guava-jdk5 - - - - - com.google.api-client - google-api-client - 1.20.0 - - - com.google.http-client - google-http-client - 1.20.0 - compile - - - com.google.apis - google-api-services-drive - v3-rev83-1.20.0 - - - com.google.oauth-client - google-oauth-client-jetty - 1.20.0 - - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.sunbird.job.cspmigrator.task.CSPMigratorStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - csp-migrator-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/csp-migrator/src/main/resources/csp-migrator.conf b/csp-migrator/src/main/resources/csp-migrator.conf deleted file mode 100644 index dfed1c541..000000000 --- a/csp-migrator/src/main/resources/csp-migrator.conf +++ /dev/null @@ -1,98 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.csp.migration.job.request" - failed.topic = "sunbirddev.csp.migration.job.request.failed" - groupId = "sunbirddev-csp-migrator-group" - live_video_stream.topic = "sunbirddev.live.video.stream.request" - live_content_node_republish.topic = "sunbirddev.republish.job.request" - live_question_node_republish.topic = "sunbirddev.assessment.republish.request" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - cassandra-migrator.parallelism = 1 - checkpointing.timeout = 4200000 -} - -redis { - database { - relationCache.id = 10 - collectionCache.id = 5 - } -} - -hierarchy { - keyspace = "hierarchy_store" - table = "content_hierarchy" -} - -content { - keyspace = "content_store" - content_table = "content_data" - assessment_table = "question_data" -} - -key_value_strings_to_migrate = { - "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging": "CLOUD_STORAGE_BASE_PATH" - "https://ekstep-public-dev.s3-ap-south-1.amazonaws.com": "CLOUD_STORAGE_BASE_PATH", - "https://community.ekstep.in/assets/public": "CLOUD_STORAGE_BASE_PATH", - "https://vdn.diksha.gov.in/assets/public": "CLOUD_STORAGE_BASE_PATH" -} - -neo4j_fields_to_migrate = { - "asset": ["artifactUrl","thumbnail"], - "content": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "assetsMap", "certTemplate", "itemSetPreviewUrl", "grayScaleAppIcon", "sourceURL", "downloadUrl", "variants"], - "contentimage": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "assetsMap", "certTemplate", "itemSetPreviewUrl", "grayScaleAppIcon", "sourceURL", "downloadUrl", "variants"], - "collection": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "toc_url", "grayScaleAppIcon","variants"], - "collectionimage": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "toc_url", "grayScaleAppIcon","variants"], - "plugins": ["artifactUrl"], - "itemset": ["previewUrl"], - "assessmentitem": ["data", "question", "solutions", "editorState", "media"], - "question": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"], - "questionimage": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"], - "questionset": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"], - "questionsetimage": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"] -} - -cassandra_fields_to_migrate = { - "assessmentitem": ["body", "editorState", "answer", "solutions", "instructions", "media"] -} - -cloud_storage { - folder { - content = "content" - artifact = "artifact" - } -} - -migrationVersion = 1 - -video_stream_regeneration_enable = false -live_node_republish_enable = false -copy_missing_files_to_cloud = false - -download_path = /tmp - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "CONTENT_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"] -cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] - -cloud_storage_type="" -cloud_storage_key="" -cloud_storage_secret="" -cloud_storage_container="" -cloud_storage_endpoint="" - -questionset.hierarchy.keyspace="hierarchy_store" -questionset.hierarchy.table="questionset_hierarchy" - -gdrive.application_name="{{ gdrive_application_name }}" -g_service_acct_cred="{{ auto_creator_g_service_acct_cred }}" diff --git a/csp-migrator/src/main/resources/log4j.properties b/csp-migrator/src/main/resources/log4j.properties deleted file mode 100644 index d86b28fce..000000000 --- a/csp-migrator/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=csp-migrator.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/domain/Event.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/domain/Event.scala deleted file mode 100644 index 04576c5cc..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/domain/Event.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.sunbird.job.cspmigrator.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.domain.reader.JobRequest - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "csp-migrator" - - def eData: Map[String, AnyRef] = readOrDefault("edata", Map()).asInstanceOf[Map[String, AnyRef]] - - def context: Map[String, AnyRef] = readOrDefault("context", Map()).asInstanceOf[Map[String, AnyRef]] - - def obj: Map[String, AnyRef] = readOrDefault("object", Map()).asInstanceOf[Map[String, AnyRef]] - - def channel: String = readOrDefault[String]("context.channel", "") - - def metadata: Map[String, AnyRef] = readOrDefault("edata.metadata", Map()) - - def action: String = readOrDefault[String]("edata.action", "") - - def mimeType: String = readOrDefault[String]("edata.metadata.mimeType", "") - - def status: String = readOrDefault[String]("edata.metadata.status", "") - - def objectType: String = readOrDefault[String]("edata.metadata.objectType", "") - - def identifier: String = readOrDefault[String]("object.id", "") - - def currentIteration: Int = readOrDefault[Int]("edata.iteration", 1) - - def isValid(objMetadata: Map[String, AnyRef], config: CSPMigratorConfig): Boolean = { - (StringUtils.equals("csp-migration", action) && StringUtils.isNotBlank(identifier)) && StringUtils.isNotBlank(status) - } - -} \ No newline at end of file diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/functions/CSPCassandraMigratorFunction.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/functions/CSPCassandraMigratorFunction.scala deleted file mode 100644 index 9edfcf4db..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/functions/CSPCassandraMigratorFunction.scala +++ /dev/null @@ -1,110 +0,0 @@ -package org.sunbird.job.cspmigrator.functions - -import org.apache.commons.lang3.StringUtils -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.cspmigrator.domain.Event -import org.sunbird.job.cspmigrator.helpers.CSPCassandraMigrator -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.ServerException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.util._ -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import scala.collection.JavaConverters._ - -import java.util - -class CSPCassandraMigratorFunction(config: CSPMigratorConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with CSPCassandraMigrator with FailedEventHelper { - - private[this] lazy val logger = LoggerFactory.getLogger(classOf[CSPCassandraMigratorFunction]) - lazy val defCache: DefinitionCache = new DefinitionCache() - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount, config.errorEventCount, config.assetVideoStreamCount, config.liveContentNodePublishCount, config.liveQuestionNodePublishCount, config.liveQuestionSetNodePublishCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - cloudStorageUtil = new CloudStorageUtil(config) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - logger.info("CSPCassandraMigratorFunction::processElement:: event context : " + event.context) - logger.info("CSPCassandraMigratorFunction::processElement:: event edata : " + event.eData) - - val objMetadata: Map[String, AnyRef] = getMetadata(event.identifier)(neo4JUtil) - val identifier: String = objMetadata.getOrElse("identifier", "").asInstanceOf[String] - - try { - val fieldsToMigrate: List[String] = if (config.getConfig.hasPath("neo4j_fields_to_migrate."+event.objectType.toLowerCase())) config.getConfig.getStringList("neo4j_fields_to_migrate."+event.objectType.toLowerCase()).asScala.toList - else throw new ServerException("ERR_CONFIG_NOT_FOUND", "Fields to migrate configuration not found for objectType: " + event.objectType) - val migratedMetadataFields: Map[String, String] = fieldsToMigrate.flatMap(migrateField => { - if(objMetadata.contains(migrateField)) { - val metadataFieldValue = objMetadata.getOrElse(migrateField, "").asInstanceOf[String] - - val tempMetadataFieldValue = handleExternalURLS(metadataFieldValue, identifier, config, httpUtil, cloudStorageUtil) - - val migrateValue: String = if (StringUtils.isNotBlank(tempMetadataFieldValue)) - StringUtils.replaceEach(tempMetadataFieldValue, config.keyValueMigrateStrings.keySet().toArray().map(_.asInstanceOf[String]), config.keyValueMigrateStrings.values().toArray().map(_.asInstanceOf[String])) - else null - if (config.copyMissingFiles && StringUtils.isNotBlank(migrateValue)) verifyFile(event.identifier, tempMetadataFieldValue, migrateValue, migrateField, config)(httpUtil, cloudStorageUtil) - - Map(migrateField -> migrateValue) - } else Map.empty[String, String] - }).filter(record => record._1.nonEmpty).toMap[String, String] - - val migratedObjMetadata = objMetadata ++ migratedMetadataFields - logger.info("CSPCassandraMigratorFunction::processElement:: migratedObjMetadata : " + migratedObjMetadata) - - updateNeo4j(migratedObjMetadata, event)(defCache, neo4JUtil, config) - - process(migratedObjMetadata, config, httpUtil, cassandraUtil, cloudStorageUtil) - - event.objectType match { - case "Content" | "Collection" => - finalizeMigration(migratedObjMetadata, event, metrics, config)(defCache, neo4JUtil) - if(config.liveNodeRepublishEnabled && (event.status.equalsIgnoreCase("Live") || - event.status.equalsIgnoreCase("Unlisted"))) { - pushLiveNodePublishEvent(objMetadata, context, metrics, config, config.liveCollectionNodePublishEventOutTag) - metrics.incCounter(config.liveContentNodePublishCount) - } - case "QuestionSet"| "QuestionSetImage" => - finalizeMigration(migratedObjMetadata, event, metrics, config)(defCache, neo4JUtil) - if(config.liveNodeRepublishEnabled && (event.status.equalsIgnoreCase("Live") || - event.status.equalsIgnoreCase("Unlisted"))) { - pushQuestionPublishEvent(migratedObjMetadata, context, metrics, config, config.liveQuestionSetNodePublishEventOutTag, config.liveQuestionSetNodePublishCount) - } - case _ => finalizeMigration(migratedObjMetadata, event, metrics, config)(defCache, neo4JUtil) - } - } catch { - case se: Exception => - logger.error("CSPCassandraMigratorFunction :: Message processing failed for mid : " + event.mid() + " || " + event , se) - logger.error("CSPCassandraMigratorFunction :: Error while migrating content :: " + se.getMessage) - metrics.incCounter(config.failedEventCount) - se.printStackTrace() - logger.info(s"""{ identifier: \"${objMetadata.getOrElse("identifier", "").asInstanceOf[String]}\", mimetype: \"${objMetadata.getOrElse("mimeType", "").asInstanceOf[String]}\", status: \"Failed\", stage: \"Static Migration\"}""") - // Insert into neo4j with migrationVersion as 0.1 - updateNeo4j(objMetadata + ("migrationVersion" -> 0.1.asInstanceOf[Number]), event)(defCache, neo4JUtil, config) - } - } - - -} diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/functions/CSPNeo4jMigratorFunction.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/functions/CSPNeo4jMigratorFunction.scala deleted file mode 100644 index 840f4e8b2..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/functions/CSPNeo4jMigratorFunction.scala +++ /dev/null @@ -1,124 +0,0 @@ -package org.sunbird.job.cspmigrator.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.cspmigrator.domain.Event -import org.sunbird.job.cspmigrator.helpers.CSPNeo4jMigrator -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.util._ -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.util - -class CSPNeo4jMigratorFunction(config: CSPMigratorConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with CSPNeo4jMigrator with FailedEventHelper { - - private[this] lazy val logger = LoggerFactory.getLogger(classOf[CSPNeo4jMigratorFunction]) - lazy val defCache: DefinitionCache = new DefinitionCache() - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount, config.errorEventCount, config.assetVideoStreamCount, config.liveContentNodePublishCount, config.liveQuestionNodePublishCount, config.liveQuestionSetNodePublishCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - cloudStorageUtil = new CloudStorageUtil(config) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - logger.info("CSPNeo4jMigratorFunction::processElement:: event context : " + event.context) - logger.info("CSPNeo4jMigratorFunction::processElement:: event edata : " + event.eData) - - val objMetadata: Map[String, AnyRef] = getMetadata(event.identifier)(neo4JUtil) - logger.info("CSPNeo4jMigratorFunction::processElement:: objMetadata : " + objMetadata) - try { - if (event.isValid(objMetadata, config)) { - val migratedMetadataFields = process(objMetadata, config, httpUtil, cloudStorageUtil) - - val migratedMap = objMetadata ++ migratedMetadataFields - - event.objectType match { - case "Asset" => - event.mimeType.toLowerCase match { - case "video/mp4" | "video/webm" => - finalizeMigration(migratedMap, event, metrics, config)(defCache, neo4JUtil) - if (config.videStreamRegenerationEnabled && event.status.equalsIgnoreCase("Live")) { - logger.info("CSPNeo4jMigratorFunction :: Sending Asset For streaming URL generation: " + event.identifier) - pushStreamingUrlEvent(migratedMap, context, config)(metrics) - metrics.incCounter(config.assetVideoStreamCount) - } - case _ => finalizeMigration(migratedMap, event, metrics, config)(defCache, neo4JUtil) - } - case "Content" | "ContentImage" | "Collection" | "CollectionImage" => - event.mimeType.toLowerCase match { - case "application/vnd.ekstep.ecml-archive" | "application/vnd.ekstep.content-collection" => - updateNeo4j(migratedMap, event)(defCache, neo4JUtil, config) - logger.info("CSPNeo4jMigratorFunction :: Sending Content/Collection For cassandra migration: " + event.identifier) - context.output(config.cassandraMigrationOutputTag, event) - case _ => - finalizeMigration(migratedMap, event, metrics, config)(defCache, neo4JUtil) - if(config.liveNodeRepublishEnabled && (event.status.equalsIgnoreCase("Live") || - event.status.equalsIgnoreCase("Unlisted"))) { - pushLiveNodePublishEvent(objMetadata, context, metrics, config, config.liveContentNodePublishEventOutTag) - metrics.incCounter(config.liveContentNodePublishCount) - } - } - case "AssessmentItem" => { - updateNeo4j(migratedMap, event)(defCache, neo4JUtil, config) - logger.info("CSPNeo4jMigratorFunction :: Sending AssessmentItem For cassandra migration: " + event.identifier) - context.output(config.cassandraMigrationOutputTag, event) - } - case "Question" | "QuestionImage" => { - finalizeMigration(migratedMap, event, metrics, config)(defCache, neo4JUtil) - if(config.liveNodeRepublishEnabled && (event.status.equalsIgnoreCase("Live") || - event.status.equalsIgnoreCase("Unlisted"))) { - pushQuestionPublishEvent(objMetadata, context, metrics, config, config.liveQuestionNodePublishEventOutTag, config.liveQuestionNodePublishCount) - } - } - case "QuestionSet" | "QuestionSetImage" => { - updateNeo4j(migratedMap, event)(defCache, neo4JUtil, config) - logger.info("CSPNeo4jMigratorFunction :: Sending QuestionSet/QuestionSetImage For cassandra migration: " + event.identifier) - context.output(config.cassandraMigrationOutputTag, event) - } - case _ => finalizeMigration(migratedMap, event, metrics, config)(defCache, neo4JUtil) - } - } else { - if(!objMetadata.contains("migrationVersion")) { - // Insert into neo4j with migrationVersion as 0.5 for skipped events for easy identification - updateNeo4j(objMetadata + ("migrationVersion" -> 0.5.asInstanceOf[Number]), event)(defCache, neo4JUtil, config) - } - logger.info("CSPNeo4jMigratorFunction::processElement:: Event is not qualified for csp migration having identifier : " + event.identifier + " | objectType : " + event.objectType) - metrics.incCounter(config.skippedEventCount) - } - } catch { - case se: Exception => - logger.error("CSPNeo4jMigratorFunction :: Message processing failed for mid : " + event.mid() + " || " + event , se) - logger.error("CSPNeo4jMigratorFunction :: Error while migrating content :: " + se.getMessage) - metrics.incCounter(config.failedEventCount) - se.printStackTrace() - logger.info(s"""{ identifier: \"${objMetadata.getOrElse("identifier", "").asInstanceOf[String]}\", mimetype: \"${objMetadata.getOrElse("mimeType", "").asInstanceOf[String]}\", status: \"Failed\", stage: \"Static Migration\"}""") - // Insert into neo4j with migrationVersion as 0.1 for failed scenarios - updateNeo4j(objMetadata + ("migrationVersion" -> 0.1.asInstanceOf[Number]), event)(defCache, neo4JUtil, config) - - } - } - - -} diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/CSPCassandraMigrator.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/CSPCassandraMigrator.scala deleted file mode 100644 index c02b1b35b..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/CSPCassandraMigrator.scala +++ /dev/null @@ -1,55 +0,0 @@ -package org.sunbird.job.cspmigrator.helpers - -import com.datastax.driver.core.Row -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.util._ - -import scala.collection.JavaConverters._ - -trait CSPCassandraMigrator extends MigrationObjectReader with MigrationObjectUpdater{ - - private[this] val logger = LoggerFactory.getLogger(classOf[CSPCassandraMigrator]) - - def process(objMetadata: Map[String, AnyRef], config: CSPMigratorConfig, httpUtil: HttpUtil, cassandraUtil: CassandraUtil, cloudStorageUtil: CloudStorageUtil): Unit = { - - val objectType: String = objMetadata.getOrElse("objectType","").asInstanceOf[String] - val identifier: String = objMetadata.getOrElse("identifier", "").asInstanceOf[String] - - objectType match { - case "AssessmentItem" => - val row: Row = getAssessmentItemData(identifier, config)(cassandraUtil) - val extProps = config.getConfig.getStringList("cassandra_fields_to_migrate.assessmentitem").asScala.toList - val data: Map[String, String] = if (null != row) extProps.map(prop => prop -> row.getString(prop.toLowerCase())).toMap.filter(p => StringUtils.isNotBlank(p._2)) else Map[String, String]() - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: Fetched Cassandra data:: $data""") - val migrateData = data.flatMap(rec => { - Map(rec._1 -> StringUtils.replaceEach(rec._2, config.keyValueMigrateStrings.keySet().toArray().map(_.asInstanceOf[String]), config.keyValueMigrateStrings.values().toArray().map(_.asInstanceOf[String]))) - }) - updateAssessmentItemData(identifier, migrateData, config)(cassandraUtil) - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: Migrated Cassandra data:: $migrateData""") - case "Content" | "ContentImage" => - val ecmlBody: String = getContentBody(identifier, config)(cassandraUtil) - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: ECML Fetched body:: $ecmlBody""") - val migratedECMLBody: String = extractAndValidateUrls(identifier, ecmlBody, config, httpUtil, cloudStorageUtil) - updateContentBody(identifier, migratedECMLBody, config)(cassandraUtil) - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: ECML Migrated body:: $migratedECMLBody""") - case "Collection" | "CollectionImage" => - val collectionHierarchy: String = getCollectionHierarchy(identifier, config)(cassandraUtil) - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: Fetched Hierarchy:: $collectionHierarchy""") - if(collectionHierarchy != null && collectionHierarchy.nonEmpty) { - val migratedCollectionHierarchy: String = extractAndValidateUrls(identifier, collectionHierarchy, config, httpUtil, cloudStorageUtil, false) - updateCollectionHierarchy(identifier, migratedCollectionHierarchy, config)(cassandraUtil) - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: Migrated Hierarchy:: $migratedCollectionHierarchy""") - } - case "QuestionSet" | "QuestionSetImage" => { - val qsH: String = getQuestionSetHierarchy(identifier, config)(cassandraUtil) - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: Fetched Hierarchy:: $qsH""") - val migratedQsHierarchy: String = extractAndValidateUrls(identifier, qsH, config, httpUtil, cloudStorageUtil) - updateQuestionSetHierarchy(identifier, migratedQsHierarchy, config)(cassandraUtil) - logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: Migrated Hierarchy:: $migratedQsHierarchy""") - } - case _ => logger.info(s"""CSPCassandraMigrator:: process:: $identifier - $objectType :: NO CASSANDRA MIGRATION PERFORMED!! """) - } - } -} \ No newline at end of file diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/CSPNeo4jMigrator.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/CSPNeo4jMigrator.scala deleted file mode 100644 index 010887c75..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/CSPNeo4jMigrator.scala +++ /dev/null @@ -1,60 +0,0 @@ -package org.sunbird.job.cspmigrator.helpers - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.exception.ServerException -import org.sunbird.job.util.{CloudStorageUtil, HttpUtil} - -import scala.collection.JavaConverters._ - -trait CSPNeo4jMigrator extends MigrationObjectReader with MigrationObjectUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[CSPNeo4jMigrator]) - val streamableMimeTypes=List("video/mp4", "video/webm") - val imageAttributes=List("appIcon", "thumbnail", "posterImage") - - def process(objMetadata: Map[String, AnyRef], config: CSPMigratorConfig, httpUtil: HttpUtil, cloudStorageUtil: CloudStorageUtil): Map[String, AnyRef] = { - - // fetch the objectType. - // check if the object has only Draft OR only Live OR live/image if the objectType is content/collection. - // fetch the string replace data from config - // fetch the list of attributes for string replace from config - // check the string of each property and perform string replace. - // commit updated data to DB. - // if objectType is Asset and mimeType is video, trigger streamingUrl generation - // if objectType is content and mimeType is ECML, need to update ECML content body - // if objectType is collection, fetch hierarchy data and update cassandra data with the replaced string - // if objectType is content/collection and the node is Live, trigger the LiveNodePublisher flink job - // if objectType is AssessmentItem, migrate cassandra data as well - // update the migrationVersion of the object - - // validation of the replace URL paths to be done. If not available, Fail migration - // For Migration Failed contents set migrationVersion to 0.1 - // For collection, verify if all childNodes are having live migrated contents - REQUIRED for Draft/Image version? - - val objectType: String = objMetadata.getOrElse("objectType","").asInstanceOf[String] - val mimeType: String = objMetadata.getOrElse("mimeType","").asInstanceOf[String] - val identifier: String = objMetadata.getOrElse("identifier", "").asInstanceOf[String] - val fieldsToMigrate: List[String] = if (config.getConfig.hasPath("neo4j_fields_to_migrate."+objectType.toLowerCase())) config.getConfig.getStringList("neo4j_fields_to_migrate."+objectType.toLowerCase()).asScala.toList - else throw new ServerException("ERR_CONFIG_NOT_FOUND", "Fields to migrate configuration not found for objectType: " + objectType) - - logger.info(s"""CSPNeo4jMigrator:: process:: starting neo4j fields migration for $identifier - $objectType fields:: $fieldsToMigrate""") - val migratedMetadataFields: Map[String, String] = fieldsToMigrate.flatMap(migrateField => { - if(objMetadata.contains(migrateField) && (!migrateField.equalsIgnoreCase("streamingUrl") || !streamableMimeTypes.contains(mimeType)) && (!mimeType.equalsIgnoreCase("video/x-youtube") || imageAttributes.contains(migrateField))) { - val metadataFieldValue = objMetadata.getOrElse(migrateField, "").asInstanceOf[String] - - val tempMetadataFieldValue = handleExternalURLS(metadataFieldValue,identifier, config, httpUtil, cloudStorageUtil) - - val migrateValue: String = if(StringUtils.isNotBlank(tempMetadataFieldValue)) - StringUtils.replaceEach(tempMetadataFieldValue, config.keyValueMigrateStrings.keySet().toArray().map(_.asInstanceOf[String]), config.keyValueMigrateStrings.values().toArray().map(_.asInstanceOf[String])) - else null - if(config.copyMissingFiles && StringUtils.isNotBlank(migrateValue)) verifyFile(identifier, tempMetadataFieldValue, migrateValue, migrateField, config)(httpUtil, cloudStorageUtil) - Map(migrateField -> migrateValue) - } else Map.empty[String, String] - }).filter(record => record._1.nonEmpty).toMap[String, String] - - logger.info(s"""CSPNeo4jMigrator:: process:: $identifier - $objectType migratedMetadataFields:: $migratedMetadataFields""") - migratedMetadataFields - } -} diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/GoogleDriveUtil.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/GoogleDriveUtil.scala deleted file mode 100644 index 41fdabc3a..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/GoogleDriveUtil.scala +++ /dev/null @@ -1,84 +0,0 @@ -package org.sunbird.job.cspmigrator.helpers - -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport -import com.google.api.client.googleapis.json.GoogleJsonResponseException -import com.google.api.client.http.HttpResponseException -import com.google.api.client.json.jackson2.JacksonFactory -import com.google.api.services.drive.{Drive, DriveScopes} -import org.apache.commons.lang.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.exception.ServerException -import org.sunbird.job.util.Slug - -import java.io.{ByteArrayInputStream, File, FileOutputStream} -import java.nio.charset.Charset -import java.util - - -object GoogleDriveUtil { - private[this] val logger = LoggerFactory.getLogger("GoogleDriveUtil") - - private def initialize(config: BaseJobConfig): Drive = { - val jacksonFactory = new JacksonFactory - val gDriveAppName = config.getString("gdrive.application_name","drive-download-sunbird") - - try { - val httpTransport = GoogleNetHttpTransport.newTrustedTransport - new Drive.Builder(httpTransport, jacksonFactory, getCredentials(config)).setApplicationName(gDriveAppName).build - } catch { - case e: Exception => - logger.error("GoogleDriveUtil:: Error occurred while creating google drive client ::: " + e.getMessage, e) - throw new Exception("Error occurred while creating google drive client ::: " + e.getMessage) - } - } - - @throws[Exception] - private def getCredentials(config: BaseJobConfig): GoogleCredential = { - val scope = util.Arrays.asList(DriveScopes.DRIVE_FILE, DriveScopes.DRIVE, DriveScopes.DRIVE_METADATA) - val gDriveCredentials = config.getString("g_service_acct_cred","") - val credentialsStream = new ByteArrayInputStream(gDriveCredentials.getBytes(Charset.forName("UTF-8"))) - val credential = GoogleCredential.fromStream(credentialsStream).createScoped(scope) - credential - } - - @throws[Exception] - def downloadFile(fileId: String, saveDir: String)(implicit config: BaseJobConfig): File = { - var file: File = null - try { - val drive = initialize(config) - logger.info("GoogleDriveUtil:: downloadFile:: drive: " + drive + " || drive.files: " + drive.files().list()) - val getFile = drive.files.get(fileId) - logger.info("GoogleDriveUtil:: downloadFile:: getFile: " + getFile) - getFile.setFields("id,name,size,owners,mimeType,properties,permissionIds,webContentLink") - logger.info("GoogleDriveUtil:: downloadFile:: getFile.setFields: ") - val googleDriveFile = getFile.execute - logger.info("GoogleDriveUtil :: downloadFile ::: Drive File Details:: " + googleDriveFile) - val fileName = Slug.makeSlug(googleDriveFile.getName) - logger.info("GoogleDriveUtil :: downloadFile ::: Slug fileName :: " + fileName) - val saveFile = new File(saveDir) - if (!saveFile.exists) saveFile.mkdirs - val saveFilePath: String = saveDir + File.separator + fileName - logger.info("GoogleDriveUtil :: downloadFile :: File Id :" + fileId + " | Save File Path: " + saveFilePath) - val outputStream = new FileOutputStream(saveFilePath) - getFile.executeMediaAndDownloadTo(outputStream) - outputStream.close() - file = new File(saveFilePath) - file = Slug.createSlugFile(file) - logger.info("GoogleDriveUtil :: downloadFile :: File Downloaded Successfully. Sluggified File Name: " + file.getAbsolutePath) - return file - } catch { - case ge: GoogleJsonResponseException => logger.error("GoogleDriveUtil :: downloadFile :: GoogleJsonResponseException :: Error Occurred while downloading file having id " + fileId + " | Error is ::" + ge.getDetails.toString, ge) - throw new ServerException("ERR_INVALID_UPLOAD_FILE_URL", "Invalid Response Received From Google API for file Id : " + fileId + " | Error is : " + ge.getDetails.toString) - case he: HttpResponseException => logger.error("GoogleDriveUtil :: downloadFile :: HttpResponseException :: Error Occurred while downloading file having id " + fileId + " | Error is ::" + he.getContent, he) - throw new ServerException("ERR_INVALID_UPLOAD_FILE_URL", "Invalid Response Received From Google API for file Id : " + fileId + " | Error is : " + he.getContent) - case e: Exception => - logger.error("GoogleDriveUtil :: downloadFile :: Exception :: Error Occurred While Downloading Google Drive File having Id " + fileId + " : " + e.getMessage, e) - if (e.isInstanceOf[ServerException]) throw e - else throw new ServerException("ERR_INVALID_UPLOAD_FILE_URL", "Invalid Response Received From Google API for file Id : " + fileId + " | Error is : " + e.getMessage) - } - file - } - -} diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/MigrationObjectReader.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/MigrationObjectReader.scala deleted file mode 100644 index a0c69b7f8..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/MigrationObjectReader.scala +++ /dev/null @@ -1,122 +0,0 @@ -package org.sunbird.job.cspmigrator.helpers - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} -import org.apache.commons.lang3 -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.cspmigrator.domain.Event -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} -import java.util.UUID - -import org.apache.flink.streaming.api.scala.OutputTag - -import scala.collection.JavaConverters._ - -trait MigrationObjectReader { - - private[this] val logger = LoggerFactory.getLogger(classOf[MigrationObjectReader]) - - def getMetadata(identifier: String)(implicit neo4JUtil: Neo4JUtil): Map[String, AnyRef] = { - val metaData = neo4JUtil.getNodeProperties(identifier).asScala.toMap - val id = metaData.getOrElse("IL_UNIQUE_ID", identifier).asInstanceOf[String] - val objType = metaData.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String] - logger.info("MigrationObjectReader:: getMetadata:: identifier: " + identifier + " with objType: " + objType) - metaData ++ Map[String, AnyRef]("identifier" -> id, "objectType" -> objType) - ("IL_UNIQUE_ID", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE") - } - - def getContentBody(identifier: String, config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): String = { - // fetch content body from cassandra - val selectId = QueryBuilder.select() - selectId.fcall("blobAsText", QueryBuilder.column("body")).as("body") - val selectWhereId: Select.Where = selectId.from(config.contentKeyspaceName, config.contentTableName).where().and(QueryBuilder.eq("content_id", identifier)) - logger.info("MigrationObjectReader:: getContentBody:: ECML Body Fetch Query :: " + selectWhereId.toString) - val rowId = cassandraUtil.findOne(selectWhereId.toString) - if (null != rowId && null != rowId.getString("body") ) rowId.getString("body") else "" - } - - def getAssessmentItemData(identifier: String, config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): Row = { - logger.info("MigrationObjectReader ::: getAssessmentItemData ::: Reading Question External Data For : " + identifier) - val select = QueryBuilder.select() - val extProps = config.getConfig.getStringList("cassandra_fields_to_migrate.assessmentitem").asScala.toList - extProps.foreach(prop => select.fcall("blobAsText", QueryBuilder.column(prop.toLowerCase())).as(prop.toLowerCase())) - val selectWhere: Select.Where = select.from(config.contentKeyspaceName, config.assessmentTableName).where().and(QueryBuilder.eq("question_id", identifier)) - logger.info("MigrationObjectReader ::: getAssessmentItemData:: Cassandra Fetch Query :: " + selectWhere.toString) - cassandraUtil.findOne(selectWhere.toString) - } - - def getCollectionHierarchy(identifier: String, config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): String = { - val selectId = QueryBuilder.select().column("hierarchy") - val selectWhereId: Select.Where = selectId.from(config.hierarchyKeyspaceName, config.hierarchyTableName).where().and(QueryBuilder.eq("identifier", identifier)) - logger.info("MigrationObjectReader:: getCollectionHierarchy:: Hierarchy Fetch Query :: " + selectWhereId.toString) - val rowId = cassandraUtil.findOne(selectWhereId.toString) - if (null != rowId && null != rowId.getString("hierarchy")) rowId.getString("hierarchy") else "" - } - - def getQuestionSetHierarchy(identifier: String, config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): String = { - val selectId = QueryBuilder.select().column("hierarchy") - val selectWhereId: Select.Where = selectId.from(config.qsHierarchyKeyspaceName, config.qsHierarchyTableName).where().and(QueryBuilder.eq("identifier", identifier)) - logger.info("MigrationObjectReader:: getQuestionSetHierarchy:: Hierarchy Fetch Query :: " + selectWhereId.toString) - val rowId = cassandraUtil.findOne(selectWhereId.toString) - if (null != rowId && null != rowId.getString("hierarchy")) rowId.getString("hierarchy") else "" - } - - - def pushStreamingUrlEvent(objMetadata: Map[String, AnyRef], context: ProcessFunction[Event, String]#Context, config: CSPMigratorConfig)(implicit metrics: Metrics): Unit = { - val event = getStreamingEvent(objMetadata, config) - context.output(config.generateVideoStreamingOutTag, event) - } - - def getStreamingEvent(objMetadata: Map[String, AnyRef], config: CSPMigratorConfig): String = { - val ets = System.currentTimeMillis - val mid = s"""LP.$ets.${UUID.randomUUID}""" - val channelId = objMetadata.getOrElse("channel", "").asInstanceOf[String] - val ver = objMetadata.getOrElse("versionKey", "").asInstanceOf[String] - val artifactUrl = objMetadata.getOrElse("artifactUrl", "").asInstanceOf[String] - val contentType = objMetadata.getOrElse("contentType", "").asInstanceOf[String] - val mimeType = objMetadata.getOrElse("mimeType", "").asInstanceOf[String] - val status = objMetadata.getOrElse("status", "").asInstanceOf[String] - val identifier = objMetadata.getOrElse("identifier", "").asInstanceOf[String] - val pkgVersion = objMetadata.getOrElse("pkgVersion", "").asInstanceOf[Number] - val event = s"""{"eid":"BE_JOB_REQUEST", "ets": $ets, "mid": "$mid", "actor": {"id": "Live Video Stream Generator", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"}, "channel":"$channelId","env":"${config.jobEnv}"},"object":{"ver":"$ver","id":"$identifier"},"edata": {"action":"live-video-stream-generate","iteration":1,"identifier":"$identifier","channel":"$channelId","artifactUrl":"$artifactUrl","mimeType":"$mimeType","contentType":"$contentType","pkgVersion":$pkgVersion,"status":"$status"}}""".stripMargin - logger.info(s"MigrationObjectReader :: Asset Video Streaming Event for identifier $identifier is : $event") - event - } - - def pushLiveNodePublishEvent(objMetadata: Map[String, AnyRef], context: ProcessFunction[Event, String]#Context, metrics: Metrics, config: CSPMigratorConfig, tag: OutputTag[String]): Unit = { - val epochTime = System.currentTimeMillis - val identifier = objMetadata.getOrElse("identifier", "").asInstanceOf[String] - val pkgVersion = objMetadata.getOrElse("pkgVersion", "").asInstanceOf[Number] - val objectType = objMetadata.getOrElse("objectType", "").asInstanceOf[String] - val contentType = objMetadata.getOrElse("contentType", "").asInstanceOf[String] - val mimeType = objMetadata.getOrElse("mimeType", "").asInstanceOf[String] - val status = objMetadata.getOrElse("status", "").asInstanceOf[String] - val publishType = if(status.equalsIgnoreCase("Live")) "Public" else "Unlisted" - - val event = s"""{"eid":"BE_JOB_REQUEST","ets":$epochTime,"mid":"LP.$epochTime.${UUID.randomUUID()}","actor":{"id":"content-republish","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"sunbird","env":"${config.jobEnv}"},"object":{"ver":"$pkgVersion","id":"$identifier"},"edata":{"publish_type":"$publishType","metadata":{"identifier":"$identifier", "mimeType":"$mimeType","objectType":"$objectType","lastPublishedBy":"System","pkgVersion":$pkgVersion},"action":"republish","iteration":1,"contentType":"$contentType"}}""" - context.output(tag, event) - metrics.incCounter(config.liveContentNodePublishCount) - logger.info("MigrationObjectReader :: Live content publish triggered for " + identifier) - logger.info("MigrationObjectReader :: Live content publish event: " + event) - } - - def pushQuestionPublishEvent(objMetadata: Map[String, AnyRef], context: ProcessFunction[Event, String]#Context, metrics: Metrics, config: CSPMigratorConfig, tag: OutputTag[String], countMetric: String): Unit = { - val epochTime = System.currentTimeMillis - val identifier = objMetadata.getOrElse("identifier", "").asInstanceOf[String] - val pkgVersion = objMetadata.getOrElse("pkgVersion", "").asInstanceOf[Number] - val objectType = objMetadata.getOrElse("objectType", "").asInstanceOf[String] - val mimeType = objMetadata.getOrElse("mimeType", "").asInstanceOf[String] - val status = objMetadata.getOrElse("status", "").asInstanceOf[String] - val publishType = if(status.equalsIgnoreCase("Live")) "Public" else "Unlisted" - val channel = objMetadata.getOrElse("channel", "").asInstanceOf[String] - val lastPublishedBy = objMetadata.getOrElse("lastPublishedBy", "System").asInstanceOf[String] - val event = s"""{"eid":"BE_JOB_REQUEST","ets":$epochTime,"mid":"LP.$epochTime.${UUID.randomUUID()}","actor":{"id":"question-republish","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":"${channel}","env":"${config.jobEnv}"},"object":{"ver":"$pkgVersion","id":"$identifier"},"edata":{"publish_type":"$publishType","metadata":{"identifier":"$identifier", "mimeType":"$mimeType","objectType":"$objectType","lastPublishedBy":"${lastPublishedBy}","pkgVersion":$pkgVersion},"action":"republish","iteration":1}}""" - context.output(tag, event) - metrics.incCounter(countMetric) - logger.info(s"MigrationObjectReader :: Live ${objectType} publish triggered for " + identifier) - logger.info(s"MigrationObjectReader :: Live ${objectType} publish event: " + event) - } - -} diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/MigrationObjectUpdater.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/MigrationObjectUpdater.scala deleted file mode 100644 index 890c7b4cc..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/MigrationObjectUpdater.scala +++ /dev/null @@ -1,283 +0,0 @@ -package org.sunbird.job.cspmigrator.helpers - -import com.datastax.driver.core.querybuilder.QueryBuilder -import org.apache.commons.io.{FileUtils, FilenameUtils} -import org.apache.commons.lang3.StringUtils -import org.neo4j.driver.v1.StatementResult -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.cspmigrator.domain.Event -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.{InvalidInputException, ServerException} -import org.sunbird.job.util.CSPMetaUtil.updateAbsolutePath -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, JSONUtil, Neo4JUtil, ScalaJsonUtil, Slug} -import org.sunbird.job.util.CSPMetaUtil.updateAbsolutePath -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, JSONUtil, Neo4JUtil, ScalaJsonUtil, Slug} - -import scala.collection.JavaConverters._ -import scala.collection.JavaConverters._ -import java.io.{File, IOException} -import java.net.URL -import java.util - -trait MigrationObjectUpdater extends URLExtractor { - - private[this] val logger = LoggerFactory.getLogger(classOf[MigrationObjectUpdater]) - - def updateContentBody(identifier: String, ecmlBody: String, config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val updateQuery = QueryBuilder.update(config.contentKeyspaceName, config.contentTableName) - .where(QueryBuilder.eq("content_id", identifier)) - .`with`(QueryBuilder.set("body", QueryBuilder.fcall("textAsBlob", ecmlBody))) - logger.info(s"""MigrationObjectUpdater:: updateContentBody:: Updating Content Body in Cassandra For $identifier : ${updateQuery.toString}""") - val result = cassandraUtil.upsert(updateQuery.toString) - if (result) logger.info(s"""MigrationObjectUpdater:: updateContentBody:: Content Body Updated Successfully For $identifier""") - else { - logger.error(s"""MigrationObjectUpdater:: updateContentBody:: Content Body Update Failed For $identifier""") - throw new InvalidInputException(s"""Content Body Update Failed For $identifier""") - } - } - - def updateAssessmentItemData(identifier: String, updatedData: Map[String, String], config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val updateQuery = QueryBuilder.update(config.contentKeyspaceName, config.assessmentTableName) - .where(QueryBuilder.eq("question_id", identifier)) - .`with`(QueryBuilder.set("body", QueryBuilder.fcall("textAsBlob", updatedData.getOrElse("body", null)))) - .and(QueryBuilder.set("question", QueryBuilder.fcall("textAsBlob", updatedData.getOrElse("question", null)))) - .and(QueryBuilder.set("editorstate", QueryBuilder.fcall("textAsBlob", updatedData.getOrElse("editorstate", null)))) - .and(QueryBuilder.set("solutions", QueryBuilder.fcall("textAsBlob", updatedData.getOrElse("solutions", null)))) - - logger.info(s"""MigrationObjectUpdater:: updateAssessmentItemData:: Updating Assessment Body in Cassandra For $identifier : ${updateQuery.toString}""") - val result = cassandraUtil.upsert(updateQuery.toString) - if (result) logger.info(s"""MigrationObjectUpdater:: updateAssessmentItemData:: Assessment Body Updated Successfully For $identifier""") - else { - logger.error(s"""MigrationObjectUpdater:: updateAssessmentItemData:: Assessment Body Update Failed For $identifier""") - throw new InvalidInputException(s"""Assessment Body Update Failed For $identifier""") - } - } - - def updateCollectionHierarchy(identifier: String, hierarchy: String, config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val updateQuery = QueryBuilder.update(config.hierarchyKeyspaceName, config.hierarchyTableName) - .where(QueryBuilder.eq("identifier", identifier)) - .`with`(QueryBuilder.set("hierarchy", hierarchy)) - logger.info(s"""MigrationObjectUpdater:: updateCollectionHierarchy:: Updating Hierarchy in Cassandra For $identifier : ${updateQuery.toString}""") - val result = cassandraUtil.upsert(updateQuery.toString) - if (result) logger.info(s"""MigrationObjectUpdater:: updateCollectionHierarchy:: Hierarchy Updated Successfully For $identifier""") - else { - logger.error(s"""MigrationObjectUpdater:: updateCollectionHierarchy:: Hierarchy Update Failed For $identifier""") - throw new InvalidInputException(s"""Hierarchy Update Failed For $identifier""") - } - } - - def updateQuestionSetHierarchy(identifier: String, hierarchy: String, config: CSPMigratorConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val updateQuery = QueryBuilder.update(config.qsHierarchyKeyspaceName, config.qsHierarchyTableName) - .where(QueryBuilder.eq("identifier", identifier)) - .`with`(QueryBuilder.set("hierarchy", hierarchy)) - logger.info(s"""MigrationObjectUpdater:: updateQuestionSetHierarchy:: Updating Hierarchy in Cassandra For $identifier : ${updateQuery.toString}""") - val result = cassandraUtil.upsert(updateQuery.toString) - if (result) logger.info(s"""MigrationObjectUpdater:: updateQuestionSetHierarchy:: Hierarchy Updated Successfully For $identifier""") - else { - logger.error(s"""MigrationObjectUpdater:: updateQuestionSetHierarchy:: Hierarchy Update Failed For $identifier""") - throw new InvalidInputException(s"""Hierarchy Update Failed For $identifier""") - } - } - - def updateNeo4j(updatedMetadata: Map[String, AnyRef], event: Event)(definitionCache: DefinitionCache, neo4JUtil: Neo4JUtil, config: CSPMigratorConfig): Unit = { - logger.info(s"""MigrationObjectUpdater:: process:: ${event.identifier} - ${event.objectType} updated fields data:: $updatedMetadata""") - val metadataUpdateQuery = metaDataQuery(event.objectType, updatedMetadata)(definitionCache, config) - val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"${event.identifier}"}) SET $metadataUpdateQuery return n;""" - logger.info(s"""MigrationObjectUpdater:: process:: ${event.identifier} - ${event.objectType} updated fields :: Query: """ + query) - val sResult: StatementResult = neo4JUtil.executeQuery(query) - if(sResult !=null) logger.info("MigrationObjectUpdater:: process:: sResult:: " + sResult) - logger.info("MigrationObjectUpdater:: process:: static fields migration completed for " + event.identifier) - } - - - def extractAndValidateUrls(identifier: String, contentString: String, config: CSPMigratorConfig, httpUtil: HttpUtil, cloudStorageUtil: CloudStorageUtil, isContent: Boolean = true): String = { - val extractedUrls: List[String] = extractUrls(contentString) - logger.info("MigrationObjectUpdater::extractAndValidateUrls:: extractedUrls : " + extractedUrls) - if(extractedUrls.nonEmpty) { - var tempContentString: String = contentString - - val migratedString = if(config.copyMissingFiles) { - extractedUrls.toSet[String].foreach(urlString => { - // TODO: call a method to validate the url, upload to cloud set the url to migrated value - val tempUrlString = if(isContent) handleExternalURLS(urlString, identifier, config, httpUtil, cloudStorageUtil) else urlString - - config.keyValueMigrateStrings.keySet().toArray().map(migrateDomain => { - if (StringUtils.isNotBlank(tempUrlString) && tempUrlString.contains(migrateDomain.asInstanceOf[String])) { - tempContentString = StringUtils.replace(tempContentString, urlString, tempUrlString) - - val migrateValue: String = StringUtils.replaceEach(tempUrlString, config.keyValueMigrateStrings.keySet().toArray().map(_.asInstanceOf[String]), config.keyValueMigrateStrings.values().toArray().map(_.asInstanceOf[String])) - verifyFile(identifier, tempUrlString, migrateValue, migrateDomain.asInstanceOf[String], config)(httpUtil, cloudStorageUtil) - } - }) - }) - tempContentString - }else{ - extractedUrls.toSet[String].foreach(urlString => { - logger.info("MigrationObjectUpdater::extractAndValidateUrls:: urlString : " + urlString) - val tempUrlString = if(isContent) handleExternalURLS(urlString, identifier, config, httpUtil, cloudStorageUtil) else urlString - logger.info("MigrationObjectUpdater::extractAndValidateUrls:: tempUrlString : " + tempUrlString) - tempContentString = if(StringUtils.isNotBlank(tempUrlString)) StringUtils.replace(tempContentString, urlString, tempUrlString) else StringUtils.replace(tempContentString, urlString, "") - }) - tempContentString - } - - StringUtils.replaceEach(migratedString, config.keyValueMigrateStrings.keySet().toArray().map(_.asInstanceOf[String]), config.keyValueMigrateStrings.values().toArray().map(_.asInstanceOf[String])) - } else contentString - } - - - def downloadFile(downloadPath: String, fileUrl: String): File = try { - createDirectory(downloadPath) - var file = new File(downloadPath + File.separator + Slug.makeSlug(FilenameUtils.getName(fileUrl))) - FileUtils.copyURLToFile(new URL(fileUrl), file) - file = Slug.createSlugFile(file) - logger.info("MigrationObjectUpdater:: downloadFile:: external URL file download status:: " + file.exists() + " || " + file.getAbsolutePath) - file - } catch { - case e: IOException => logger.info("ERR_INVALID_FILE_URL", "File not found in the old path to migrate: " + fileUrl) - null - } - - private def createDirectory(directoryName: String): Unit = { - val theDir = new File(directoryName) - if (!theDir.exists) theDir.mkdirs - } - - def finalizeMigration(migratedMap: Map[String, AnyRef], event: Event, metrics: Metrics, config: CSPMigratorConfig)(defCache: DefinitionCache, neo4JUtil: Neo4JUtil): Unit = { - updateNeo4j(migratedMap + ("migrationVersion" -> config.migrationVersion.asInstanceOf[AnyRef]), event)(defCache, neo4JUtil, config) - logger.info("MigrationObjectUpdater::finalizeMigration:: CSP migration operation completed for : " + event.identifier) - metrics.incCounter(config.successEventCount) - } - - def verifyFile(identifier: String, originalUrl: String, migrateUrl: String, migrateDomain: String, config: CSPMigratorConfig)(implicit httpUtil: HttpUtil, cloudStorageUtil: CloudStorageUtil): Unit = { - val updateMigrateUrl = updateAbsolutePath(migrateUrl)(config) - logger.info("MigrationObjectUpdater::verifyFile:: originalUrl :: " + originalUrl + " || updateMigrateUrl:: " + updateMigrateUrl + " || identifier: " + identifier) - if(httpUtil.getSize(updateMigrateUrl) <= 0) { - if (config.copyMissingFiles) { - if(FilenameUtils.getExtension(originalUrl) != null && !FilenameUtils.getExtension(originalUrl).isBlank && FilenameUtils.getExtension(originalUrl).nonEmpty) { - try { - // code to download file from old cloud path and upload to new cloud path - val downloadedFile: File = downloadFile(s"/tmp/$identifier", originalUrl) - val exDomain: String = originalUrl.replace(migrateDomain, "") - val folderName: String = exDomain.substring(1, exDomain.indexOf(FilenameUtils.getName(originalUrl)) - 1) - cloudStorageUtil.uploadFile(folderName, downloadedFile) - } catch { - case f: IllegalArgumentException => logger.info("ERR_INVALID_FILE_URL", "File is not valid : " + originalUrl + " || identifier: " + identifier) - } - } - } else throw new ServerException("ERR_NEW_PATH_NOT_FOUND", "File not found in the new path to migrate: " + updateMigrateUrl) - } - } - - - def metaDataQuery(objectType: String, objMetadata: Map[String, AnyRef])(definitionCache: DefinitionCache, config: CSPMigratorConfig): String = { - val version = if(objectType.equalsIgnoreCase("itemset")) "2.0" else "1.0" - val definition = definitionCache.getDefinition(objectType, version, config.definitionBasePath) - val metadata = objMetadata - ("IL_UNIQUE_ID", "identifier", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE", "pkgVersion", "lastStatusChangedOn", "lastUpdatedOn", "status", "objectType", "publish_type") - metadata.map(prop => { - if (null == prop._2) s"n.${prop._1}=${prop._2}" - else if (definition.objectTypeProperties.contains(prop._1)) { - prop._2 match { - case _: Map[String, AnyRef] => - val strValue = JSONUtil.serialize(ScalaJsonUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: util.Map[String, AnyRef] => - val strValue = JSONUtil.serialize(JSONUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _ => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - } - } else { - prop._2 match { - case _: Map[String, AnyRef] => - val strValue = JSONUtil.serialize(ScalaJsonUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: util.Map[String, AnyRef] => - val strValue = JSONUtil.serialize(JSONUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: List[String] => - val strValue = ScalaJsonUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - case _: util.List[String] => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - case _ => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - } - } - }).mkString(",") - } - - @throws[Exception] - private def getFile(identifier: String, fileUrl: String, config: CSPMigratorConfig, httpUtil: HttpUtil): File = { - try { - val fileId = fileUrl.split("download&id=")(1) - if (StringUtils.isBlank(fileId)) { - logger.info("Invalid fileUrl received for : " + identifier + " | fileUrl : " + fileUrl) - null - } else { - GoogleDriveUtil.downloadFile(fileId, getBasePath(identifier, config))(config) - } - } catch { - case e: ServerException =>{ - logger.info("Invalid fileUrl received for : " + identifier + " | fileUrl : " + fileUrl + "Exception is : " + e.getMessage) - null - } - case ex: Exception => { - logger.info("Invalid fileUrl received for : " + identifier + " | fileUrl : " + fileUrl + "Exception is : " + ex.getMessage) - null - } - } - } - - private def getBasePath(objectId: String, config: CSPMigratorConfig): String = { - if (StringUtils.isNotBlank(objectId)) config.temp_file_location + File.separator + objectId + File.separator + "_temp_" + System.currentTimeMillis - else config.temp_file_location + File.separator + "_temp_" + System.currentTimeMillis - } - - def handleExternalURLS(fileUrl: String, contentId: String, config: CSPMigratorConfig, httpUtil: HttpUtil, cloudStorageUtil: CloudStorageUtil): String = { - val validCSPSource: List[String] = config.config.getStringList("cloudstorage.write_base_path").asScala.toList - val relativePathPrefix: String = config.config.getString("cloudstorage.relative_path_prefix") - - if (StringUtils.isNotBlank(fileUrl) && !fileUrl.contains(relativePathPrefix) && !validCSPSource.exists(writeURL=> fileUrl.contains(writeURL))) { - try { - val file = if (fileUrl.contains("drive.google.com")) getFile(contentId, fileUrl, config, httpUtil) else downloadFile(getBasePath(contentId, config), fileUrl) - logger.info("MigrationObjectUpdater :: update :: Icon downloaded for : " + contentId + " | appIconUrl : " + fileUrl) - - if (null == file || !file.exists) { - logger.info("Error Occurred while downloading appIcon file for " + contentId + " | File Url : " + fileUrl) - null - } else { - val urls = uploadArtifact(file, contentId, config, cloudStorageUtil) - val url = if (null != urls && StringUtils.isNotBlank(urls(1))) { - val blobUrl = urls(1) - logger.info("CSPNeo4jMigrator :: handleExternalURLS :: Icon Uploaded Successfully to cloud for : " + contentId + " | appIconUrl : " + fileUrl + " | appIconBlobUrl : " + blobUrl) - FileUtils.deleteQuietly(file) - blobUrl - } else null - url - } - } catch { - case f: Exception => logger.info("ERR_INVALID_FILE_URL", "File is not valid to migrate: " + fileUrl + " || identifier: " + contentId) - fileUrl - } - } else fileUrl - } - - private def uploadArtifact(uploadedFile: File, identifier: String, config: CSPMigratorConfig, cloudStorageUtil: CloudStorageUtil): Array[String] = { - try { - var folder = config.contentFolder - folder = folder + "/" + Slug.makeSlug(identifier, isTransliterate = true) + "/" + config.artifactFolder - cloudStorageUtil.uploadFile(folder, uploadedFile, Option(true)) - } catch { - case e: Exception => e.printStackTrace() - logger.info("MigrationObjectUpdater :: uploadArtifact :: Exception occurred while uploading artifact for : " + identifier + "Exception is : " + e.getMessage) - throw new ServerException("ERR_CONTENT_UPLOAD_FILE", "Error while uploading the File.", e) - } - } - -} diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/URLExtractor.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/URLExtractor.scala deleted file mode 100644 index 907e57014..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/helpers/URLExtractor.scala +++ /dev/null @@ -1,34 +0,0 @@ -package org.sunbird.job.cspmigrator.helpers - -import java.util -import java.util.regex.Pattern -import scala.collection.JavaConverters._ - -trait URLExtractor { - - // Function to extract all the URL// Function to extract all the URL from the string - def extractUrls(str: String): List[String] = { - // Creating an empty ArrayList - val list = new util.ArrayList[String] - - // Regular Expression to extract URL from the string - val regex = "\\b((?:https?):" + "//[-a-zA-Z0-9+&@#/%?=" + "~_|!:, .;]*[-a-zA-Z0-9+" + "&@#/%=~_|])" //(?<=")https?:\/\/[^\"]+ - // Compile the Regular Expression - val p = Pattern.compile(regex, Pattern.CASE_INSENSITIVE) - // Find the match between string and the regular expression - val m = p.matcher(str) - // Find the next subsequence of the input subsequence that find the pattern - while (m.find) { - // Find the substring from the first index of match result to the last index of match result and add in the list - list.add(str.substring(m.start(0), m.end(0))) - } - // IF there no URL present - if (list.size == 0) { - System.out.println("-1") - return List.empty[String] - } - // Print all the URLs stored - - list.asScala.toList - } -} \ No newline at end of file diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/task/CSPMigratorConfig.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/task/CSPMigratorConfig.scala deleted file mode 100644 index 8fa4d6523..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/task/CSPMigratorConfig.scala +++ /dev/null @@ -1,91 +0,0 @@ -package org.sunbird.job.cspmigrator.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.cspmigrator.domain.Event - -import java.util - -class CSPMigratorConfig(override val config: Config) extends BaseJobConfig(config, "csp-migrator") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val contentAutoCreatorTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val kafkaFailedTopic: String = config.getString("kafka.failed.topic") - val liveVideoStreamingTopic: String = config.getString("kafka.live_video_stream.topic") - val liveContentNodeRepublishTopic: String = config.getString("kafka.live_content_node_republish.topic") - val liveQuestionNodeRepublishTopic: String = config.getString("kafka.live_question_node_republish.topic") - - val jobEnv: String = config.getString("job.env") - - val cassandraMigrationOutputTag: OutputTag[Event] = OutputTag[Event]("csp-cassandra-migration") - - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - val cassandraMigratorParallelism: Int = if (config.hasPath("task.cassandra-migrator.parallelism")) - config.getInt("task.cassandra-migrator.parallelism") else 1 - - // Metric List - val totalEventsCount = "csp-total-message-count" - val successEventCount = "csp-success-message-count" - val failedEventCount = "csp-failed-message-count" - val skippedEventCount = "csp-skipped-message-count" - val errorEventCount = "csp-error-message-count" - val liveContentNodePublishCount = "live-content-node-publish-count" - val assetVideoStreamCount = "asset-video-stream-count" - val liveQuestionNodePublishCount = "live-question-node-publish-count" - val liveQuestionSetNodePublishCount = "live-questionset-node-publish-count" - - // Consumers - val eventConsumer = "csp-migrator-consumer" - val cspMigratorFunction = "csp-migrator-process" - val cspMigratorEventProducer = "csp-migrator-producer" - val cassandraMigratorFunction = "csp-cassandra-migrator-process" - - // Tags - val contentAutoCreatorOutputTag: OutputTag[Event] = OutputTag[Event]("csp-migrator") - val failedEventOutTag: OutputTag[String] = OutputTag[String]("csp-migrator-failed-event") - val generateVideoStreamingOutTag: OutputTag[String] = OutputTag[String]("live-video-streaming-generator-request") - val liveContentNodePublishEventOutTag: OutputTag[String] = OutputTag[String]("live-content-node-republish-request") - val liveQuestionSetNodePublishEventOutTag: OutputTag[String] = OutputTag[String]("live-questionset-node-republish-request") - val liveQuestionNodePublishEventOutTag: OutputTag[String] = OutputTag[String]("live-question-node-republish-request") - val liveCollectionNodePublishEventOutTag: OutputTag[String] = OutputTag[String]("live-collection-node-republish-request") - - val configVersion = "1.0" - - // Cassandra Configurations - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - val contentKeyspaceName: String = config.getString("content.keyspace") - val contentTableName: String = config.getString("content.content_table") - val assessmentTableName: String = config.getString("content.assessment_table") - val hierarchyKeyspaceName: String = config.getString("hierarchy.keyspace") - val hierarchyTableName: String = config.getString("hierarchy.table") - val qsHierarchyKeyspaceName: String = config.getString("questionset.hierarchy.keyspace") - val qsHierarchyTableName: String = config.getString("questionset.hierarchy.table") - - // Neo4J Configurations - val graphRoutePath: String = config.getString("neo4j.routePath") - val graphName: String = config.getString("neo4j.graph") - - val fieldsToMigrate: util.Map[String, AnyRef] = if(config.hasPath("neo4j_fields_to_migrate")) config.getAnyRef("neo4j_fields_to_migrate").asInstanceOf[util.Map[String, AnyRef]] else new util.HashMap[String, AnyRef]() - val keyValueMigrateStrings: util.Map[String, String] = config.getAnyRef("key_value_strings_to_migrate").asInstanceOf[util.Map[String, String]] - val migrationVersion: Double = if(config.hasPath("migrationVersion")) config.getDouble("migrationVersion") else 1.0 - - val videStreamRegenerationEnabled: Boolean = if(config.hasPath("video_stream_regeneration_enable")) config.getBoolean("video_stream_regeneration_enable") else true - val liveNodeRepublishEnabled: Boolean = if(config.hasPath("live_node_republish_enable")) config.getBoolean("live_node_republish_enable") else true - val copyMissingFiles: Boolean = if(config.hasPath("copy_missing_files_to_cloud")) config.getBoolean("copy_missing_files_to_cloud") else true - - val definitionBasePath: String = if (config.hasPath("schema.basePath")) config.getString("schema.basePath") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - - val contentFolder: String = if (config.hasPath("cloud_storage.folder.content")) config.getString("cloud_storage.folder.content") else "content" - val artifactFolder: String = if (config.hasPath("cloud_storage.folder.artifact")) config.getString("cloud_storage.folder.artifact") else "artifact" - val temp_file_location: String = if (config.hasPath("download_path")) config.getString("download_path") else "/tmp" - def getConfig: Config = config -} diff --git a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/task/CSPMigratorStreamTask.scala b/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/task/CSPMigratorStreamTask.scala deleted file mode 100644 index 81cc189cf..000000000 --- a/csp-migrator/src/main/scala/org/sunbird/job/cspmigrator/task/CSPMigratorStreamTask.scala +++ /dev/null @@ -1,65 +0,0 @@ -package org.sunbird.job.cspmigrator.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.cspmigrator.domain.Event -import org.sunbird.job.cspmigrator.functions.{CSPNeo4jMigratorFunction, CSPCassandraMigratorFunction} -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - -import java.io.File -import java.util - - -class CSPMigratorStreamTask(config: CSPMigratorConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val processStreamTask = env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.eventConsumer) - .uid(config.eventConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new CSPNeo4jMigratorFunction(config, httpUtil)) - .name(config.cspMigratorFunction) - .uid(config.cspMigratorFunction) - .setParallelism(config.parallelism) - - - val cassandraStream = processStreamTask.getSideOutput(config.cassandraMigrationOutputTag).process(new CSPCassandraMigratorFunction(config, httpUtil)) - .name(config.cassandraMigratorFunction).uid(config.cassandraMigratorFunction).setParallelism(config.cassandraMigratorParallelism) - - processStreamTask.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaFailedTopic)) - processStreamTask.getSideOutput(config.generateVideoStreamingOutTag).addSink(kafkaConnector.kafkaStringSink(config.liveVideoStreamingTopic)) - processStreamTask.getSideOutput(config.liveContentNodePublishEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.liveContentNodeRepublishTopic)) - processStreamTask.getSideOutput(config.liveQuestionNodePublishEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.liveQuestionNodeRepublishTopic)) - - cassandraStream.getSideOutput(config.liveCollectionNodePublishEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.liveContentNodeRepublishTopic)) - cassandraStream.getSideOutput(config.liveQuestionSetNodePublishEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.liveQuestionNodeRepublishTopic)) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object CSPMigratorStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("csp-migrator.conf").withFallback(ConfigFactory.systemEnvironment())) - val CSPMigratorConfig = new CSPMigratorConfig(config) - val kafkaUtil = new FlinkKafkaConnector(CSPMigratorConfig) - val httpUtil = new HttpUtil - val task = new CSPMigratorStreamTask(CSPMigratorConfig, kafkaUtil, httpUtil) - task.process() - } -} - -// $COVERAGE-ON$ diff --git a/csp-migrator/src/test/resources/base-test.conf b/csp-migrator/src/test/resources/base-test.conf deleted file mode 100644 index e03e30144..000000000 --- a/csp-migrator/src/test/resources/base-test.conf +++ /dev/null @@ -1,54 +0,0 @@ -kafka { - broker-servers = "localhost:9093" - zookeeper = "localhost:2183" - map.input.topic = "local.telemetry.map.input" - map.output.topic = "local.telemetry.map.output" - string.input.topic = "local.telemetry.string.input" - string.output.topic = "local.telemetry.string.output" - jobRequest.input.topic = "local.jobrequest.input" - jobRequest.output.topic = "local.jobrequest.output" - groupId = "test-consumer-group" - auto.offset.reset = "earliest" - producer { - max-request-size = 102400 - } -} - -task { - checkpointing.compressed = true - checkpointing.pause.between.seconds = 30000 - checkpointing.interval = 60000 - restart-strategy.attempts = 1 - restart-strategy.delay = 10000 - parallelism = 1 - consumer.parallelism = 1 -} - -redisdb.connection.timeout = 30000 - -redis { - host = localhost - port = 6340 - database { - key.expiry.seconds = 3600 - } -} - -lms-cassandra { - host = "localhost" - port = 9142 -} - -neo4j { - routePath = "bolt://localhost:7687" - graph = "domain" -} - -es { - basePath = "localhost:9200" -} - -schema { - basePath = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - supportedVersion = {"itemset": "2.0"} -} \ No newline at end of file diff --git a/csp-migrator/src/test/resources/test.conf b/csp-migrator/src/test/resources/test.conf deleted file mode 100644 index 89b23ee41..000000000 --- a/csp-migrator/src/test/resources/test.conf +++ /dev/null @@ -1,87 +0,0 @@ -include "base-test.conf" - -job.env = "local" - -kafka { - input.topic = "sunbirddev.csp.migration.request" - failed.topic = "sunbirddev.csp.migration.job.request.failed" - groupId = "sunbirddev-csp-migrator-group" - live_video_stream.topic = "sunbirddev.live.video.stream.request" - live_content_node_republish.topic = "sunbirddev.republish.job.request" - live_question_node_republish.topic = "sunbirddev.republish.job.request" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - csp-migrator.parallelism = 1 -} - -redis { - database { - relationCache.id = 10 - collectionCache.id = 5 - } -} - -service { - search.basePath = "http://11.2.6.6/search" - lms.basePath = "http://11.2.6.6/lms" - learning_service.basePath = "http://11.2.4.22:8080/learning-service" - content_service.basePath = "http://11.2.6.6/content" -} - - -hierarchy { - keyspace = "hierarchy_store" - table = "content_hierarchy" -} - -content { - keyspace = "content_store" - content_table = "content_data" - assessment_table = "question_data" -} - -key_value_strings_to_migrate = { - "https://sunbirddev.blob.core.windows.net": "https://store.migrationdomain", - "https://ekstep-public-dev.s3-ap-south-1.amazonaws.com": "https://store.migrationdomain", - "https://community.ekstep.in/assets/public": "https://store.migrationdomain" - "https://ntpproductionall.blob.core.windows.net/ntp-content-production": "https://store.migrationdomain" - "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging": "https://store.migrationdomain" -} - -neo4j_fields_to_migrate = { - "asset": ["artifactUrl","thumbnail"], - "content": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "assetsMap", "certTemplate", "itemSetPreviewUrl", "grayScaleAppIcon"], - "contentimage": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "assetsMap", "certTemplate", "itemSetPreviewUrl", "grayScaleAppIcon"], - "collection": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "toc_url", "grayScaleAppIcon"], - "collectionimage": ["appIcon","artifactUrl", "posterImage", "previewUrl", "thumbnail", "toc_url", "grayScaleAppIcon"], - "plugins": ["artifactUrl"], - "itemset": ["previewUrl"], - "assessmentitem": ["data", "question", "solutions", "editorState", "media"], - "question": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"], - "questionimage": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"], - "questionset": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"], - "questionsetimage": ["appIcon","artifactUrl", "posterImage", "previewUrl","downloadUrl", "variants","pdfUrl"], -} - -cassandra_fields_to_migrate = { - "assessmentitem": ["body", "editorState", "answer", "solutions", "instructions", "media"] -} - -questionset.hierarchy.keyspace="hierarchy_store" -questionset.hierarchy.table="questionset_hierarchy" - -migrationVersion = 1 - -video_stream_regeneration_enable = false -live_node_republish_enable = false -copy_missing_files_to_cloud = false - -cloud_storage_container = sample-content -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "CONTENT_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"] -cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] diff --git a/csp-migrator/src/test/resources/test.cql b/csp-migrator/src/test/resources/test.cql deleted file mode 100644 index de3e9cfd8..000000000 --- a/csp-migrator/src/test/resources/test.cql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS content_store WITH replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS content_store.content_data ( - content_id text, - body blob, - PRIMARY KEY (content_id) -); - -CREATE KEYSPACE IF NOT EXISTS hierarchy_store WITH replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS hierarchy_store.content_hierarchy ( - identifier text, - hierarchy text, - relational_metadata text, - PRIMARY KEY (identifier) -); - -INSERT INTO content_store.content_data(content_id, body) VALUES ('do_31270597860728832015700', textAsBlob('{"theme":{"id":"theme","version":"1.0","startStage":"b8b47094-1d69-43a1-9c88-c02e760996c5","stage":[{"x":0,"y":0,"w":100,"h":100,"id":"b8b47094-1d69-43a1-9c88-c02e760996c5","rotate":null,"config":{"__cdata":"{\"opacity\":100,\"strokeWidth\":1,\"stroke\":\"rgba(255, 255, 255, 0)\",\"autoplay\":false,\"visible\":true,\"color\":\"#FFFFFF\",\"genieControls\":false,\"instructions\":\"\"}"},"param":[{"name":"next","value":"85c23e06-51c9-416b-ac9b-b9d5386a37ee"}],"manifest":{"media":[{"assetId":"385467069"},{"assetId":"QuizImage"},{"assetId":"org.ekstep.questionset.audioicon"},{"assetId":"org.ekstep.questionset.default-imgageicon"}]},"org.ekstep.questionset":[{"x":9,"y":6,"w":80,"h":85,"rotate":0,"z-index":0,"id":"b27c19a3-768b-4b07-9809-00d7a1e0e5c6","data":{"__cdata":"[{\"code\":\"NA\",\"isShuffleOption\":false,\"body\":\"{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)3/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)1/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)5/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)3/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)1/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)5/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q)The difference of the molecules of water in gypsum and plaster       

\\\\n\\\\n

  of Paris is

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)3/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)1/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)5/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) కార్బన్ ఎలక్ట్రాన్ విన్యాసాన్ని ఈ క్రింది విధంగా రాయరాదని సాత్విక చెప్పింది. ఎందువలన?

\\\\n\\\",\\\"image\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)ఆఫ్  బౌ నియమాన్ని పాటించుట లేదు

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)హుండు నియమాన్ని పాటించుట లేదు

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)పౌలీవర్జన నియమాన్ని పాటించుట లేదు

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)పై వన్నీ సరైనవే

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\\\n\\\\n

Be cause this electronic configuration does not obey

\\\\n\\\",\\\"image\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) aufbau principle  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) Hund”s rule

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) Paul”s exclusion princple 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D) All the above

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\\\n\\\\n

Be cause this electronic configuration does not obey

\\\\n\\\",\\\"image\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) aufbau principle  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) Hund”s rule

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) Paul”s exclusion princple 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D) All the above

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) ఒక మూలకం పరమాణువ సంఖ్య 20 అయినా ఆవర్తన పట్టికలో ఆ మూలక స్థానం

\\\\n\\\\n
    \\\\n\\\\t
  1. 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్ C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్
    2. \\\\n
\\\\n\\\\n

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) If an atomic number of an element is 20 then what is the position of that element in a modern periodic table

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 4th period and 2nd group  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) 1st period and 2nd group

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) 3rd period and 13th group

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

 D) 2nd period and 2nd group

\\\\n\\\\n

 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) In a Doberiner triad the atomic weight of first and third elements are 32 and 125 , then what is the atomic weight of second one

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 78.5

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)80 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)23

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)60

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) Find the frequency of a radiowave of wave lenth 100 metres?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 3 HZ  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)3 KHZ  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)3 MHZ 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)30 MHZ

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

 Q .When l=2 , what is the maximum value of ml

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)-2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2(2)+1

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)0

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)-2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2(2)+1

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)0

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)-2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2(2)+1

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)0

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"max_score\\\":1,\\\"isShuffleOption\\\":false,\\\"isPartialScore\\\":true,\\\"templateType\\\":\\\"Horizontal\\\",\\\"name\\\":\\\"\\\\n\\\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\\\n\\\\n\\\",\\\"title\\\":\\\"Copy of - \\\\n\\\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\\\n\\\\n\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"topic\\\":[],\\\"medium\\\":\\\"Telugu\\\",\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"subject\\\":\\\"Physical Science\\\",\\\"qlevel\\\":\\\"EASY\\\",\\\"description\\\":\\\"10 th ps bits\\\",\\\"category\\\":\\\"MCQ\\\",\\\"topicData\\\":\\\"(0) topics selected\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - \\\\n\\\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\\\n\\\\n\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"questionDesc\\\":\\\"10 th ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm_ps_cha8_Q1\\\",\\\"title\\\":\\\"Copy of - Copy of - Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"topicData\\\":\\\"(0) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm_ps_cha8_Q1\\\",\\\"title\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em_ps_cha8_Q1\\\",\\\"title\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q1\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q1\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(0) topics selected\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"max_score\\\":1,\\\"name\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Copy of - Examprep_10tm_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha4 Q3\\\",\\\"title\\\":\\\"Examprep_10em ps_cha4 Q3\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Examprep_10em ps_cha4 Q3\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha8- Q3\\\",\\\"title\\\":\\\"Examprep_10em ps_cha8- Q3\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Examprep_10em ps_cha8- Q3\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm ps_cha8- Q3\\\",\\\"title\\\":\\\"Copy of - Examprep_10tm ps_cha8- Q3\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm ps_cha8- Q3\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"title\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"title\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]}}\",\"itemType\":\"UNIT\",\"version\":2,\"category\":\"MCQ\",\"createdBy\":\"c252c844-4178-498f-9006-d63540319254\",\"channel\":\"0123207707019919361056\",\"type\":\"mcq\",\"template\":\"NA\",\"template_id\":\"NA\",\"framework\":\"ap_k-12_1\",\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)The difference of the molecules of water in gypsum and plaster       

\\n\\n

  of Paris is

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) కార్బన్ ఎలక్ట్రాన్ విన్యాసాన్ని ఈ క్రింది విధంగా రాయరాదని సాత్విక చెప్పింది. ఎందువలన?

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)ఆఫ్  బౌ నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)హుండు నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)పౌలీవర్జన నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)పై వన్నీ సరైనవే

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) ఒక మూలకం పరమాణువ సంఖ్య 20 అయినా ఆవర్తన పట్టికలో ఆ మూలక స్థానం

\\n\\n
    \\n\\t
  1. 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్ C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్
    2. \\n
\\n\\n

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) If an atomic number of an element is 20 then what is the position of that element in a modern periodic table

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4th period and 2nd group  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1st period and 2nd group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3rd period and 13th group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

 D) 2nd period and 2nd group

\\n\\n

 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) In a Doberiner triad the atomic weight of first and third elements are 32 and 125 , then what is the atomic weight of second one

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 78.5

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)80 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)23

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)60

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Find the frequency of a radiowave of wave lenth 100 metres?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 3 HZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)3 KHZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)3 MHZ 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)30 MHZ

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

 Q .When l=2 , what is the maximum value of ml

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"max_score\":1,\"isShuffleOption\":false,\"isPartialScore\":true,\"templateType\":\"Horizontal\",\"name\":\"\\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"title\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"board\":\"State (Andhra Pradesh)\",\"topic\":[],\"medium\":\"Telugu\",\"gradeLevel\":[\"Class 10\"],\"subject\":\"Physical Science\",\"qlevel\":\"EASY\",\"description\":\"10 th ps bits\",\"category\":\"MCQ\",\"topicData\":\"(0) topics selected\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 th ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Copy of - Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(0) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(0) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10tm_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q2\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha4 Q3\",\"title\":\"Examprep_10em ps_cha4 Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha4 Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q3\",\"title\":\"Examprep_10em ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha8- Q3\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Examprep_10tm ps_cha8- Q3\",\"title\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Examprep_10em ps_cha8- Q5\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"description\":\"10 PS BITS\",\"options\":[{\"answer\":true,\"value\":{\"type\":\"text\",\"asset\":\"1\"}}],\"identifier\":\"do_31270597409132544015699\",\"isSelected\":true,\"$$hashKey\":\"object:1142\"}]"},"config":{"__cdata":"{\"title\":\"10 PS BITS\",\"max_score\":1,\"allow_skip\":true,\"show_feedback\":true,\"shuffle_questions\":false,\"shuffle_options\":false,\"total_items\":1,\"btn_edit\":\"Edit\"}"},"org.ekstep.question":[{"id":"6d5e4460-7393-49e9-b158-430441c0e6c0","type":"mcq","pluginId":"org.ekstep.questionunit.mcq","pluginVer":"1.1","templateId":"horizontalMCQ","data":{"__cdata":"{\"question\":{\"text\":\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]}"},"config":{"__cdata":"{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)The difference of the molecules of water in gypsum and plaster       

\\n\\n

  of Paris is

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) కార్బన్ ఎలక్ట్రాన్ విన్యాసాన్ని ఈ క్రింది విధంగా రాయరాదని సాత్విక చెప్పింది. ఎందువలన?

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)ఆఫ్  బౌ నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)హుండు నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)పౌలీవర్జన నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)పై వన్నీ సరైనవే

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) ఒక మూలకం పరమాణువ సంఖ్య 20 అయినా ఆవర్తన పట్టికలో ఆ మూలక స్థానం

\\n\\n
    \\n\\t
  1. 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్ C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్
    2. \\n
\\n\\n

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) If an atomic number of an element is 20 then what is the position of that element in a modern periodic table

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4th period and 2nd group  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1st period and 2nd group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3rd period and 13th group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

 D) 2nd period and 2nd group

\\n\\n

 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) In a Doberiner triad the atomic weight of first and third elements are 32 and 125 , then what is the atomic weight of second one

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 78.5

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)80 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)23

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)60

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Find the frequency of a radiowave of wave lenth 100 metres?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 3 HZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)3 KHZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)3 MHZ 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)30 MHZ

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

 Q .When l=2 , what is the maximum value of ml

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"max_score\":1,\"isShuffleOption\":false,\"isPartialScore\":true,\"templateType\":\"Horizontal\",\"name\":\"\\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"title\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"board\":\"State (Andhra Pradesh)\",\"topic\":[],\"medium\":\"Telugu\",\"gradeLevel\":[\"Class 10\"],\"subject\":\"Physical Science\",\"qlevel\":\"EASY\",\"description\":\"10 th ps bits\",\"category\":\"MCQ\",\"topicData\":\"(0) topics selected\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 th ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Copy of - Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(0) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(0) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10tm_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q2\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha4 Q3\",\"title\":\"Examprep_10em ps_cha4 Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha4 Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q3\",\"title\":\"Examprep_10em ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha8- Q3\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Examprep_10tm ps_cha8- Q3\",\"title\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Examprep_10em ps_cha8- Q5\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1}"},"w":80,"h":85,"x":9,"y":6}]}]},{"x":0,"y":0,"w":100,"h":100,"id":"85c23e06-51c9-416b-ac9b-b9d5386a37ee","rotate":null,"config":{"__cdata":"{\"opacity\":100,\"strokeWidth\":1,\"stroke\":\"rgba(255, 255, 255, 0)\",\"autoplay\":false,\"visible\":true,\"color\":\"#FFFFFF\",\"genieControls\":false,\"instructions\":\"\"}"},"param":[{"name":"previous","value":"b8b47094-1d69-43a1-9c88-c02e760996c5"}],"manifest":{"media":[{"assetId":"8dbbc49b-6776-458c-b211-c1a260437421"}]},"org.ekstep.video":[{"y":7.9,"x":10.97,"w":78.4,"h":79.51,"rotate":0,"z-index":0,"id":"8dbbc49b-6776-458c-b211-c1a260437421","config":{"__cdata":"{\"autoplay\":true,\"controls\":true,\"muted\":false,\"visible\":true,\"url\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_3128517134878638081614/artifact/ms-new-bit-1.mp4\"}"}}]}],"manifest":{"media":[{"id":"2381510b-4d49-47d5-b765-c130233028dc","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/renderer/controller/navigation_ctrl.js","type":"js"},{"id":"e34e434b-c4de-4907-b025-56c156957112","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/renderer/templates/navigation.html","type":"js"},{"id":"org.ekstep.navigation","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.navigation_manifest","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/manifest.json","type":"json"},{"id":"org.ekstep.questionunit.renderer.audioicon","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/audio-icon.png","type":"image"},{"id":"org.ekstep.questionunit.renderer.downarrow","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/down_arrow.png","type":"image"},{"id":"aa57e9c4-e978-4ab1-be74-09a657aa092c","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/js/components.js","type":"js"},{"id":"07bac729-0b71-42d1-b4e3-d5398474d0d0","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/css/components.css","type":"css"},{"id":"6b74a0ea-d9dc-4ae9-9134-abe3124260da","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.js","type":"js"},{"id":"d5b4d576-7477-490b-9713-d9857b5beab1","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.css","type":"css"},{"id":"8aa00822-5d00-400c-bdbd-9847737db6a5","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-bold.ttf","type":"js"},{"id":"bdace240-7a3b-4b7b-b9d1-564747334578","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-bolditalic.ttf","type":"js"},{"id":"363edb27-4782-4ee5-b7c7-cb8e85e29d52","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-italic.ttf","type":"js"},{"id":"939ed1b9-7e48-4c1d-aeee-831d44698d6c","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-regular.ttf","type":"js"},{"id":"9e8dca64-6a00-4dde-b840-3ee96d50a2b8","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_math-bolditalic.ttf","type":"js"},{"id":"d7135270-d403-48ce-be16-c449d26e6da7","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_math-italic.ttf","type":"js"},{"id":"3d410ae3-3072-43d8-b452-9620eb5585ed","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_math-regular.ttf","type":"js"},{"id":"5447cb80-7597-411c-90dc-13a0c02cc863","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size1-regular.ttf","type":"js"},{"id":"08358247-a9e0-4cb3-8bb3-9b97b7d6174e","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size2-regular.ttf","type":"js"},{"id":"03556441-a8cd-45a6-93b7-6e98743490b1","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size3-regular.ttf","type":"js"},{"id":"4713a36c-49dc-4b66-a387-79e04c050598","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size4-regular.ttf","type":"js"},{"id":"org.ekstep.questionunit","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionunit_manifest","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/manifest.json","type":"json"},{"id":"b6a388cc-6d04-4cb1-9ca0-5919a75ddb9c","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/style.css","type":"css"},{"id":"97b66a43-6b00-4753-aebc-d2b270e1c702","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/horizontal_and_vertical.css","type":"css"},{"id":"367e9c5d-aa82-436c-9767-31a4e3d89356","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/mcq-layouts.js","type":"js"},{"id":"95e6a633-711c-4e51-ae0a-4207874b6be9","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/template_controller.js","type":"js"},{"id":"28903dd9-4012-4866-9d1c-061a53db19a3","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1//renderer/assets/tick_icon.png","type":"image"},{"id":"008a394a-399c-4010-9b83-a727abe607c0","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1//renderer/assets/audio-icon2.png","type":"image"},{"id":"030d52c7-669a-49e7-ad10-2632bdb949e8","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1//renderer/assets/music-blue.png","type":"image"},{"id":"org.ekstep.questionunit.mcq","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionunit.mcq_manifest","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/manifest.json","type":"json"},{"id":"org.ekstep.questionset.quiz","plugin":"org.ekstep.questionset.quiz","ver":"1.0","src":"/content-plugins/org.ekstep.questionset.quiz-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionset.quiz_manifest","plugin":"org.ekstep.questionset.quiz","ver":"1.0","src":"/content-plugins/org.ekstep.questionset.quiz-1.0/manifest.json","type":"json"},{"id":"org.ekstep.iterator","plugin":"org.ekstep.iterator","ver":"1.0","src":"/content-plugins/org.ekstep.iterator-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.iterator_manifest","plugin":"org.ekstep.iterator","ver":"1.0","src":"/content-plugins/org.ekstep.iterator-1.0/manifest.json","type":"json"},{"id":"ec943bc6-5684-483c-95a0-db88990c2ea6","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/telemetry_logger.js","type":"js"},{"id":"87862711-b63c-4140-a355-dd4cd5d336b4","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/html_audio_plugin.js","type":"js"},{"id":"4b619cde-e3da-429c-bc05-b329365bc9b0","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/qs_feedback_popup.js","type":"js"},{"id":"org.ekstep.questionset","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionset_manifest","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/manifest.json","type":"json"},{"id":"2caa1e61-bc5a-4104-a74c-65dcda49a148","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/libs/video.js","type":"js"},{"id":"7a98d0c5-d4c0-4eda-bd52-5c793ab29aab","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/libs/videoyoutube.js","type":"js"},{"id":"026257be-900a-4929-ae15-8ba4632d1225","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/libs/videojs.css","type":"css"},{"id":"org.ekstep.video","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/videoplugin.js","type":"plugin"},{"id":"org.ekstep.video_manifest","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/manifest.json","type":"json"},{"id":385467069,"src":"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png","assetId":"do_31270089678451507213885","type":"image","preload":false},{"id":"QuizImage","src":"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png","assetId":"QuizImage","type":"image","preload":true},{"id":"org.ekstep.questionset.audioicon","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png","assetId":"org.ekstep.questionset.audioicon","type":"image","preload":true},{"id":"org.ekstep.questionset.default-imgageicon","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png","assetId":"org.ekstep.questionset.default-imgageicon","type":"image","preload":true},{"id":"8dbbc49b-6776-458c-b211-c1a260437421","src":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_3128517134878638081614/artifact/ms-new-bit-1.mp4","assetId":"8dbbc49b-6776-458c-b211-c1a260437421","type":"video"}]},"plugin-manifest":{"plugin":[{"id":"org.ekstep.navigation","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.questionunit","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.questionunit.mcq","ver":"1.1","type":"plugin","depends":"org.ekstep.questionunit"},{"id":"org.ekstep.questionset.quiz","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.iterator","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.questionset","ver":"1.0","type":"plugin","depends":"org.ekstep.questionset.quiz,org.ekstep.iterator"},{"id":"org.ekstep.video","ver":"1.3","type":"widget","depends":""}]},"compatibilityVersion":4}} | [{"src":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_3128517134878638081614/artifact/ms-new-bit-1.mp4","id":"8dbbc49b-6776-458c-b211-c1a260437421","type":"video"}] | {"theme":{"id":"theme","version":"1.0","startStage":"b8b47094-1d69-43a1-9c88-c02e760996c5","stage":[{"x":0,"y":0,"w":100,"h":100,"id":"b8b47094-1d69-43a1-9c88-c02e760996c5","rotate":null,"config":{"__cdata":"{\"opacity\":100,\"strokeWidth\":1,\"stroke\":\"rgba(255, 255, 255, 0)\",\"autoplay\":false,\"visible\":true,\"color\":\"#FFFFFF\",\"genieControls\":false,\"instructions\":\"\"}"},"param":[{"name":"next","value":"85c23e06-51c9-416b-ac9b-b9d5386a37ee"}],"manifest":{"media":[{"assetId":"385467069"},{"assetId":"QuizImage"},{"assetId":"org.ekstep.questionset.audioicon"},{"assetId":"org.ekstep.questionset.default-imgageicon"}]},"org.ekstep.questionset":[{"x":9,"y":6,"w":80,"h":85,"rotate":0,"z-index":0,"id":"b27c19a3-768b-4b07-9809-00d7a1e0e5c6","data":{"__cdata":"[{\"code\":\"NA\",\"isShuffleOption\":false,\"body\":\"{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)3/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)1/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)5/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)3/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)1/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)5/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q)The difference of the molecules of water in gypsum and plaster       

\\\\n\\\\n

  of Paris is

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)3/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)1/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)5/2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) కార్బన్ ఎలక్ట్రాన్ విన్యాసాన్ని ఈ క్రింది విధంగా రాయరాదని సాత్విక చెప్పింది. ఎందువలన?

\\\\n\\\",\\\"image\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A)ఆఫ్  బౌ నియమాన్ని పాటించుట లేదు

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)హుండు నియమాన్ని పాటించుట లేదు

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)పౌలీవర్జన నియమాన్ని పాటించుట లేదు

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)పై వన్నీ సరైనవే

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\\\n\\\\n

Be cause this electronic configuration does not obey

\\\\n\\\",\\\"image\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) aufbau principle  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) Hund”s rule

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) Paul”s exclusion princple 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D) All the above

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\\\n\\\\n

Be cause this electronic configuration does not obey

\\\\n\\\",\\\"image\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) aufbau principle  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) Hund”s rule

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) Paul”s exclusion princple 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D) All the above

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) ఒక మూలకం పరమాణువ సంఖ్య 20 అయినా ఆవర్తన పట్టికలో ఆ మూలక స్థానం

\\\\n\\\\n
    \\\\n\\\\t
  1. 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్ C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్
    2. \\\\n
\\\\n\\\\n

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) If an atomic number of an element is 20 then what is the position of that element in a modern periodic table

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 4th period and 2nd group  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B) 1st period and 2nd group

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C) 3rd period and 13th group

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

 D) 2nd period and 2nd group

\\\\n\\\\n

 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) In a Doberiner triad the atomic weight of first and third elements are 32 and 125 , then what is the atomic weight of second one

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 78.5

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)80 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)23

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)60

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

Q) Find the frequency of a radiowave of wave lenth 100 metres?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 3 HZ  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)3 KHZ  

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)3 MHZ 

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)30 MHZ

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

 Q .When l=2 , what is the maximum value of ml

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)-2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2(2)+1

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)0

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)-2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2(2)+1

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)0

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"data\\\":{\\\"plugin\\\":{\\\"id\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.1\\\",\\\"templateId\\\":\\\"horizontalMCQ\\\"},\\\"data\\\":{\\\"question\\\":{\\\"text\\\":\\\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\"},\\\"options\\\":[{\\\"text\\\":\\\"

A) 2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1034\\\"},{\\\"text\\\":\\\"

B)-2

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"hint\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1035\\\"},{\\\"text\\\":\\\"

C)2(2)+1

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":true,\\\"$$hashKey\\\":\\\"object:1045\\\"},{\\\"text\\\":\\\"

D)0

\\\\n\\\",\\\"image\\\":\\\"\\\",\\\"audio\\\":\\\"\\\",\\\"audioName\\\":\\\"\\\",\\\"isCorrect\\\":false,\\\"$$hashKey\\\":\\\"object:1050\\\"}],\\\"questionCount\\\":0,\\\"media\\\":[]},\\\"config\\\":{\\\"metadata\\\":{\\\"max_score\\\":1,\\\"isShuffleOption\\\":false,\\\"isPartialScore\\\":true,\\\"templateType\\\":\\\"Horizontal\\\",\\\"name\\\":\\\"\\\\n\\\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\\\n\\\\n\\\",\\\"title\\\":\\\"Copy of - \\\\n\\\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\\\n\\\\n\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"topic\\\":[],\\\"medium\\\":\\\"Telugu\\\",\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"subject\\\":\\\"Physical Science\\\",\\\"qlevel\\\":\\\"EASY\\\",\\\"description\\\":\\\"10 th ps bits\\\",\\\"category\\\":\\\"MCQ\\\",\\\"topicData\\\":\\\"(0) topics selected\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - \\\\n\\\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\\\n\\\\n\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"questionDesc\\\":\\\"10 th ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm_ps_cha8_Q1\\\",\\\"title\\\":\\\"Copy of - Copy of - Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"topicData\\\":\\\"(0) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm_ps_cha8_Q1\\\",\\\"title\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q1\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em_ps_cha8_Q1\\\",\\\"title\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q1\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q1\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(0) topics selected\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"max_score\\\":1,\\\"name\\\":\\\"Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm_ps_cha8_Q2\\\",\\\"title\\\":\\\"Copy of - Copy of - Examprep_10tm_ps_cha8_Q2\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm_ps_cha8_Q2\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha4 Q3\\\",\\\"title\\\":\\\"Examprep_10em ps_cha4 Q3\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Examprep_10em ps_cha4 Q3\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha8- Q3\\\",\\\"title\\\":\\\"Examprep_10em ps_cha8- Q3\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"Telugu\\\",\\\"questionTitle\\\":\\\"Examprep_10em ps_cha8- Q3\\\",\\\"qlevel\\\":\\\"DIFFICULT\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Classification Of Elements\\\"],\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10tm ps_cha8- Q3\\\",\\\"title\\\":\\\"Copy of - Examprep_10tm ps_cha8- Q3\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 ps bits\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Copy of - Examprep_10tm ps_cha8- Q3\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[],\\\"questionDesc\\\":\\\"10 ps bits\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"title\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"topicData\\\":\\\"(1) topics selected\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]},\\\"medium\\\":\\\"English\\\",\\\"questionTitle\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"qlevel\\\":\\\"MEDIUM\\\",\\\"subject\\\":\\\"Physical Science\\\",\\\"board\\\":\\\"State (Andhra Pradesh)\\\",\\\"templateType\\\":\\\"Horizontal\\\",\\\"isPartialScore\\\":true,\\\"gradeLevel\\\":[\\\"Class 10\\\"],\\\"isShuffleOption\\\":false,\\\"topic\\\":[\\\"Structure Of Atom\\\"],\\\"questionDesc\\\":\\\"10 PS BITS\\\",\\\"max_score\\\":1,\\\"name\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"title\\\":\\\"Examprep_10em ps_cha8- Q5\\\",\\\"description\\\":\\\"10 PS BITS\\\",\\\"category\\\":\\\"MCQ\\\"},\\\"max_time\\\":0,\\\"max_score\\\":1,\\\"partial_scoring\\\":true,\\\"layout\\\":\\\"Horizontal\\\",\\\"isShuffleOption\\\":false,\\\"questionCount\\\":1},\\\"media\\\":[{\\\"id\\\":385467069,\\\"src\\\":\\\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\\\",\\\"assetId\\\":\\\"do_31270089678451507213885\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":false},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.audioicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true},{\\\"id\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"src\\\":\\\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\\\",\\\"assetId\\\":\\\"org.ekstep.questionset.default-imgageicon\\\",\\\"type\\\":\\\"image\\\",\\\"preload\\\":true}]}}\",\"itemType\":\"UNIT\",\"version\":2,\"category\":\"MCQ\",\"createdBy\":\"c252c844-4178-498f-9006-d63540319254\",\"channel\":\"0123207707019919361056\",\"type\":\"mcq\",\"template\":\"NA\",\"template_id\":\"NA\",\"framework\":\"ap_k-12_1\",\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)The difference of the molecules of water in gypsum and plaster       

\\n\\n

  of Paris is

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) కార్బన్ ఎలక్ట్రాన్ విన్యాసాన్ని ఈ క్రింది విధంగా రాయరాదని సాత్విక చెప్పింది. ఎందువలన?

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)ఆఫ్  బౌ నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)హుండు నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)పౌలీవర్జన నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)పై వన్నీ సరైనవే

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) ఒక మూలకం పరమాణువ సంఖ్య 20 అయినా ఆవర్తన పట్టికలో ఆ మూలక స్థానం

\\n\\n
    \\n\\t
  1. 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్ C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్
    2. \\n
\\n\\n

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) If an atomic number of an element is 20 then what is the position of that element in a modern periodic table

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4th period and 2nd group  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1st period and 2nd group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3rd period and 13th group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

 D) 2nd period and 2nd group

\\n\\n

 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) In a Doberiner triad the atomic weight of first and third elements are 32 and 125 , then what is the atomic weight of second one

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 78.5

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)80 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)23

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)60

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Find the frequency of a radiowave of wave lenth 100 metres?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 3 HZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)3 KHZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)3 MHZ 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)30 MHZ

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

 Q .When l=2 , what is the maximum value of ml

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"max_score\":1,\"isShuffleOption\":false,\"isPartialScore\":true,\"templateType\":\"Horizontal\",\"name\":\"\\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"title\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"board\":\"State (Andhra Pradesh)\",\"topic\":[],\"medium\":\"Telugu\",\"gradeLevel\":[\"Class 10\"],\"subject\":\"Physical Science\",\"qlevel\":\"EASY\",\"description\":\"10 th ps bits\",\"category\":\"MCQ\",\"topicData\":\"(0) topics selected\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 th ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Copy of - Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(0) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(0) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10tm_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q2\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha4 Q3\",\"title\":\"Examprep_10em ps_cha4 Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha4 Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q3\",\"title\":\"Examprep_10em ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha8- Q3\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Examprep_10tm ps_cha8- Q3\",\"title\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Examprep_10em ps_cha8- Q5\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"description\":\"10 PS BITS\",\"options\":[{\"answer\":true,\"value\":{\"type\":\"text\",\"asset\":\"1\"}}],\"identifier\":\"do_31270597409132544015699\",\"isSelected\":true,\"$$hashKey\":\"object:1142\"}]"},"config":{"__cdata":"{\"title\":\"10 PS BITS\",\"max_score\":1,\"allow_skip\":true,\"show_feedback\":true,\"shuffle_questions\":false,\"shuffle_options\":false,\"total_items\":1,\"btn_edit\":\"Edit\"}"},"org.ekstep.question":[{"id":"6d5e4460-7393-49e9-b158-430441c0e6c0","type":"mcq","pluginId":"org.ekstep.questionunit.mcq","pluginVer":"1.1","templateId":"horizontalMCQ","data":{"__cdata":"{\"question\":{\"text\":\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]}"},"config":{"__cdata":"{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)జిప్సమ్ మరియు ప్లాస్టర్ ఆఫ్ పారిస్ ల నందు ఉండే నీటి అణువులలో తేడా ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q)The difference of the molecules of water in gypsum and plaster       

\\n\\n

  of Paris is

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)3/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)1/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)5/2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) కార్బన్ ఎలక్ట్రాన్ విన్యాసాన్ని ఈ క్రింది విధంగా రాయరాదని సాత్విక చెప్పింది. ఎందువలన?

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A)ఆఫ్  బౌ నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)హుండు నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)పౌలీవర్జన నియమాన్ని పాటించుట లేదు

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)పై వన్నీ సరైనవే

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Satwika told that the electronic configuration of carbon cannot be written like this……why?

\\n\\n

Be cause this electronic configuration does not obey

\\n\",\"image\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) aufbau principle  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) Hund”s rule

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) Paul”s exclusion princple 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) All the above

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) ఒక మూలకం పరమాణువ సంఖ్య 20 అయినా ఆవర్తన పట్టికలో ఆ మూలక స్థానం

\\n\\n
    \\n\\t
  1. 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్ C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్
    2. \\n
\\n\\n

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4 వ పీరియడ్ మరియు 2 వ గ్రూప్  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3 వ పీరియడ్ మరియు 13 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D) 2 వ పీరియడ్ మరియు 2 వ గ్రూప్

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) If an atomic number of an element is 20 then what is the position of that element in a modern periodic table

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 4th period and 2nd group  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B) 1st period and 2nd group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C) 3rd period and 13th group

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

 D) 2nd period and 2nd group

\\n\\n

 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) In a Doberiner triad the atomic weight of first and third elements are 32 and 125 , then what is the atomic weight of second one

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 78.5

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)80 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)23

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)60

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

Q) Find the frequency of a radiowave of wave lenth 100 metres?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 3 HZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)3 KHZ  

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)3 MHZ 

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)30 MHZ

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

 Q .When l=2 , what is the maximum value of ml

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"data\":{\"plugin\":{\"id\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\",\"templateId\":\"horizontalMCQ\"},\"data\":{\"question\":{\"text\":\"

1) l=2 అయినపుడు  యొక్క mlగరిష్ట విలువ ఎంత?

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\"},\"options\":[{\"text\":\"

A) 2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1034\"},{\"text\":\"

B)-2

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"hint\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1035\"},{\"text\":\"

C)2(2)+1

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":true,\"$$hashKey\":\"object:1045\"},{\"text\":\"

D)0

\\n\",\"image\":\"\",\"audio\":\"\",\"audioName\":\"\",\"isCorrect\":false,\"$$hashKey\":\"object:1050\"}],\"questionCount\":0,\"media\":[]},\"config\":{\"metadata\":{\"max_score\":1,\"isShuffleOption\":false,\"isPartialScore\":true,\"templateType\":\"Horizontal\",\"name\":\"\\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"title\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"board\":\"State (Andhra Pradesh)\",\"topic\":[],\"medium\":\"Telugu\",\"gradeLevel\":[\"Class 10\"],\"subject\":\"Physical Science\",\"qlevel\":\"EASY\",\"description\":\"10 th ps bits\",\"category\":\"MCQ\",\"topicData\":\"(0) topics selected\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - \\n\\tఈ క్రింది వాటిలో విశిష్టోష్ణానికి ప్రమాణం\\n\\n\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 th ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Copy of - Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(0) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q1\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q1\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(0) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Copy of - Examprep_10em_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Copy of - Examprep_10em_ps_cha8_Q2\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10tm_ps_cha8_Q2\",\"title\":\"Copy of - Copy of - Examprep_10tm_ps_cha8_Q2\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[]},\"medium\":\"Telugu\",\"questionTitle\":\"Copy of - Examprep_10tm_ps_cha8_Q2\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha4 Q3\",\"title\":\"Examprep_10em ps_cha4 Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha4 Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q3\",\"title\":\"Examprep_10em ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"Telugu\",\"questionTitle\":\"Examprep_10em ps_cha8- Q3\",\"qlevel\":\"DIFFICULT\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Classification Of Elements\"],\"max_score\":1,\"name\":\"Examprep_10tm ps_cha8- Q3\",\"title\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"topicData\":\"(1) topics selected\",\"description\":\"10 ps bits\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Copy of - Examprep_10tm ps_cha8- Q3\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[],\"questionDesc\":\"10 ps bits\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"topicData\":\"(1) topics selected\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1},\"media\":[{\"id\":385467069,\"src\":\"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png\",\"assetId\":\"do_31270089678451507213885\",\"type\":\"image\",\"preload\":false},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.audioicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png\",\"assetId\":\"org.ekstep.questionset.audioicon\",\"type\":\"image\",\"preload\":true},{\"id\":\"org.ekstep.questionset.default-imgageicon\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png\",\"assetId\":\"org.ekstep.questionset.default-imgageicon\",\"type\":\"image\",\"preload\":true}]},\"medium\":\"English\",\"questionTitle\":\"Examprep_10em ps_cha8- Q5\",\"qlevel\":\"MEDIUM\",\"subject\":\"Physical Science\",\"board\":\"State (Andhra Pradesh)\",\"templateType\":\"Horizontal\",\"isPartialScore\":true,\"gradeLevel\":[\"Class 10\"],\"isShuffleOption\":false,\"topic\":[\"Structure Of Atom\"],\"questionDesc\":\"10 PS BITS\",\"max_score\":1,\"name\":\"Examprep_10em ps_cha8- Q5\",\"title\":\"Examprep_10em ps_cha8- Q5\",\"description\":\"10 PS BITS\",\"category\":\"MCQ\"},\"max_time\":0,\"max_score\":1,\"partial_scoring\":true,\"layout\":\"Horizontal\",\"isShuffleOption\":false,\"questionCount\":1}"},"w":80,"h":85,"x":9,"y":6}]}]},{"x":0,"y":0,"w":100,"h":100,"id":"85c23e06-51c9-416b-ac9b-b9d5386a37ee","rotate":null,"config":{"__cdata":"{\"opacity\":100,\"strokeWidth\":1,\"stroke\":\"rgba(255, 255, 255, 0)\",\"autoplay\":false,\"visible\":true,\"color\":\"#FFFFFF\",\"genieControls\":false,\"instructions\":\"\"}"},"param":[{"name":"previous","value":"b8b47094-1d69-43a1-9c88-c02e760996c5"}],"manifest":{"media":[{"assetId":"8dbbc49b-6776-458c-b211-c1a260437421"}]},"org.ekstep.video":[{"y":7.9,"x":10.97,"w":78.4,"h":79.51,"rotate":0,"z-index":0,"id":"8dbbc49b-6776-458c-b211-c1a260437421","config":{"__cdata":"{\"autoplay\":true,\"controls\":true,\"muted\":false,\"visible\":true,\"url\":\"https://drive.google.com/uc?export=download&id=1evKCELQOUYdT4unEdUdpemdS18vu4euZ\"}"}}]}],"manifest":{"media":[{"id":"2381510b-4d49-47d5-b765-c130233028dc","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/renderer/controller/navigation_ctrl.js","type":"js"},{"id":"e34e434b-c4de-4907-b025-56c156957112","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/renderer/templates/navigation.html","type":"js"},{"id":"org.ekstep.navigation","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.navigation_manifest","plugin":"org.ekstep.navigation","ver":"1.0","src":"/content-plugins/org.ekstep.navigation-1.0/manifest.json","type":"json"},{"id":"org.ekstep.questionunit.renderer.audioicon","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/audio-icon.png","type":"image"},{"id":"org.ekstep.questionunit.renderer.downarrow","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/down_arrow.png","type":"image"},{"id":"aa57e9c4-e978-4ab1-be74-09a657aa092c","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/js/components.js","type":"js"},{"id":"07bac729-0b71-42d1-b4e3-d5398474d0d0","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/css/components.css","type":"css"},{"id":"6b74a0ea-d9dc-4ae9-9134-abe3124260da","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.js","type":"js"},{"id":"d5b4d576-7477-490b-9713-d9857b5beab1","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.css","type":"css"},{"id":"8aa00822-5d00-400c-bdbd-9847737db6a5","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-bold.ttf","type":"js"},{"id":"bdace240-7a3b-4b7b-b9d1-564747334578","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-bolditalic.ttf","type":"js"},{"id":"363edb27-4782-4ee5-b7c7-cb8e85e29d52","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-italic.ttf","type":"js"},{"id":"939ed1b9-7e48-4c1d-aeee-831d44698d6c","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_main-regular.ttf","type":"js"},{"id":"9e8dca64-6a00-4dde-b840-3ee96d50a2b8","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_math-bolditalic.ttf","type":"js"},{"id":"d7135270-d403-48ce-be16-c449d26e6da7","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_math-italic.ttf","type":"js"},{"id":"3d410ae3-3072-43d8-b452-9620eb5585ed","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_math-regular.ttf","type":"js"},{"id":"5447cb80-7597-411c-90dc-13a0c02cc863","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size1-regular.ttf","type":"js"},{"id":"08358247-a9e0-4cb3-8bb3-9b97b7d6174e","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size2-regular.ttf","type":"js"},{"id":"03556441-a8cd-45a6-93b7-6e98743490b1","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size3-regular.ttf","type":"js"},{"id":"4713a36c-49dc-4b66-a387-79e04c050598","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/fonts/katex_size4-regular.ttf","type":"js"},{"id":"org.ekstep.questionunit","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionunit_manifest","plugin":"org.ekstep.questionunit","ver":"1.0","src":"/content-plugins/org.ekstep.questionunit-1.0/manifest.json","type":"json"},{"id":"b6a388cc-6d04-4cb1-9ca0-5919a75ddb9c","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/style.css","type":"css"},{"id":"97b66a43-6b00-4753-aebc-d2b270e1c702","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/horizontal_and_vertical.css","type":"css"},{"id":"367e9c5d-aa82-436c-9767-31a4e3d89356","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/mcq-layouts.js","type":"js"},{"id":"95e6a633-711c-4e51-ae0a-4207874b6be9","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/template_controller.js","type":"js"},{"id":"28903dd9-4012-4866-9d1c-061a53db19a3","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1//renderer/assets/tick_icon.png","type":"image"},{"id":"008a394a-399c-4010-9b83-a727abe607c0","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1//renderer/assets/audio-icon2.png","type":"image"},{"id":"030d52c7-669a-49e7-ad10-2632bdb949e8","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1//renderer/assets/music-blue.png","type":"image"},{"id":"org.ekstep.questionunit.mcq","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionunit.mcq_manifest","plugin":"org.ekstep.questionunit.mcq","ver":"1.1","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/manifest.json","type":"json"},{"id":"org.ekstep.questionset.quiz","plugin":"org.ekstep.questionset.quiz","ver":"1.0","src":"/content-plugins/org.ekstep.questionset.quiz-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionset.quiz_manifest","plugin":"org.ekstep.questionset.quiz","ver":"1.0","src":"/content-plugins/org.ekstep.questionset.quiz-1.0/manifest.json","type":"json"},{"id":"org.ekstep.iterator","plugin":"org.ekstep.iterator","ver":"1.0","src":"/content-plugins/org.ekstep.iterator-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.iterator_manifest","plugin":"org.ekstep.iterator","ver":"1.0","src":"/content-plugins/org.ekstep.iterator-1.0/manifest.json","type":"json"},{"id":"ec943bc6-5684-483c-95a0-db88990c2ea6","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/telemetry_logger.js","type":"js"},{"id":"87862711-b63c-4140-a355-dd4cd5d336b4","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/html_audio_plugin.js","type":"js"},{"id":"4b619cde-e3da-429c-bc05-b329365bc9b0","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/qs_feedback_popup.js","type":"js"},{"id":"org.ekstep.questionset","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/renderer/plugin.js","type":"plugin"},{"id":"org.ekstep.questionset_manifest","plugin":"org.ekstep.questionset","ver":"1.0","src":"/content-plugins/org.ekstep.questionset-1.0/manifest.json","type":"json"},{"id":"2caa1e61-bc5a-4104-a74c-65dcda49a148","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/libs/video.js","type":"js"},{"id":"7a98d0c5-d4c0-4eda-bd52-5c793ab29aab","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/libs/videoyoutube.js","type":"js"},{"id":"026257be-900a-4929-ae15-8ba4632d1225","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/libs/videojs.css","type":"css"},{"id":"org.ekstep.video","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/renderer/videoplugin.js","type":"plugin"},{"id":"org.ekstep.video_manifest","plugin":"org.ekstep.video","ver":"1.3","src":"/content-plugins/org.ekstep.video-1.3/manifest.json","type":"json"},{"id":385467069,"src":"/assets/public//content/do_31270089678451507213885/artifact/ice_screenshot_20190217-164338_1550402439889.png","assetId":"do_31270089678451507213885","type":"image","preload":false},{"id":"QuizImage","src":"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png","assetId":"QuizImage","type":"image","preload":true},{"id":"org.ekstep.questionset.audioicon","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio.png","assetId":"org.ekstep.questionset.audioicon","type":"image","preload":true},{"id":"org.ekstep.questionset.default-imgageicon","src":"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/default-image.png","assetId":"org.ekstep.questionset.default-imgageicon","type":"image","preload":true},{"id":"8dbbc49b-6776-458c-b211-c1a260437421","src":"https://drive.google.com/uc?export=download&id=1evKCELQOUYdT4unEdUdpemdS18vu4euZ","assetId":"8dbbc49b-6776-458c-b211-c1a260437421","type":"video"}]},"plugin-manifest":{"plugin":[{"id":"org.ekstep.navigation","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.questionunit","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.questionunit.mcq","ver":"1.1","type":"plugin","depends":"org.ekstep.questionunit"},{"id":"org.ekstep.questionset.quiz","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.iterator","ver":"1.0","type":"plugin","depends":""},{"id":"org.ekstep.questionset","ver":"1.0","type":"plugin","depends":"org.ekstep.questionset.quiz,org.ekstep.iterator"},{"id":"org.ekstep.video","ver":"1.3","type":"widget","depends":""}]},"compatibilityVersion":4}}')); - - - -INSERT INTO hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ('do_21351537329604198413739', '{"copyright":"tn","lastStatusChangedOn":"2022-04-13T04:57:21.808+0000","originData":"{\"name\":\"4.8 upcoming batch course\",\"copyType\":\"deep\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Tamil Nadu\",\"MPPS HANUMANNAHALLI\"],\"author\":\"newtncc guy\"}","publish_type":"public","author":"newtncc guy","se_mediumIds":["tn_k-12_5_medium_english","tn_k-12_5_medium_tamil"],"organisation":["Tamil Nadu","MPPS HANUMANNAHALLI"],"children":[{"lastStatusChangedOn":"2022-04-13T04:57:24.497+0000","parent":"do_21351537329604198413739","children":[{"copyright":"Tamil Nadu, PUPS, REDDIYARPATTI","lastStatusChangedOn":"2021-10-29T07:21:31.809+0000","parent":"do_21351537331811942413741","licenseterms":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.","organisation":["Tamil Nadu","PUPS, REDDIYARPATTI"],"mediaType":"content","name":"AK course assess","discussionForum":"{\"enabled\":\"No\"}","createdOn":"2021-10-29T07:18:02.486+0000","createdFor":["01269878797503692810","012741962549116928720"],"channel":"01269878797503692810","assets":[],"plugins":"[{\"identifier\":\"org.ekstep.stage\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.questionset\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.navigation\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.questionset.quiz\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.iterator\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.questionunit\",\"semanticVersion\":\"1.2\"},{\"identifier\":\"org.ekstep.keyboard\",\"semanticVersion\":\"1.1\"},{\"identifier\":\"org.ekstep.questionunit.ftb\",\"semanticVersion\":\"1.1\"},{\"identifier\":\"org.ekstep.questionunit.mcq\",\"semanticVersion\":\"1.3\"},{\"identifier\":\"org.ekstep.questionunit.mtf\",\"semanticVersion\":\"1.2\"},{\"identifier\":\"org.ekstep.summary\",\"semanticVersion\":\"1.0\"}]","lastUpdatedOn":"2021-10-29T07:21:31.809+0000","subject":["Skills","Environmental Science"],"size":585405,"editorState":"{\"plugin\":{\"noOfExtPlugins\":12,\"extPlugins\":[{\"plugin\":\"org.ekstep.contenteditorfunctions\",\"version\":\"1.2\"},{\"plugin\":\"org.ekstep.keyboardshortcuts\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.richtext\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.iterator\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.navigation\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.reviewercomments\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.questionunit.mtf\",\"version\":\"1.2\"},{\"plugin\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.3\"},{\"plugin\":\"org.ekstep.keyboard\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.reorder\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.sequence\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.ftb\",\"version\":\"1.1\"}]},\"stage\":{\"noOfStages\":1,\"currentStage\":\"ce1a09a4-7553-416d-9791-a1c343676425\",\"selectedPluginObject\":\"922b926e-1104-439c-ad35-06301265ea6d\"},\"sidebar\":{\"selectedMenu\":\"settings\"}}","se_topics":[],"streamingUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/ecml/do_21339794950117785611-latest","identifier":"do_21339794950117785611","description":"Enter description for Assessment","gradeLevel":["Class 1"],"ownershipType":["createdBy"],"compatibilityLevel":5,"audience":["Student"],"se_boards":["State (Tamil Nadu)"],"os":["All"],"cloudStorageKey":"content/do_21339794950117785611/artifact/1635492085307_do_21339794950117785611.zip","primaryCategory":"Course Assessment","appIcon":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21339794950117785611/artifact/do_2133016737715240961616_1623739474410_pexels-photo-594364.thumb.jpeg","se_mediums":["English"],"downloadUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21339794950117785611/ak-course-assess_1635492091534_do_21339794950117785611_1.ecar","se_subjects":["Skills","Environmental Science"],"lockKey":"6c7cb7b8-cf43-423b-b146-bbdb21809574","medium":["English"],"framework":"tn_k-12_5","displayScore":true,"posterImage":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_2133016737715240961616/artifact/do_2133016737715240961616_1623739474410_pexels-photo-594364.jpeg","creator":"newtncc","versionKey":"1635492303529","mimeType":"application/vnd.ekstep.ecml-archive","code":"org.sunbird.l1vyPd","license":"CC BY 4.0","maxAttempts":2,"version":2,"prevStatus":"Review","contentType":"SelfAssess","prevState":"Review","language":["English"],"board":"State (Tamil Nadu)","totalQuestions":5,"lastPublishedOn":"2021-10-29T07:21:27.739+0000","totalScore":5,"objectType":"Content","lastUpdatedBy":"deec6352-0f62-4306-9818-aba349a0e0f8","status":"Live","pragma":[],"createdBy":"deec6352-0f62-4306-9818-aba349a0e0f8","dialcodeRequired":"No","lastSubmittedOn":"2021-10-29T07:25:03.508+0000","interceptionPoints":"{}","idealScreenSize":"normal","contentEncoding":"gzip","depth":2,"consumerId":"cb069f8d-e4e1-46c5-831f-d4a83b323ada","lastPublishedBy":"91a81041-bbbd-4bd7-947f-09f9e469213c","topic":[],"se_gradeLevels":["Class 1"],"osId":"org.ekstep.quiz.app","copyrightYear":2021,"se_FWIds":["tn_k-12_5"],"contentDisposition":"inline","additionalCategories":["Classroom Teaching Video"],"previewUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/ecml/do_21339794950117785611-latest","artifactUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21339794950117785611/artifact/1635492085307_do_21339794950117785611.zip","visibility":"Default","credentials":"{\"enabled\":\"No\"}","variants":"{\"full\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21339794950117785611/ak-course-assess_1635492091534_do_21339794950117785611_1.ecar\",\"size\":\"587280\"},\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21339794950117785611/ak-course-assess_1635492091732_do_21339794950117785611_1_SPINE.ecar\",\"size\":\"13396\"}}","index":1,"pkgVersion":1,"idealScreenDensity":"hdpi"},{"copyright":"NCERT","lastStatusChangedOn":"2020-10-14T05:10:21.536+0000","parent":"do_21351537331811942413741","licenseterms":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.","organisation":["NCERT"],"mediaType":"content","name":"UT Self assess1`","createdOn":"2020-10-14T05:03:28.570+0000","createdFor":["01283607456185548825093"],"channel":"01283607456185548825093","assets":[],"plugins":"[{\"identifier\":\"org.ekstep.stage\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.questionset\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.navigation\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.questionset.quiz\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.iterator\",\"semanticVersion\":\"1.0\"},{\"identifier\":\"org.ekstep.questionunit\",\"semanticVersion\":\"1.2\"},{\"identifier\":\"org.ekstep.keyboard\",\"semanticVersion\":\"1.1\"},{\"identifier\":\"org.ekstep.questionunit.ftb\",\"semanticVersion\":\"1.1\"},{\"identifier\":\"org.ekstep.questionunit.mcq\",\"semanticVersion\":\"1.3\"},{\"identifier\":\"org.ekstep.questionunit.mtf\",\"semanticVersion\":\"1.2\"},{\"identifier\":\"org.ekstep.summary\",\"semanticVersion\":\"1.0\"}]","lastUpdatedOn":"2020-10-14T05:10:17.434+0000","subject":["Mathematics"],"size":219320.0,"editorState":"{\"plugin\":{\"noOfExtPlugins\":12,\"extPlugins\":[{\"plugin\":\"org.ekstep.contenteditorfunctions\",\"version\":\"1.2\"},{\"plugin\":\"org.ekstep.keyboardshortcuts\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.richtext\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.iterator\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.navigation\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.reviewercomments\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.questionunit.mtf\",\"version\":\"1.2\"},{\"plugin\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.3\"},{\"plugin\":\"org.ekstep.keyboard\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.reorder\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.sequence\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.ftb\",\"version\":\"1.1\"}]},\"stage\":{\"noOfStages\":1,\"currentStage\":\"73920173-9880-48bc-a29d-2ae495c2d81c\",\"selectedPluginObject\":\"450c8034-83e1-4e56-9c26-b790621b693e\"},\"sidebar\":{\"selectedMenu\":\"settings\"}}","streamingUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/ecml/do_2131289236157972481377-latest","identifier":"do_2131289236157972481377","description":"Enter description for Assessment","gradeLevel":["Class 2"],"ownershipType":["createdBy"],"compatibilityLevel":2,"audience":["Student"],"os":["All"],"primaryCategory":"Explanation Content","appIcon":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_2131289236157972481377/artifact/634x951-potrait_1577356719873.thumb.jpeg","SYS_INTERNAL_LAST_UPDATED_ON":"2020-10-14T05:10:21.544+0000","downloadUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/ecar_files/do_2131289236157972481377/ut-self-assess1_1602652220063_do_2131289236157972481377_1.0.ecar","lockKey":"6baae97d-931e-4933-9dbf-8674bd6c48d9","medium":["English"],"framework":"ncert_k-12","posterImage":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_212921706247528448196/artifact/634x951-potrait_1577356719873.jpeg","creator":"Creator","versionKey":"1602652217434","mimeType":"application/vnd.ekstep.ecml-archive","code":"org.sunbird.RgaKqV","license":"CC BY 4.0","version":2,"prevStatus":"Processing","contentType":"PracticeResource","prevState":"Review","language":["English"],"totalQuestions":3,"lastPublishedOn":"2020-10-14T05:10:20.063+0000","totalScore":3,"objectType":"Content","lastUpdatedBy":"3f0825d1-3543-4354-a5ef-5aed4fbe2af9","status":"Live","pragma":[],"createdBy":"3f0825d1-3543-4354-a5ef-5aed4fbe2af9","dialcodeRequired":"No","lastSubmittedOn":"2020-10-14T05:06:55.143+0000","idealScreenSize":"normal","contentEncoding":"gzip","depth":2,"consumerId":"2eaff3db-cdd1-42e5-a611-bebbf906e6cf","lastPublishedBy":"fc87d47a-dce0-46a6-98b7-b530965fd392","osId":"org.ekstep.quiz.app","copyrightYear":2020,"appId":"preprod.diksha.portal","s3Key":"ecar_files/do_2131289236157972481377/ut-self-assess1_1602652220063_do_2131289236157972481377_1.0.ecar","contentDisposition":"inline","previewUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/ecml/do_2131289236157972481377-latest","artifactUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_2131289236157972481377/artifact/1602652219910_do_2131289236157972481377.zip","visibility":"Default","credentials":"{\"enabled\":\"No\"}","variants":"{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/ecar_files/do_2131289236157972481377/ut-self-assess1_1602652220191_do_2131289236157972481377_1.0_spine.ecar\",\"size\":29949.0}}","index":2,"pkgVersion":1.0,"idealScreenDensity":"hdpi"}],"mediaType":"content","name":"1","discussionForum":{"enabled":"No"},"createdOn":"2022-04-13T04:57:24.497+0000","channel":"01269878797503692810","generateDIALCodes":"No","lastUpdatedOn":"2022-04-13T04:58:05.342+0000","identifier":"do_21351537331811942413741","ownershipType":["createdBy"],"compatibilityLevel":1,"audience":["Student"],"trackable":{"enabled":"No","autoBatch":"No"},"os":["All"],"primaryCategory":"Course Unit","languageCode":["en"],"downloadUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926657_do_21351537329604198413739_1_SPINE.ecar","attributions":[],"versionKey":"1649825844497","mimeType":"application/vnd.ekstep.content-collection","code":"31d50c12-10ed-4de6-9758-7f4f5dbbadc2","license":"CC BY 4.0","leafNodes":["do_21339794950117785611","do_2131289236157972481377"],"version":2,"contentType":"CourseUnit","language":["English"],"lastPublishedOn":"2022-04-13T04:58:46.167+0000","objectType":"Collection","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","leafNodesCount":2,"depth":1,"osId":"org.ekstep.launcher","contentDisposition":"inline","visibility":"Parent","credentials":{"enabled":"No"},"variants":"{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926657_do_21351537329604198413739_1_SPINE.ecar\",\"size\":\"42727\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926838_do_21351537329604198413739_1_ONLINE.ecar\",\"size\":\"7931\"}}","index":1,"pkgVersion":1,"idealScreenDensity":"hdpi"}],"body":null,"mediaType":"content","name":" 4.8.1 RC end to end verification ","toc_url":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/artifact/do_21351537329604198413739_toc.json","batches":[{"createdFor":["01269878797503692810","01275630321992499239077"],"endDate":null,"name":"2","batchId":"01350077393890508864","enrollmentType":"open","enrollmentEndDate":null,"startDate":"2022-03-23","status":1},{"createdFor":["01269878797503692810","01275630321992499239077"],"endDate":"2022-07-31","name":"upcoming ","batchId":"013514875102142464293","enrollmentType":"open","enrollmentEndDate":"2022-07-30","startDate":"2022-05-07","status":0},{"createdFor":["01269878797503692810","01275630321992499239077"],"endDate":null,"name":"4.8.1 End to end","batchId":"013515380194869248299","enrollmentType":"open","enrollmentEndDate":null,"startDate":"2022-04-13","status":1}],"discussionForum":{"enabled":"Yes"},"createdOn":"2022-04-13T04:57:21.808+0000","createdFor":["01269878797503692810","01275630321992499239077"],"channel":"01269878797503692810","generateDIALCodes":"No","lastUpdatedOn":"2022-04-13T04:58:05.342+0000","subject":["Accounting And Auditing"],"size":42727,"publishError":null,"targetMediumIds":["tn_k-12_5_medium_english","tn_k-12_5_medium_tamil"],"identifier":"do_21351537329604198413739","se_gradeLevelIds":["tn_k-12_5_gradelevel_class2"],"description":"Enter description for Course","resourceType":"Course","ownershipType":["createdBy"],"compatibilityLevel":4,"targetBoardIds":["tn_k-12_5_board_statetamilnadu"],"audience":["Student"],"trackable":{"enabled":"Yes","autoBatch":"No"},"se_boards":["State (Tamil Nadu)"],"os":["All"],"primaryCategory":"Course","languageCode":["en"],"se_mediums":["English","Tamil"],"se_subjectIds":["tn_k-12_5_subject_accountingandauditing","tn_k-12_5_subject_mathematics"],"downloadUrl":"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926657_do_21351537329604198413739_1_SPINE.ecar","se_subjects":["Accounting And Auditing"],"lockKey":"63d3921f-defc-4838-b465-6ca222f5d180","attributions":[],"framework":"tn_k-12_5","creator":"Guest name changed","totalCompressedSize":804725,"versionKey":"1649825885342","mimeType":"application/vnd.ekstep.content-collection","sYS_INTERNAL_LAST_UPDATED_ON":"2022-04-13T04:58:46.656+0000","code":"org.sunbird.nldxzf.copy.copy","se_boardIds":["tn_k-12_5_board_statetamilnadu"],"license":"CC BY 4.0","leafNodes":["do_21339794950117785611","do_2131289236157972481377"],"version":2,"contentType":"Course","language":["English"],"lastPublishedOn":"2022-04-13T04:58:46.167+0000","contentTypesCount":"{\"PracticeResource\":1,\"SelfAssess\":1,\"CourseUnit\":1}","objectType":"content","origin":"do_21351486882984755213610","subjectIds":["tn_k-12_5_subject_accountingandauditing"],"status":"Live","targetFWIds":["tn_k-12_5"],"createdBy":"fca2925f-1eee-4654-9177-fece3fd6afc9","dialcodeRequired":"No","lastSubmittedOn":"2022-04-13T04:58:05.031+0000","keywords":["sandhya"],"dialcodes":null,"userConsent":"Yes","idealScreenSize":"normal","contentEncoding":"gzip","leafNodesCount":2,"depth":0,"consumerId":"cb069f8d-e4e1-46c5-831f-d4a83b323ada","lastPublishedBy":"08631a74-4b94-4cf7-a818-831135248a4a","flagReasons":null,"targetSubjectIds":["tn_k-12_5_subject_mathematics"],"mimeTypesCount":"{\"application/vnd.ekstep.ecml-archive\":2,\"application/vnd.ekstep.content-collection\":1}","se_gradeLevels":["Class 2"],"osId":"org.ekstep.quiz.app","copyrightYear":2022,"se_FWIds":["tn_k-12_5"],"appId":"staging.sunbird.portal","s3Key":"content/do_21351537329604198413739/artifact/do_21351537329604198413739_toc.json","contentDisposition":"inline","additionalCategories":["Lesson Plan","Textbook"],"childNodes":["do_21339794950117785611","do_21351537331811942413741","do_2131289236157972481377"],"visibility":"Default","credentials":{"enabled":"Yes"},"targetGradeLevelIds":["tn_k-12_5_gradelevel_class2"],"variants":"{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926657_do_21351537329604198413739_1_SPINE.ecar\",\"size\":\"42727\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926838_do_21351537329604198413739_1_ONLINE.ecar\",\"size\":\"7931\"}}","pkgVersion":1,"idealScreenDensity":"hdpi","reservedDialcodes":{"J3U7Y2":0}}', -''); \ No newline at end of file diff --git a/csp-migrator/src/test/scala/org.sunbird.job.cspmigrator.spec/CSPMigratorSpec.scala b/csp-migrator/src/test/scala/org.sunbird.job.cspmigrator.spec/CSPMigratorSpec.scala deleted file mode 100644 index 584444c70..000000000 --- a/csp-migrator/src/test/scala/org.sunbird.job.cspmigrator.spec/CSPMigratorSpec.scala +++ /dev/null @@ -1,288 +0,0 @@ -package org.sunbird.job.cspmigrator.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, anyBoolean, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.cspmigrator.helpers.{CSPCassandraMigrator, CSPNeo4jMigrator} -import org.sunbird.job.cspmigrator.task.CSPMigratorConfig -import org.sunbird.job.util._ - -import java.io.File -import scala.collection.JavaConverters._ - -class CSPMigratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: CSPMigratorConfig = new CSPMigratorConfig(config) - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - val mockHttpUtil: HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - implicit val mockCloudUtil: CloudStorageUtil = mock[CloudStorageUtil](Mockito.withSettings().serializable()) - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => ex.printStackTrace() - } - } - - "ECML Body" should " get updated with migrate data in cassandra database" in { - - val cspneo4jMigrator = new TestCSPNeo4jMigrator() - val fieldsToMigrate: List[String] = jobConfig.getConfig.getStringList("neo4j_fields_to_migrate.content").asScala.toList - - val objectMetadata = Map[String, AnyRef]("ownershipType" -> Array("createdFor"), "previewUrl" -> "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_31270597860728832015700-latest", - "keywords" -> Array("10 PS BITS"), - "channel" -> "0123207707019919361056", - "downloadUrl" -> "https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31270597860728832015700/examprep_10tm_ps_cha-4-q5_1551025589101_do_31270597860728832015700_1.0.ecar", - "mimeType" -> "application/vnd.ekstep.ecml-archive", - "variants" -> Map[String,AnyRef]( - "spine" -> Map[String,AnyRef]( - "ecarUrl" -> "https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31270597860728832015700/examprep_10tm_ps_cha-4-q5_1551025590393_do_31270597860728832015700_1.0_spine.ecar", - "size" -> 8820.asInstanceOf[Number] - )), - "editorState" -> "{\"plugin\":{\"noOfExtPlugins\":14,\"extPlugins\":[{\"plugin\":\"org.ekstep.contenteditorfunctions\",\"version\":\"1.2\"},{\"plugin\":\"org.ekstep.keyboardshortcuts\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.richtext\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.iterator\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.navigation\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.mathtext\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.libs.ckeditor\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.questionunit\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.keyboard\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.questionunit.mcq\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.mtf\",\"version\":\"1.1\"},{\"plugin\":\"org.ekstep.questionunit.reorder\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.questionunit.sequence\",\"version\":\"1.0\"},{\"plugin\":\"org.ekstep.questionunit.ftb\",\"version\":\"1.0\"}]},\"stage\":{\"noOfStages\":2,\"currentStage\":\"b8b47094-1d69-43a1-9c88-c02e760996c5\"},\"sidebar\":{\"selectedMenu\":\"settings\"}}", - "objectType" -> "Content", - "appIcon" -> "https://upload.wikimedia.org/wikipedia/en/c/c2/CC_201_01R_SDT_2009.jpg", - "primaryCategory" -> "Learning Resource", - "artifactUrl" -> "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31270597860728832015700/artifact/1551025588108_do_31270597860728832015700.zip", - "contentType" -> "Resource", - "identifier" -> "do_31270597860728832015700", - "visibility" -> "Default", - "author" -> "APEKX", - "lastPublishedBy" -> "8587c52d-755b-4473-8d47-4b47f81eb56b", - "version" -> 2.asInstanceOf[Number], - "license" -> "CC BY 4.0", - "prevState" -> "Review", - "size" -> 9312393.asInstanceOf[Number], - "lastPublishedOn" -> "2019-02-24T16 ->26 ->29.101+0000", - "name" -> "\tExamprep_10tm_ps_cha 4-Q5", - "status" -> "Live", - "totalQuestions" -> 1.asInstanceOf[Number], - "code" -> "org.sunbird.zRuXiC", - "description" -> "10 PS BITS", - "streamingUrl" -> "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_31270597860728832015700-latest", - "posterImage" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_31270069910858956813856/artifact/10-ps-tm_1550378309450.png", - "idealScreenSize" -> "normal", - "createdOn" -> "2019-02-24T15:39:39.209+0000", - "copyrightYear" -> 2019.asInstanceOf[Number], - "contentDisposition" -> "inline", - "lastUpdatedOn" -> "2019-02-24T16:26:24.241+0000", - "dialcodeRequired" -> "No", - "owner" -> "IT_CELL CSE_AP AMARAVATI", - "creator" -> "MALLIMOGGALA SUBBAYYA", - "totalScore" -> 1.asInstanceOf[Number], - "pkgVersion" -> 1.asInstanceOf[Number], - "versionKey" -> "1551025584241", - "idealScreenDensity" -> "hdpi", - "framework" -> "ap_k-12_1", - "s3Key" -> "ecar_files/do_31270597860728832015700/examprep_10tm_ps_cha-4-q5_1551025589101_do_31270597860728832015700_1.0.ecar", - "lastSubmittedOn" -> "2019-02-24T15 ->44:04.741+0000", - "compatibilityLevel" -> 4.asInstanceOf[Number], - "ownedBy" -> "01232154589404364810952", - "board" -> "State (Andhra Pradesh)", - "resourceType" -> "Learn" - ) - - when(mockCloudUtil.uploadFile(anyString(),any[File](),Option(anyBoolean()),anyString())).thenReturn(Array.empty[String]) - val cspCassandraMigrator = new TestCSPCassandraMigrator() - - cspCassandraMigrator.process(objectMetadata, jobConfig, mockHttpUtil, cassandraUtil, mockCloudUtil) - - when(mockHttpUtil.getSize(anyString(), any())).thenReturn(200) - - val migratedMetadata = cspneo4jMigrator.process(objectMetadata, jobConfig, mockHttpUtil, mockCloudUtil) - fieldsToMigrate.map(migrateField => { - jobConfig.keyValueMigrateStrings.keySet().toArray().map(key => { - assert(migratedMetadata.getOrElse(migrateField, "") == null || !migratedMetadata.getOrElse(migrateField, "").asInstanceOf[String].contains(key)) - }) - }) - } - - "Collection Hierarchy" should " get updated with migrate data in cassandra database" in { - val cspMigrator = new TestCSPNeo4jMigrator() - - val objectMetadata = Map[String, AnyRef]( - "ownershipType" -> Array( - "createdBy" - ), - "copyright" -> "tn", - "se_gradeLevelIds" -> Array( - "tn_k-12_5_gradelevel_class2" - ), - "keywords" -> Array( - "sandhya" - ), - "subject" -> Array( - "Accounting And Auditing" - ), - "targetMediumIds" -> Array( - "tn_k-12_5_medium_english", - "tn_k-12_5_medium_tamil" - ), - "channel" -> "01269878797503692810", - "downloadUrl" -> "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926657_do_21351537329604198413739_1_SPINE.ecar", - "mimeType" -> "application/vnd.ekstep.content-collection", - "variants" -> Map[String, AnyRef] ( - "spine" -> Map[String, AnyRef] ( - "ecarUrl" -> "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926657_do_21351537329604198413739_1_SPINE.ecar", - "size" -> "42727" - ), - "online" -> Map[String, AnyRef] ( - "ecarUrl" -> "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/4.8.1-rc-end-to-end-verification_1649825926838_do_21351537329604198413739_1_ONLINE.ecar", - "size" -> "7931" - ) - ), - "leafNodes" -> Array( - "do_21339794950117785611", - "do_2131289236157972481377" - ), - "targetGradeLevelIds" -> Array( - "tn_k-12_5_gradelevel_class2" - ), - "objectType" -> "Content", - "se_mediums" -> Array( - "English", - "Tamil" - ), - "primaryCategory" -> "Course", - "appId" -> "staging.sunbird.portal", - "contentEncoding" -> "gzip", - "lockKey" -> "63d3921f-defc-4838-b465-6ca222f5d180", - "generateDIALCodes" -> "No", - "totalCompressedSize" -> 804725.asInstanceOf[Number], - "mimeTypesCount" -> "{\"application/vnd.ekstep.ecml-archive\":2,\"application/vnd.ekstep.content-collection\":1}", - "sYS_INTERNAL_LAST_UPDATED_ON" -> "2022-04-13T05:00:21.380+0000", - "contentType" -> "Course", - "se_gradeLevels" -> Array( - "Class 2" - ), - "identifier" -> "do_21351537329604198413739", - "se_boardIds" -> Array( - "tn_k-12_5_board_statetamilnadu" - ), - "subjectIds" -> Array( - "tn_k-12_5_subject_accountingandauditing" - ), - "toc_url" -> "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21351537329604198413739/artifact/do_21351537329604198413739_toc.json", - "visibility" -> "Default", - "contentTypesCount" -> "{\"PracticeResource\":1,\"SelfAssess\":1,\"CourseUnit\":1}", - "author" -> "newtncc guy", - "consumerId" -> "cb069f8d-e4e1-46c5-831f-d4a83b323ada", - "childNodes" -> Array( - "do_21339794950117785611", - "do_21351537331811942413741", - "do_2131289236157972481377" - ), - "discussionForum" -> { - "enabled" -> "Yes" - }, - "mediaType" -> "content", - "osId" -> "org.ekstep.quiz.app", - "lastPublishedBy" -> "08631a74-4b94-4cf7-a818-831135248a4a", - "version" -> 2.asInstanceOf[Number], - "se_subjects" -> Array( - "Accounting And Auditing" - ), - "license" -> "CC BY 4.0", - "prevState" -> "Review", - "size" -> 42727.asInstanceOf[Number], - "lastPublishedOn" -> "2022-04-13T04:58:46.167+0000", - "name" -> " 4.8.1 RC end to end verification ", - "targetBoardIds" -> Array( - "tn_k-12_5_board_statetamilnadu" - ), - "status" -> "Live", - "code" -> "org.sunbird.nldxzf.copy.copy", - "credentials" -> { - "enabled" -> "Yes" - }, - "prevStatus" -> "Processing", - "origin" -> "do_21351486882984755213610", - "description" -> "Enter description for Course", - "idealScreenSize" -> "normal", - "createdOn" -> "2022-04-13T04:57:21.808+0000", - "reservedDialcodes" -> Map[String, AnyRef] ( - "J3U7Y2" -> 0.asInstanceOf[Number] - ), - "se_boards" -> Array( - "State (Tamil Nadu)" - ), - "targetSubjectIds" -> Array( - "tn_k-12_5_subject_mathematics" - ), - "se_mediumIds" -> Array( - "tn_k-12_5_medium_english", - "tn_k-12_5_medium_tamil" - ), - "copyrightYear" -> 2022.asInstanceOf[Number], - "contentDisposition" -> "inline", - "additionalCategories" -> Array( - "Lesson Plan", - "Textbook" - ), - "dialcodeRequired" -> "No", - "lastStatusChangedOn" -> "2022-04-13T04:58:46.915+0000", - "createdFor" -> Array( - "01269878797503692810", - "01275630321992499239077" - ), - "creator" -> "Guest name changed", - "se_subjectIds" -> Array( - "tn_k-12_5_subject_accountingandauditing", - "tn_k-12_5_subject_mathematics" - ), - "se_FWIds" -> Array( - "tn_k-12_5" - ), - "targetFWIds" -> Array( - "tn_k-12_5" - ), - "pkgVersion" -> 1.asInstanceOf[Number], - "versionKey" -> "1649825885342", - "idealScreenDensity" -> "hdpi", - "framework" -> "tn_k-12_5", - "dialcodes" -> Array("J3U7Y2"), - "depth" -> 0.asInstanceOf[Number], - "s3Key" -> "content/do_21351537329604198413739/artifact/do_21351537329604198413739_toc.json", - "lastSubmittedOn" -> "2022-04-13T04:58:05.031+0000", - "createdBy" -> "fca2925f-1eee-4654-9177-fece3fd6afc9", - "compatibilityLevel" -> 4.asInstanceOf[Number], - "leafNodesCount" -> 2.asInstanceOf[Number], - "userConsent" -> "Yes", - "resourceType" -> "Course" - ) - - when(mockCloudUtil.uploadFile(anyString(),any[File](),Option(anyBoolean()),anyString())).thenReturn(Array.empty[String]) - when(mockHttpUtil.getSize(anyString(), any())).thenReturn(200) - val fieldsToMigrate: List[String] = jobConfig.getConfig.getStringList("neo4j_fields_to_migrate.collection").asScala.toList - val migratedMetadata = cspMigrator.process(objectMetadata, jobConfig, mockHttpUtil, mockCloudUtil) - fieldsToMigrate.map(migrateField => { - jobConfig.keyValueMigrateStrings.keySet().toArray().map(key => { - assert(!migratedMetadata.getOrElse(migrateField, "").asInstanceOf[String].contains(key)) - }) - }) - } - -} - -class TestCSPNeo4jMigrator extends CSPNeo4jMigrator {} - -class TestCSPCassandraMigrator extends CSPCassandraMigrator {} diff --git a/csp-migrator/src/test/scala/org.sunbird.job.cspmigrator.spec/URLExtractorSpec.scala b/csp-migrator/src/test/scala/org.sunbird.job.cspmigrator.spec/URLExtractorSpec.scala deleted file mode 100644 index 4c8fdabd3..000000000 --- a/csp-migrator/src/test/scala/org.sunbird.job.cspmigrator.spec/URLExtractorSpec.scala +++ /dev/null @@ -1,30 +0,0 @@ -package org.sunbird.job.cspmigrator.spec - -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.cspmigrator.helpers.URLExtractor - -class URLExtractorSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - - } - - "Extractor" should "extract all URLs from the ECML body" in { - val body: String = "{\"theme\":{\"id\":\"theme\",\"version\":\"1.0\",\"startStage\":\"c3e9064a-9470-40b8-a38a-cd2286a01eeb\",\"stage\":[{\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"id\":\"c3e9064a-9470-40b8-a38a-cd2286a01eeb\",\"rotate\":null,\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true,\\\"color\\\":\\\"#FFFFFF\\\",\\\"instructions\\\":\\\"\\\",\\\"genieControls\\\":false}\"},\"manifest\":{\"media\":[{\"assetId\":\"do_213613202536579072134\"}]},\"image\":[{\"asset\":\"do_213613202536579072134\",\"x\":20,\"y\":20,\"w\":49.72,\"h\":63.31,\"rotate\":0,\"z-index\":0,\"id\":\"ea1dc233-9cbb-4617-b137-be4d8c8c6150\",\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true}\"}}]}],\"manifest\":{\"media\":[{\"id\":\"2288347e-475b-4e93-a72e-e780977e7c13\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/controller/navigation_ctrl.js\",\"type\":\"js\"},{\"id\":\"14164b50-a751-4d44-bae1-8a4d5016266a\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/templates/navigation.html\",\"type\":\"js\"},{\"id\":\"org.ekstep.navigation\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.navigation_manifest\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"do_213613202536579072134\",\"src\":\"https://drive.google.com/uc?export=download&id=1wygNKORDwe6rJ7MTW_ohbUVX4Qw4nWa0\",\"type\":\"image\"}]},\"plugin-manifest\":{\"plugin\":[{\"id\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"type\":\"plugin\",\"depends\":\"\"}]},\"compatibilityVersion\":2}}" - val extractor = new TestURLExtractor() - val extractedURLs: List[String] = extractor.extractUrls(body) - assert(extractedURLs.nonEmpty) - } -} - - -class TestURLExtractor extends URLExtractor {} - - diff --git a/dialcode-context-updater/pom.xml b/dialcode-context-updater/pom.xml deleted file mode 100644 index 915b045f1..000000000 --- a/dialcode-context-updater/pom.xml +++ /dev/null @@ -1,215 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - dialcode-context-updater - 1.0.0 - jar - dialcode-context-updater - - Dialcode Context Updater Flink Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.sunbird.job.dialcodecontextupdater.task.DialcodeContextUpdaterStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - dialcode-context-updater-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/dialcode-context-updater/src/main/resources/contextMapping.json b/dialcode-context-updater/src/main/resources/contextMapping.json deleted file mode 100644 index 27b93b138..000000000 --- a/dialcode-context-updater/src/main/resources/contextMapping.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "$defs": { - "content": { - "@type": "sbed:Content", - "identifier": "identifier", - "name": "name", - "status": "status", - "channel": "channel", - "lastPublishedOn": "lastPublishedOn", - "primaryCategory": "primaryCategory" - }, - "collection": { - "@type": "sbed:Collection", - "identifier": "identifier", - "name": "name", - "status": "status", - "channel": "channel", - "lastPublishedOn": "lastPublishedOn", - "primaryCategory": "primaryCategory" - }, - "collectionUnit": { - "@type": "sbed:CollectionUnit", - "identifier": "identifier", - "name": "name" - }, - "framework": { - "@type": "sbed:Framework", - "identifier": "framework", - "board": "board", - "medium": "medium", - "class": "gradeLevel", - "subject": "subject", - "topic": "topic", - "learningOutcome": "learningOutcome", - "purpose": "purpose" - } - }, - "digital_textbook": { - "@type": "sbed:TextBook", - "$ref": "#/$defs/collection", - "framework" : { "$ref" : "#/$defs/framework" } - }, - "textbook_unit": { - "@type": "sbed:TextBookUnit", - "$ref": "#/$defs/collectionUnit", - "parentInfo": {"$ref": "#/digital_textbook"} - }, - "course": { - "@type": "sbed:Course", - "$ref": "#/$defs/collection", - "framework": { "$ref" : "#/$defs/framework" } - }, - "course_unit": { - "@type": "sbed:CourseUnit", - "$ref": "#/$defs/collectionUnit", - "parentInfo": {"$ref": "#/course" } - } -} \ No newline at end of file diff --git a/dialcode-context-updater/src/main/resources/dialcode-context-updater.conf b/dialcode-context-updater/src/main/resources/dialcode-context-updater.conf deleted file mode 100644 index d57369635..000000000 --- a/dialcode-context-updater/src/main/resources/dialcode-context-updater.conf +++ /dev/null @@ -1,29 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.dialcode.context.job.request" - failed.topic = "sunbirddev.dialcode.context.job.request.failed" - groupId = "sunbirddev-dialcode-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - dialcode-context-updater.parallelism = 1 -} - -dialcode_context_updater { - actions="dialcode-context-update" - search_mode="Collection" - context_map_path = "https://raw.githubusercontent.com/project-sunbird/knowledge-platform-jobs/master/dialcode-context-updater/src/main/resources/contextMapping.json" - identifier_search_fields = ["identifier", "primaryCategory", "channel"] - dial_code_context_read_api_path = "/dialcode/v4/read/" - dial_code_context_update_api_path = "/dialcode/v4/update/" -} - -service { - search.basePath = "http://11.2.6.6/search" - dial_service.basePath = "http://11.2.6.6/dial" -} - -es_sync_wait_time = 5000 \ No newline at end of file diff --git a/dialcode-context-updater/src/main/resources/log4j.properties b/dialcode-context-updater/src/main/resources/log4j.properties deleted file mode 100644 index 74377e41c..000000000 --- a/dialcode-context-updater/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=dialcode-context-updater.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/domain/Event.scala b/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/domain/Event.scala deleted file mode 100644 index 6314fd9a6..000000000 --- a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/domain/Event.scala +++ /dev/null @@ -1,26 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.domain - -import org.sunbird.job.dialcodecontextupdater.task.DialcodeContextUpdaterConfig -import org.sunbird.job.domain.reader.JobRequest - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "dialcode-context-updater" - - def eData: Map[String, AnyRef] = readOrDefault("edata", Map()).asInstanceOf[Map[String, AnyRef]] - - def obj: Map[String, AnyRef] = readOrDefault("object", Map()).asInstanceOf[Map[String, AnyRef]] - - def channel: String = readOrDefault[String]("context.channel", "") - - def action: String = readOrDefault[String]("edata.action", "") - - def dialcode: String = readOrDefault[String]("edata.dialcode", "") - - def identifier: String = readOrDefault[String]("edata.identifier", "") - - - def isValid(): Boolean = { - dialcode.nonEmpty - } -} \ No newline at end of file diff --git a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/functions/DialcodeContextUpdaterFunction.scala b/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/functions/DialcodeContextUpdaterFunction.scala deleted file mode 100644 index 7e3e6b4ff..000000000 --- a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/functions/DialcodeContextUpdaterFunction.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.dialcodecontextupdater.domain.Event -import org.sunbird.job.dialcodecontextupdater.helpers.DialcodeContextUpdater -import org.sunbird.job.dialcodecontextupdater.task.DialcodeContextUpdaterConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.ServerException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.util._ -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.util - -class DialcodeContextUpdaterFunction(config: DialcodeContextUpdaterConfig, httpUtil: HttpUtil) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with DialcodeContextUpdater with FailedEventHelper { - - private[this] lazy val logger = LoggerFactory.getLogger(classOf[DialcodeContextUpdaterFunction]) - lazy val defCache: DefinitionCache = new DefinitionCache() - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount, config.errorEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - - try { - if (event.isValid()) { - updateContext(config, event, httpUtil) - } else { - logger.info("DialcodeContextUpdaterFunction::processElement:: Event is not qualified for dial code context update for dial code : " + event.dialcode) - metrics.incCounter(config.skippedEventCount) - } - } catch { - case e: ServerException => - logger.error("DialcodeContextUpdaterFunction :: Message processing failed for mid : " + event.mid() + " || " + event , e) - } - } - - -} diff --git a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/helpers/DialcodeContextUpdater.scala b/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/helpers/DialcodeContextUpdater.scala deleted file mode 100644 index f8bd5db23..000000000 --- a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/helpers/DialcodeContextUpdater.scala +++ /dev/null @@ -1,205 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.dialcodecontextupdater.domain.Event -import org.sunbird.job.dialcodecontextupdater.task.DialcodeContextUpdaterConfig -import org.sunbird.job.dialcodecontextupdater.util.DialcodeContextUpdaterConstants -import org.sunbird.job.exception.ServerException -import org.sunbird.job.util._ - -import java.util - - -trait DialcodeContextUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[DialcodeContextUpdater]) - - def updateContext(config: DialcodeContextUpdaterConfig, event: Event, httpUtil: HttpUtil): Map[String, AnyRef] = { - logger.info("DialcodeContextUpdater::updateContext:: Processing event for dial code context update operation for event object : " + event.obj) - logger.info("DialcodeContextUpdater::updateContext:: event edata : " + event.eData) - val identifier = event.identifier - val dialcode = event.dialcode - val channel = event.channel - if(identifier.nonEmpty && event.action.equalsIgnoreCase("dialcode-context-update")) { - // 1. Read primaryCategory of the content identifier using Search API - // 2. Fetch contextMap for the content primaryCategory - // 3. Filter for search fields from the contextMap - // 4. Call Search API with dialcode and mode=collection - // 5. Fetch unit and rootNode output from the response - // 6. compose contextInfo and invoke updateDIALcode v4 API - logger.info("DialcodeContextUpdater::updateContext:: Context Update Starting for dialcode: " + dialcode + " || identifier: " + identifier + " || channel: " + channel) - val identifierObj = try { - searchContent("", identifier, config.identifierSearchFields, config, httpUtil) - } catch { - case se: ServerException => if(se.getMessage.contains("No content linking was found for dialcode")) { - try { - Thread.sleep(config.nodeESSyncWaitTime) - searchContent("", identifier, config.identifierSearchFields, config, httpUtil) - } - catch { - case ie: InterruptedException => throw new ServerException("ERR_DIAL_CONTENT_NOT_FOUND", "No content linking was found for dialcode: " + dialcode + " - to identifier: " + identifier) - case ex: Exception => throw ex - } - } else throw se - case ex: Exception => throw ex - } - - logger.info("DialcodeContextUpdater:: updateContext:: config.contextMapFilePath: " + config.contextMapFilePath) - - if(config.contextMapFilePath.isEmpty) { - throw new ServerException("ERR_CONTEXT_MAP_NOT_FOUND", "Context mapping file path was not found for dialcode: " + dialcode + " - to identifier: " + identifier) - } - - val primaryCategory = identifierObj.getOrElse("primaryCategory","").asInstanceOf[String].toLowerCase.replaceAll(" ","_") - val contextMap: Map[String, AnyRef] = getContextMapFields(primaryCategory, config.contextMapFilePath) - val contextSearchFields: List[String] = fetchFieldsFromMap(contextMap).distinct.filter(rec => !rec.startsWith("@")) ++ List("origin", "originData") - logger.info("DialcodeContextUpdater:: updateContext:: searchFields: " + contextSearchFields) - - val contextInfoSearchData: Map[String, AnyRef] = try { - searchContent(dialcode, identifier, contextSearchFields, config, httpUtil) - } catch { - case se: ServerException => if(se.getMessage.contains("No content linking was found for dialcode")) { - try { - Thread.sleep(config.nodeESSyncWaitTime) - searchContent(dialcode, identifier, contextSearchFields, config, httpUtil) - } - catch { - case ie: InterruptedException => throw new ServerException("ERR_DIAL_CONTENT_NOT_FOUND", "No content linking was found for dialcode: " + dialcode + " - to identifier: " + identifier) - case ex: Exception => throw ex - } - } else throw se - case ex: Exception => throw ex - } - logger.info("DialcodeContextUpdater:: updateContext:: contextInfoSearchData: " + ScalaJsonUtil.serialize(contextInfoSearchData)) - - // Filter for necessary fields - val filteredParentInfo = if(contextInfoSearchData.contains("parentInfo")) contextInfoSearchData.getOrElse("parentInfo", Map.empty[String, AnyRef]).asInstanceOf[Map[String, AnyRef]].filter(rec => contextSearchFields.contains(rec._1)) else Map.empty[String, AnyRef] - val inputContextData = contextInfoSearchData.filter(rec => contextSearchFields.contains(rec._1)) - - val finalFilteredData = if(filteredParentInfo!=null && filteredParentInfo.nonEmpty) inputContextData + ("parentInfo" -> filteredParentInfo) else inputContextData - logger.info("DialcodeContextUpdater:: updateContext:: finalFilteredData: " + finalFilteredData) - - val contextDataToUpdate = contextMap.map(rec => { - rec._2 match { - case stringVal: String => if(!rec._1.equalsIgnoreCase("@type") && finalFilteredData.contains(stringVal)) (rec._1 -> finalFilteredData(stringVal)) else (rec._1 -> rec._2) - case objectVal: Map[String, AnyRef] => (rec._1 -> objectVal.map(record => { - record._2 match { - case stringValSubLevel: String => - if (record._1.equalsIgnoreCase("@type")) (record._1 -> stringValSubLevel) - else if (!record._1.equalsIgnoreCase("@type") && finalFilteredData.contains(rec._1) && finalFilteredData(rec._1).isInstanceOf[String]) if(finalFilteredData.contains(stringValSubLevel))(record._1 -> finalFilteredData(stringValSubLevel)) else (record._1 -> null) - else if (!record._1.equalsIgnoreCase("@type") && finalFilteredData.contains(rec._1) && finalFilteredData(rec._1).asInstanceOf[Map[String, AnyRef]].contains(stringValSubLevel)) (record._1 -> finalFilteredData(rec._1).asInstanceOf[Map[String, AnyRef]](stringValSubLevel)) - else (record._1 -> null) - case objectValSubLevel: Map[String, AnyRef] => (record._1 -> objectValSubLevel.map(l2Record => { - if(finalFilteredData.contains(rec._1) && finalFilteredData(rec._1).asInstanceOf[Map[String, AnyRef]].contains(l2Record._2.asInstanceOf[String])) - (l2Record._1 -> finalFilteredData(rec._1).asInstanceOf[Map[String, AnyRef]](l2Record._2.asInstanceOf[String])) else (l2Record._1 -> null) - }).filter(checkRec => checkRec._2!=null)) - } - }).filter(filterRec=>filterRec._2!=null)) - } - }) - - logger.info("DialcodeContextUpdater:: updateContext:: serialize - contentDataToUpsert: " + ScalaJsonUtil.serialize(contextDataToUpdate)) - - updateDIALContext(config, dialcode, channel, ScalaJsonUtil.serialize(contextDataToUpdate))(httpUtil) - } else { - // check if contextInfo is available in metadata. If Yes, update to null - val dialCodeInfo = readDialCode(config, dialcode, channel)(httpUtil) - if(dialCodeInfo.contains("contextInfo") && dialCodeInfo("contextInfo") != null) - updateDIALContext(config, dialcode, channel, null)(httpUtil) - } - - readDialCode(config, dialcode, channel)(httpUtil) - } - - def readDialCode(config: DialcodeContextUpdaterConfig, dialcode: String, channel: String) (implicit httpUtil: HttpUtil): Map[String, AnyRef] = { - val dialCodeContextReadUrl = config.dialServiceBaseUrl + config.dialcodeContextReadPath + dialcode - val headers = Map[String, String]("X-Channel-Id" -> channel, "Content-Type"->"application/json") - val dialcodeResponse: HTTPResponse = httpUtil.get(dialCodeContextReadUrl, headers) - val obj = JSONUtil.deserialize[Map[String, AnyRef]](dialcodeResponse.body) - obj("result").asInstanceOf[Map[String, AnyRef]]("dialcode").asInstanceOf[Map[String, AnyRef]] - } - - def getContextMapFields(contextType: String, contextMapFileURL: String): Map[String, AnyRef] = { - try { - val contextMap: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](ComplexJsonCompiler.createConsolidatedSchema(contextMapFileURL, contextType)) - logger.info("DialcodeContextUpdater:: getContextSearchFields:: contextMap: " + contextMap) - contextMap - } catch { - case e: Exception => - throw new ServerException("ERR_CONTEXT_MAP_READING","Error in reading context map file: " + e.getMessage) - } - } - - def fetchFieldsFromMap(inputMap: Map[String, AnyRef]): List[String] = { - inputMap.flatMap(rec => { - rec._2 match { - case strVal: String => List(strVal) - case objVal: Map[String, AnyRef] => fetchFieldsFromMap(objVal) - } - }).toList - } - - - private def searchContent(dialcode: String, identifier: String, searchFields: List[String], config: DialcodeContextUpdaterConfig, httpUtil: HttpUtil): Map[String, AnyRef] = { - val reqMap = new java.util.HashMap[String, AnyRef]() { - put(DialcodeContextUpdaterConstants.REQUEST, new java.util.HashMap[String, AnyRef]() { - put(DialcodeContextUpdaterConstants.FILTERS, new java.util.HashMap[String, AnyRef]() { - put(DialcodeContextUpdaterConstants.VISIBILITY, new util.ArrayList[String]() { - add("Default") - add("Parent") - }) - put(DialcodeContextUpdaterConstants.IDENTIFIER, identifier) - if(dialcode.nonEmpty) { - put(DialcodeContextUpdaterConstants.DIALCODES, new util.ArrayList[String](){ - add(dialcode) - }) - } - }) - put(DialcodeContextUpdaterConstants.FIELDS, searchFields.toArray[String]) - if(dialcode.nonEmpty) { - put(DialcodeContextUpdaterConstants.SEARCH_MODE, config.searchMode) - } - }) - } - - val requestUrl = s"${config.searchServiceBaseUrl}/v3/search" - logger.info("DialcodeContextUpdater :: searchContent :: Search Content requestUrl: " + requestUrl) - logger.info("DialcodeContextUpdater :: searchContent :: Search Content reqMap: " + reqMap) - val httpResponse = httpUtil.post(requestUrl, JSONUtil.serialize(reqMap)) - if (httpResponse.status == 200) { - val response = JSONUtil.deserialize[Map[String, AnyRef]](httpResponse.body) - val result = response.getOrElse("result", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - val content = result.getOrElse("content", List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String,AnyRef]]] - val collections = result.getOrElse("collections", List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String,AnyRef]]] - val count = result.getOrElse("count", 0).asInstanceOf[Int] - if(count>0) { - if(collections.isEmpty) { - content.head - } else { - content.head + ("parentInfo" -> collections.filter(record => !record.contains("origin")).head) - } - } else { - throw new ServerException("ERR_DIAL_CONTENT_NOT_FOUND", "No content linking was found for dialcode: " + dialcode + " - to identifier: " + identifier) - } - } else { - throw new ServerException("ERR_SEARCH_API_CALL", "Invalid Response received while searching content for dialcode: " + dialcode) - } - } - - def updateDIALContext(config: DialcodeContextUpdaterConfig, dialcode: String, channel: String, contextInfo: String) (implicit httpUtil: HttpUtil): Boolean = { - val dialCodeContextUpdateUrl = config.dialServiceBaseUrl + config.dialcodeContextUpdatePath + dialcode - val requestBody = if(contextInfo!=null) "{\"request\": {\"dialcode\": {\"contextInfo\":" + contextInfo + "}}}" - else "{\"request\": {\"dialcode\": {\"contextInfo\": null }}}" - logger.info("DialcodeContextUpdater :: updateDIALContext :: Update context requestBody: " + requestBody) - val headers = Map[String, String]("X-Channel-Id" -> channel, "Content-Type"->"application/json") - val response:HTTPResponse = httpUtil.patch(dialCodeContextUpdateUrl, requestBody, headers) - - if(response.status == 200){ - true - } else { - logger.error("Error while updating context for dialcode: " + dialcode + " :: "+response.body) - throw new ServerException("ERR_DIAL_CONTEXT_UPDATE_API","Error while updating context for dialcode: " + dialcode + " :: "+response.body) - } - } - -} diff --git a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/task/DialcodeContextUpdaterConfig.scala b/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/task/DialcodeContextUpdaterConfig.scala deleted file mode 100644 index 6589d881a..000000000 --- a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/task/DialcodeContextUpdaterConfig.scala +++ /dev/null @@ -1,63 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.dialcodecontextupdater.domain.Event - -import java.util -import scala.collection.JavaConverters._ - -class DialcodeContextUpdaterConfig(override val config: Config) extends BaseJobConfig(config, "dialcode-context-updater") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val dialcodeContextUpdaterTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val kafkaFailedTopic: String = config.getString("kafka.failed.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - val dialcodeContextUpdaterParallelism: Int = if (config.hasPath("task.dialcode-context-updater.parallelism")) - config.getInt("task.dialcode-context-updater.parallelism") else 1 - - // Metric List - val totalEventsCount = "total-message-count" - val successEventCount = "success-message-count" - val failedEventCount = "failed-message-count" - val skippedEventCount = "skipped-message-count" - val errorEventCount = "error-message-count" - - // Consumers - val eventConsumer = "dialcode-context-updater-consumer" - val dialcodeContextUpdaterFunction = "dialcode-context-updater-process" - val dialcodeContextUpdaterEventProducer = "dialcode-context-updater-producer" - - // Tags - val dialcodeContextUpdaterOutputTag: OutputTag[Event] = OutputTag[Event]("dialcode-context-updater") - val failedEventOutTag: OutputTag[String] = OutputTag[String]("dialcode-context-updater-failed-event") - - val configVersion = "1.0" - - // DB Config - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - - val graphRoutePath: String = config.getString("neo4j.routePath") - val graphName: String = config.getString("neo4j.graph") - val dbHitEventCount = "db-hit-events-count" - - val searchServiceBaseUrl : String = config.getString("service.search.basePath") - val dialServiceBaseUrl : String = config.getString("service.dial_service.basePath") - val dialcodeContextUpdatePath : String = config.getString("dialcode_context_updater.dial_code_context_update_api_path") - val dialcodeContextReadPath : String = config.getString("dialcode_context_updater.dial_code_context_read_api_path") - - val searchMode: String = if (config.hasPath("dialcode_context_updater.search_mode")) config.getString("dialcode_context_updater.search_mode") else "Collection" - val contextMapFilePath: String = if (config.hasPath("dialcode_context_updater.context_map_path")) config.getString("dialcode_context_updater.context_map_path") else "" - - val identifierSearchFields: List[String] = if (config.hasPath("dialcode_context_updater.identifier_search_fields")) config.getStringList("dialcode_context_updater.identifier_search_fields").asScala.toList else List("identifier", "primaryCategory","channel") - val nodeESSyncWaitTime: Int = if (config.hasPath("es_sync_wait_time")) config.getInt("es_sync_wait_time") else 2000 -} diff --git a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/task/DialcodeContextUpdaterStreamTask.scala b/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/task/DialcodeContextUpdaterStreamTask.scala deleted file mode 100644 index 26eb0c9bc..000000000 --- a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/task/DialcodeContextUpdaterStreamTask.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.dialcodecontextupdater.domain.Event -import org.sunbird.job.dialcodecontextupdater.functions.DialcodeContextUpdaterFunction -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - -import java.io.File -import java.util - - -class DialcodeContextUpdaterStreamTask(config: DialcodeContextUpdaterConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val processStreamTask = env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.eventConsumer) - .uid(config.eventConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new DialcodeContextUpdaterFunction(config, httpUtil)) - .name(config.dialcodeContextUpdaterFunction) - .uid(config.dialcodeContextUpdaterFunction) - .setParallelism(config.parallelism) - - processStreamTask.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaFailedTopic)) - - env.execute(config.jobName) - } -} - - -object DialcodeContextUpdaterStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("dialcode-context-updater.conf").withFallback(ConfigFactory.systemEnvironment())) - val dialcodeContextUpdaterConfig = new DialcodeContextUpdaterConfig(config) - val kafkaUtil = new FlinkKafkaConnector(dialcodeContextUpdaterConfig) - val httpUtil = new HttpUtil - val task = new DialcodeContextUpdaterStreamTask(dialcodeContextUpdaterConfig, kafkaUtil, httpUtil) - task.process() - } -} - diff --git a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/util/DialcodeContextUpdaterConstants.scala b/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/util/DialcodeContextUpdaterConstants.scala deleted file mode 100644 index 482c47281..000000000 --- a/dialcode-context-updater/src/main/scala/org/sunbird/job/dialcodecontextupdater/util/DialcodeContextUpdaterConstants.scala +++ /dev/null @@ -1,36 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.util - -object DialcodeContextUpdaterConstants { - - val COMMA_SEPARATOR = ", " - val X_CHANNEL_ID = "X-Channel-Id" - val UPDATE = "UPDATE" - val CREATE = "CREATE" - val CONTENT_TYPE = "contentType" - val IDENTIFIER = "identifier" - val DESCRIPTION = "description" - val LIMIT = "limit" - val NAME = "name" - val VISIBILITY = "visibility" - val CHANNEL = "channel" - val CONTENT = "content" - val STATUS = "status" - val ORIGIN = "origin" - val OBJECT_TYPE = "objectType" - val ORIGIN_DATA = "originData" - val VERSION_KEY = "versionKey" - val RESULT = "result" - val CONTENT_TYPE_HEADER = "Content-Type" - val BEARER = "Bearer " - val MESSAGES = "messages" - val APPLICATION_JSON = "application/json" - val REQUEST = "request" - val SEARCH = "search" - val FILTERS = "filters" - val SEARCH_MODE = "mode" - val FIELDS = "fields" - val X_AUTHENTICATED_USER_TOKEN = "x-authenticated-user-token" - val CATEGORIES = "categories" - val TTL = "ttl" - val DIALCODES = "dialcodes" -} diff --git a/dialcode-context-updater/src/test/resources/test.conf b/dialcode-context-updater/src/test/resources/test.conf deleted file mode 100644 index 2a46377a7..000000000 --- a/dialcode-context-updater/src/test/resources/test.conf +++ /dev/null @@ -1,35 +0,0 @@ -include "base-test.conf" - -kafka { - input.topic = "sunbirddev.dialcode.context.job.request" - failed.topic = "sunbirddev.dialcode.context.job.request.failed" - groupId = "sunbirddev-dialcode-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - content-auto-creator.parallelism = 1 -} - -redis { - database { - relationCache.id = 10 - collectionCache.id = 5 - } -} - -dialcode_context_updater { - actions="dialcode-context-update" - search_mode="Collection" - context_map_path = "https://raw.githubusercontent.com/project-sunbird/knowledge-platform-jobs/master/dialcode-context-updater/src/main/resources/contextMapping.json" - identifier_search_fields = ["identifier", "primaryCategory", "channel"] - dial_code_context_read_api_path = "/dialcode/v4/read/" - dial_code_context_update_api_path = "/dialcode/v4/update/" -} - -service { - search.basePath = "http://11.2.6.6/search" - dial_service.basePath = "http://11.2.6.6/dial" - content_service.basePath = "http://11.2.6.6/content" -} diff --git a/dialcode-context-updater/src/test/resources/test.cql b/dialcode-context-updater/src/test/resources/test.cql deleted file mode 100644 index 2f59dd103..000000000 --- a/dialcode-context-updater/src/test/resources/test.cql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS dialcode_store with replication = {'class':'SimpleStrategy','replication_factor':1}; - -CREATE TABLE IF NOT EXISTS dialcode_store.dial_code (identifier text,dialcode_index double,publisher text,channel text,batchCode text,metadata text,status text,generated_on text,published_on text, primary key(identifier)); - -INSERT INTO dialcode_store.dial_code(identifier, dialcode_index, publisher, batchCode, status, generated_on) VALUES ('0117CH01', 0.0, '0125196274181898243', '0125196274181898243-01', 'Draft', '2020-03-04T14:15:00.000'); diff --git a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/fixture/EventFixture.scala b/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/fixture/EventFixture.scala deleted file mode 100644 index 563b3de4a..000000000 --- a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/fixture/EventFixture.scala +++ /dev/null @@ -1,10 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.fixture - -object EventFixture { - - val DIALCODE_EVENT_WITH_CONTEXT: String = - """{"eid":"BE_JOB_REQUEST","ets":1648720639981,"mid":"LP.1648720639981.d6b1d8c8-7a4a-483a-b83a-b752bede648c","actor":{"id":"DIALcodecontextupdateJob","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":"01269878797503692810","env":"dev"},"object":{"ver":"1.0","id":"0117CH01"},"edata":{"action":"dialcode-context-update","iteration":1,"dialcode":"0117CH01","identifier":"d0_1234","traceId":"2342345345"}}""".stripMargin - - val DIALCODE_EVENT_NULL_CONTEXT: String = - """{"eid":"BE_JOB_REQUEST","ets":1648720639981,"mid":"LP.1648720639981.d6b1d8c8-7a4a-483a-b83a-b752bede648c","actor":{"id":"DIALcodecontextupdateJob","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":"01269878797503692810","env":"dev"},"object":{"ver":"1.0","id":"0117CH01"},"edata":{"action":"dialcode-context-delete","iteration":1,"dialcode":"0117CH01","identifier":"d0_1234","traceId":"2342345345"}}""".stripMargin -} diff --git a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/domain/EventSpec.scala b/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/domain/EventSpec.scala deleted file mode 100644 index 2e0393905..000000000 --- a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/domain/EventSpec.scala +++ /dev/null @@ -1,21 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.spec.domain - -import com.typesafe.config.{Config, ConfigFactory} -import org.scalatest.{FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.dialcodecontextupdater.domain.Event -import org.sunbird.job.dialcodecontextupdater.task.DialcodeContextUpdaterConfig -import org.sunbird.job.util.JSONUtil - -class EventSpec extends FlatSpec with Matchers with MockitoSugar { - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: DialcodeContextUpdaterConfig = new DialcodeContextUpdaterConfig(config) - - "isValid" should "return true for a valid event" in { - val sunbirdEvent = """{"eid":"BE_JOB_REQUEST","ets":1648720639981,"mid":"LP.1648720639981.d6b1d8c8-7a4a-483a-b83a-b752bede648c","actor":{"id":"DIALcodecontextupdateJob","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":"01269878797503692810","env":"dev"},"object":{"ver":"1.0","id":"0117CH01"},"edata":{"action":"dialcode-context-update","iteration":1,"dialcode":"0117CH01","identifier":"d0_1234","traceId":"2342345345"}}""" - val event = new Event(JSONUtil.deserialize[java.util.Map[String, Any]](sunbirdEvent),0,1) - - assert(event.isValid()) - } -} \ No newline at end of file diff --git a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/helper/ComplexJsonCompilerTest.scala b/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/helper/ComplexJsonCompilerTest.scala deleted file mode 100644 index 07e21ec68..000000000 --- a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/helper/ComplexJsonCompilerTest.scala +++ /dev/null @@ -1,42 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.spec.helper - -import org.junit.Test -import org.sunbird.job.util.ComplexJsonCompiler - - -class ComplexJsonCompilerTest { - val contextMapFile: String = "https://raw.githubusercontent.com/project-sunbird/knowledge-platform-jobs/master/dialcode-context-updater/src/main/resources/contextMapping.json" - @Test - @throws[Exception] - def testTBUnit(): Unit = { - val contextType = "textbook_unit" - val compiledContext = ComplexJsonCompiler.createConsolidatedSchema(contextMapFile, contextType) - println(compiledContext) - assert(!compiledContext.contains("$ref")) - } - - @Test - @throws[Exception] - def testTextBook(): Unit = { - val contextType = "digital_textbook" - val compiledContext = ComplexJsonCompiler.createConsolidatedSchema(contextMapFile, contextType) - assert(!compiledContext.contains("$ref")) - } - - @Test - @throws[Exception] - def testCourseUnit(): Unit = { - val contextType = "course_unit" - val compiledContext = ComplexJsonCompiler.createConsolidatedSchema(contextMapFile, contextType) - assert(!compiledContext.contains("$ref")) - } - - @Test - @throws[Exception] - def testCourse(): Unit = { - val contextType = "course" - val compiledContext = ComplexJsonCompiler.createConsolidatedSchema(contextMapFile, contextType) - assert(!compiledContext.contains("$ref")) - } - -} \ No newline at end of file diff --git a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/helper/DialcodeContextUpdaterSpec.scala b/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/helper/DialcodeContextUpdaterSpec.scala deleted file mode 100644 index c3f388f30..000000000 --- a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/helper/DialcodeContextUpdaterSpec.scala +++ /dev/null @@ -1,90 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.spec.helper - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.Metrics -import org.sunbird.job.dialcodecontextupdater.domain.Event -import org.sunbird.job.dialcodecontextupdater.helpers.DialcodeContextUpdater -import org.sunbird.job.dialcodecontextupdater.task.DialcodeContextUpdaterConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.{APIException, ServerException} -import org.sunbird.job.util._ -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - - -class DialcodeContextUpdaterSpec extends BaseTestSpec { - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: DialcodeContextUpdaterConfig = new DialcodeContextUpdaterConfig(config) - val defCache = new DefinitionCache() - var mockHttpUtil: HttpUtil = mock[HttpUtil] - var mockMetrics: Metrics = mock[Metrics](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - flinkCluster.after() - super.afterAll() - } - - - val eventString = """{"eid":"BE_JOB_REQUEST","ets":1654669707334,"mid":"LP.1654669707334.589dbe02-14b2-4d1f-b4ce-cdda94a1fd73","actor":{"id":"DIAL code context update Job","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"sunbird","env":"dev"},"object":{"ver":"1.0","id":"B8V3F7"},"edata":{"action":"dialcode-context-update","iteration":1,"dialcode":"B8V3F7","identifier": "do_113555052123987968128"},"identifier": "do_113555052123987968128"}""" - val searchResponse = """{"id": "api.content.search","ver": "1.0","ts": "2022-05-17T07:23:08.145Z","params": { "resmsgid": "3331f610-d5b2-11ec-aecf-bf611e734ce0", "msgid": "332ee8d0-d5b2-11ec-bde5-a796d2268843", "status": "successful", "err": null, "errmsg": null},"responseCode": "OK","result": { "collections": [ { "identifier": "do_31307361357388185614238", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "English", "Hindi" ], "createdOn": "2020-07-28T01:34:54.470+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/collection/do_31307361357388185614238/artifact/ahhn1cc.thumb.thumb.thumb.jpg", "size": 3419633, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "contentType": "TextBook" }, { "identifier": "do_31310352462397440011697", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "English", "Hindi" ], "createdOn": "2020-09-08T07:49:05.702+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "size": 4652969, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Chandigarh)" }, { "identifier": "do_313263169135534080136703", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2021-04-21T21:06:41.119+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691606, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Himachal Pradesh)" }, { "identifier": "do_313263167296634880136425", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2021-04-21T21:02:56.644+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691616, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Himachal Pradesh)" }, { "identifier": "do_31312343322655948811154", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-10-06T10:53:15.042+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691475, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Arunachal Pradesh)" }, { "identifier": "do_31317212017094656011949", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-12-14T05:47:05.558+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691674, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "UT (Andaman and Nicobar Islands)" }, { "identifier": "do_31310347499136614411402", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi", "English" ], "createdOn": "2020-09-08T06:08:07.038+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691550, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "CBSE" }, { "identifier": "do_31310351153087283211530", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-09-08T07:22:27.424+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "size": 4652868, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Uttarakhand)" }, { "identifier": "do_31310351900026470411729", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-09-08T07:37:39.214+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "size": 4652932, "primaryCategory": "Digital Textbook", "name": "Delhi_ रिमझिम (by NCERT)", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Delhi)" } ], "count": 1, "content": [ { "identifier": "do_31307361357558579213961", "primaryCategory": "Textbook Unit", "name": "1-झूला", "mimeType": "application/vnd.ekstep.content-collection", "createdOn": "2020-07-28T01:34:54.676+0000", "objectType": "Content" } ], "facets": [ { "values": [ { "name": "0125196274181898243", "count": 1 } ], "name": "channel" } ], "collectionsCount": 9}}""" - val emptySearchResponse = """{ "id": "api.content.search", "ver": "1.0", "ts": "2022-05-18T05:36:30.364Z", "params": {"resmsgid": "783bf4d0-d66c-11ec-a3e1-bf0e5a4247e4","msgid": "783a9540-d66c-11ec-b434-c9d6df1d95b1","status": "successful","err": null,"errmsg": null }, "responseCode": "OK", "result": {"count": 0 }}""" - val dialcodeNullContextReadResponse = """{"id": "api.dialcode.read","ver": "1.0","ts": "2022-06-02T06:37:02.149Z","params": {"resmsgid": "69247750-e23e-11ec-a3e1-bf0e5a4247e4","msgid": "6920cdd0-e23e-11ec-b434-c9d6df1d95b1","status": "successful","err": null,"errmsg": null},"responseCode": "OK","result": {"dialcode": {"identifier": "0117CH01","contextInfo": null,"generatedOn": "2020-03-04T14:15:00.000","batchCode": "0125196274181898243-01","channel": "0125196274181898243","publisher": "0125196274181898243","publishedOn": null,"status": "Draft"}}}""" - - - - "getContextJson" should "return context Information" in { - println("DialcodeContextUpdater:: updateContext:: config.contextMapFilePath: " + jobConfig.contextMapFilePath) - val contextMap = new TestDialcodeContextUpdater().getContextMapFields("course_unit", jobConfig.contextMapFilePath) - val contextFields = new TestDialcodeContextUpdater().fetchFieldsFromMap(contextMap).distinct.filter(rec => rec.forall(_.isLetterOrDigit)) - println("DialcodeContextUpdaterSpec:: getContextData:: contextFields: " + contextFields) - - assert(contextFields.nonEmpty) - } - -// "getContextJson" should "should throw API exception when there is no context response" in { -// when(mockHttpUtil.get(anyString(), any())).thenReturn(HTTPResponse(200, "")) -// -// assertThrows[APIException] { -// new TestDialcodeContextUpdater().getContextJson(mockHttpUtil, jobConfig).keySet.toList -// } -// } - - "updateContext" should "should throw server exception when there is no search response" in { - when(mockHttpUtil.post(anyString(), anyString(), any())).thenReturn(HTTPResponse(200, emptySearchResponse)) - - val event = new Event(JSONUtil.deserialize[java.util.Map[String, Any]](eventString),0,1) - - assertThrows[ServerException] { - new TestDialcodeContextUpdater().updateContext(jobConfig,event, mockHttpUtil) - } - } - -} - -class TestDialcodeContextUpdater extends DialcodeContextUpdater {} \ No newline at end of file diff --git a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/task/DialcodeContextUpdaterStreamTaskSpec.scala b/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/task/DialcodeContextUpdaterStreamTaskSpec.scala deleted file mode 100644 index 1dc119f80..000000000 --- a/dialcode-context-updater/src/test/scala/org/sunbird/job/dialcodecontextupdater/spec/task/DialcodeContextUpdaterStreamTaskSpec.scala +++ /dev/null @@ -1,123 +0,0 @@ -package org.sunbird.job.dialcodecontextupdater.spec.task - -import com.google.gson.Gson -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, anyString, contains} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.sunbird.job.Metrics -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.dialcodecontextupdater.domain.Event -import org.sunbird.job.dialcodecontextupdater.fixture.EventFixture -import org.sunbird.job.dialcodecontextupdater.functions.DialcodeContextUpdaterFunction -import org.sunbird.job.dialcodecontextupdater.task.{DialcodeContextUpdaterConfig, DialcodeContextUpdaterStreamTask} -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil, Neo4JUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.util - -class DialcodeContextUpdaterStreamTaskSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - implicit val jobConfig: DialcodeContextUpdaterConfig = new DialcodeContextUpdaterConfig(config) - - implicit val mockHttpUtil: HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - var mockMetrics: Metrics = mock[Metrics](Mockito.withSettings().serializable()) - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - flinkCluster.after() - super.afterAll() - } - - - def initialize(): Unit = { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new DialcodeContextUpdaterEventSource) - } - - ignore should " update the dial context " in { - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(new util.HashMap[String, AnyRef]) - initialize() - new DialcodeContextUpdaterStreamTask(jobConfig, mockKafkaUtil, mockHttpUtil).process() - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dbHitEventCount}").getValue() should be(1) - } - - val searchResponse = """{"id": "api.content.search","ver": "1.0","ts": "2022-05-17T07:23:08.145Z","params": { "resmsgid": "3331f610-d5b2-11ec-aecf-bf611e734ce0", "msgid": "332ee8d0-d5b2-11ec-bde5-a796d2268843", "status": "successful", "err": null, "errmsg": null},"responseCode": "OK","result": { "collections": [ { "identifier": "do_31307361357388185614238", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "English", "Hindi" ], "createdOn": "2020-07-28T01:34:54.470+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/collection/do_31307361357388185614238/artifact/ahhn1cc.thumb.thumb.thumb.jpg", "size": 3419633, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "contentType": "TextBook" }, { "identifier": "do_31310352462397440011697", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "English", "Hindi" ], "createdOn": "2020-09-08T07:49:05.702+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "size": 4652969, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Chandigarh)" }, { "identifier": "do_313263169135534080136703", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2021-04-21T21:06:41.119+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691606, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Himachal Pradesh)" }, { "identifier": "do_313263167296634880136425", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2021-04-21T21:02:56.644+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691616, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Himachal Pradesh)" }, { "identifier": "do_31312343322655948811154", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-10-06T10:53:15.042+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691475, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Arunachal Pradesh)" }, { "identifier": "do_31317212017094656011949", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-12-14T05:47:05.558+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691674, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "UT (Andaman and Nicobar Islands)" }, { "identifier": "do_31310347499136614411402", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi", "English" ], "createdOn": "2020-09-08T06:08:07.038+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "appIcon": "https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31307361357388185614238/artifact/ahhn1cc.thumb.jpg", "size": 4691550, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "CBSE" }, { "identifier": "do_31310351153087283211530", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-09-08T07:22:27.424+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "size": 4652868, "primaryCategory": "Digital Textbook", "name": "(NEW) रिमझिम", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Uttarakhand)" }, { "identifier": "do_31310351900026470411729", "subject": [ "Hindi" ], "childNodes": [ "do_31307361357558579213961" ], "origin": "do_31307361357388185614238", "mimeType": "application/vnd.ekstep.content-collection", "medium": [ "Hindi" ], "createdOn": "2020-09-08T07:37:39.214+0000", "objectType": "Collection", "gradeLevel": [ "Class 1" ], "size": 4652932, "primaryCategory": "Digital Textbook", "name": "Delhi_ रिमझिम (by NCERT)", "originData": "{\"name\":\"(NEW) रिमझिम\",\"copyType\":\"shallow\",\"license\":\"CC BY-SA 4.0\",\"organisation\":[\"NCERT\"],\"pkgVersion\":44.0}", "contentType": "TextBook", "board": "State (Delhi)" } ], "count": 1, "content": [ { "identifier": "do_31307361357558579213961", "primaryCategory": "Textbook Unit", "name": "1-झूला", "mimeType": "application/vnd.ekstep.content-collection", "createdOn": "2020-07-28T01:34:54.676+0000", "objectType": "Content" } ], "facets": [ { "values": [ { "name": "0125196274181898243", "count": 1 } ], "name": "channel" } ], "collectionsCount": 9}}""" - val nullEventString = """{"eid":"BE_JOB_REQUEST","ets":1654669707334,"mid":"LP.1654669707334.589dbe02-14b2-4d1f-b4ce-cdda94a1fd73","actor":{"id":"DIAL code context update Job","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"sunbird","env":"dev"},"object":{"ver":"1.0","id":"B8V3F7"},"edata":{"action":"dialcode-context-delete","iteration":1,"dialcode":"B8V3F7","identifier": "do_113555052123987968128"},"identifier": "do_113555052123987968128"}""" - val dialcodeContextUpdateResponse = """{"id": "api.dialcode.update","ver": "1.0","ts": "2022-06-02T06:56:32.767Z","params": {"resmsgid": "22e298f0-e241-11ec-87a6-19c1a5d877c9","msgid": "22b79150-e241-11ec-a314-3942d93c4e08","status": "successful","err": null,"errmsg": null},"responseCode": "OK","result": {"identifier": "X5I6I4"}}""" - val dialcodeNullContextReadResponse = """{"id": "api.dialcode.read","ver": "1.0","ts": "2022-06-02T06:37:02.149Z","params": {"resmsgid": "69247750-e23e-11ec-a3e1-bf0e5a4247e4","msgid": "6920cdd0-e23e-11ec-b434-c9d6df1d95b1","status": "successful","err": null,"errmsg": null},"responseCode": "OK","result": {"dialcode": {"identifier": "0117CH01","contextInfo": null,"generatedOn": "2020-03-04T14:15:00.000","batchCode": "0125196274181898243-01","channel": "0125196274181898243","publisher": "0125196274181898243","publishedOn": null,"status": "Draft"}}}""" - val dialcodeNotNullContextReadResponse = """{"id": "api.dialcode.read","ver": "1.0","ts": "2022-06-02T06:58:30.080Z","params": {"resmsgid": "68cf2400-e241-11ec-87a6-19c1a5d877c9","msgid": "68cdc470-e241-11ec-a314-3942d93c4e08","status": "successful","err": null,"errmsg": null},"responseCode": "OK","result": {"dialcode": {"identifier": "X5I6I4","contextInfo": [{"gradeLevel": ["Class 2"],"@type": "https://staging.sunbirded.org/ns/collection","subject": ["Mathematics"],"medium": ["English"],"@context": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-dial-staging/schemas/local/collection/context.json","board": "CBSE"}],"generatedOn": "2020-08-26T08:24:48.338+0000","@type": "https://staging.sunbirded.org/ns/DIAL","batchCode": "do_2130943402930012161605","channel": "01272777697873100812","publisher": null,"publishedOn": null,"@id": "https://staging.sunbirded.org/dial/X5I6I4","@context": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-dial-staging/schemas/local/dialcode/context.json","status": "Draft"}}}""" - - "updateContext" should "should update the dialcode metadata with context information" in { - when(mockHttpUtil.get(contains(jobConfig.dialcodeContextReadPath), any())).thenReturn(HTTPResponse(200, dialcodeNotNullContextReadResponse)) - when(mockHttpUtil.post(anyString(), anyString(), any())).thenReturn(HTTPResponse(200, searchResponse)) - when(mockHttpUtil.patch(anyString(), anyString(), any())).thenReturn(HTTPResponse(200, dialcodeContextUpdateResponse)) - - val event = new Event(JSONUtil.deserialize[java.util.Map[String, Any]](EventFixture.DIALCODE_EVENT_WITH_CONTEXT),0,1) - - val dialcodeContextInfo: Map[String, AnyRef] = new DialcodeContextUpdaterFunction(jobConfig, mockHttpUtil).updateContext(jobConfig,event, mockHttpUtil) - - assert(dialcodeContextInfo.contains("contextInfo") && dialcodeContextInfo("contextInfo") != null) - } - - "updateContext" should "should update the dialcode metadata with null" in { - when(mockHttpUtil.get(contains(jobConfig.dialcodeContextReadPath), any())).thenReturn(HTTPResponse(200, dialcodeNullContextReadResponse)) - when(mockHttpUtil.post(anyString(), anyString(), any())).thenReturn(HTTPResponse(200, searchResponse)) - when(mockHttpUtil.patch(anyString(), anyString(), any())).thenReturn(HTTPResponse(200, dialcodeContextUpdateResponse)) - - val event = new Event(JSONUtil.deserialize[java.util.Map[String, Any]](EventFixture.DIALCODE_EVENT_NULL_CONTEXT),0,1) - - val dialcodeContextInfo: Map[String, AnyRef] = new DialcodeContextUpdaterFunction(jobConfig, mockHttpUtil).updateContext(jobConfig,event, mockHttpUtil) - - assert(dialcodeContextInfo.contains("contextInfo") && dialcodeContextInfo("contextInfo") == null) - } - -} - - -private class DialcodeContextUpdaterEventSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]): Unit = { - ctx.collect(jsonToEvent(EventFixture.DIALCODE_EVENT_WITH_CONTEXT)) - } - - override def cancel(): Unit = {} - - def jsonToEvent(json: String): Event = { - val gson = new Gson() - val data = gson.fromJson(json, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]] - new Event(data, 0, 10) - } -} - diff --git a/live-video-stream-generator/README.md b/live-video-stream-generator/README.md deleted file mode 100644 index eb8941ff7..000000000 --- a/live-video-stream-generator/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Video Stream Generator - -Live Video Stream Generator job is to generate the stream url for uploaded video (mp4/webm) files - this is a duplicate job of Video Stream Generator with difference in Input topic. Job was created to support CSP migration activity, - -## Getting Started - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. -### Prerequisites - -1. Download flink-1.13.6-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.13.6/flink-1.13.6-bin-scala_2.12.tgz). -2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. -3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. -4. Docker installed. -5. A running yarn cluster or a kubernetes cluster. - -### Build - -mvn clean install - -## Deployment - -### Yarn - -Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. - -``` -./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /video-stream-generator/target/video-stream-generator-0.0.1.jar -``` - -### Kubernetes - -``` -# Create a single node cluster -k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 -# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. -export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" - -# Only for Mac OSX -# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl -rm /usr/local/bin/kubectl -brew link --overwrite kubernetes-cli - -# Create a configmap using the flink-configuration-configmap.yaml -kubectl create -f knowledge-platform-job/kubernetes/flink-configuration-configmap.yaml - -# Create pods for jobmanager-service, job-manager and task-manager using the yaml files -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-service.yaml -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-deployment.yaml -kubectl create -f knowledge-platform-job/kubernetes/taskmanager-deployment.yaml - -# Create a port-forwarding for accessing the job-manager UI on localhost:8081 -kubectl port-forward deployment/flink-jobmanager 8081:8081 - -# Submit the job to the Kubernetes single node cluster flink-cluster -./bin/flink run -m localhost:8081 /video-stream-generator/target/video-stream-generator-0.0.1.jar - -# Commands to delete the pods created in the cluster -kubectl delete deployment/flink-jobmanager -kubectl delete deployment/flink-taskmanager -kubectl delete service/flink-jobmanager -kubectl delete configmaps/flink-config - -# Command to stop the single-node cluster -k3d stop --name="flink-cluster" -``` diff --git a/live-video-stream-generator/pom.xml b/live-video-stream-generator/pom.xml deleted file mode 100644 index a79b8883d..000000000 --- a/live-video-stream-generator/pom.xml +++ /dev/null @@ -1,214 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - live-video-stream-generator - 1.0.0 - jar - live-video-stream-generator - - Video stream URL generator Flink Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - live-video-stream-generator-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/live-video-stream-generator/src/main/resources/live-video-stream-generator.conf b/live-video-stream-generator/src/main/resources/live-video-stream-generator.conf deleted file mode 100644 index 5346dcaf9..000000000 --- a/live-video-stream-generator/src/main/resources/live-video-stream-generator.conf +++ /dev/null @@ -1,94 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.live.video.stream.request" - groupId = "sunbirddev-live-video-stream-generator-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - timer.duration = 60 - max.retries = 10 -} - -lms-cassandra { - keyspace = "sunbirddev_platform_db" - table = "job_request" -} - -service { - content { - basePath = "http://11.2.6.6/content" - } -} - -# Azure Media Service Config -azure { - location = "centralindia" - tenant = "tenant" - subscription_id = "subscription id " - - login { - endpoint="https://login.microsoftonline.com" - } - - api { - endpoint="https://management.azure.com" - version = "2018-07-01" - } - - account_name = "account name" - resource_group_name = "group name" - - transform { - default = "media_transform_default" - hls = "media_transform_hls" - } - - stream { - base_url = "https://sunbirdspikemedia-inct.streaming.media.azure.net" - endpoint_name = "default" - protocol = "Hls" - policy_name = "Predefined_ClearStreamingOnly" - } - - token { - client_key = "client key" - client_secret = "client secret" - } -} - -azure_tenant="tenant" -azure_subscription_id="subscription id" -azure_account_name="account name" -azure_resource_group_name="group name" -azure_token_client_key="client key" -azure_token_client_secret="client secret" - -# CSP Name. e.g: aws or azure -media_service_type="aws" - -#AWS Elemental Media Convert Config -aws { - region="ap-south-1" - content_bucket_name="awsmedia-spike" - token { - access_key="access key" - access_secret="access secret" - } - api { - endpoint="API Endpoint for media convert" - version="2017-08-29" - } - service { - name="mediaconvert" - queue="Media Convert Queue Id" - role="Media Convert Role Name" - } - stream { - protocol="Hls" - } -} - -media_service_job_success_status=["FINISHED", "COMPLETE"] diff --git a/live-video-stream-generator/src/main/resources/log4j.properties b/live-video-stream-generator/src/main/resources/log4j.properties deleted file mode 100644 index 04cc82649..000000000 --- a/live-video-stream-generator/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=live-video-stream-generator.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/domain/Event.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/domain/Event.scala deleted file mode 100644 index f0d8ede3e..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/domain/Event.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.sunbird.job.livevideostream.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest - -import java.util - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val jobName = "LiveVideoStreamGenerator" - - def action: String = readOrDefault[String]("edata.action", "") - - def mimeType: String = readOrDefault[String]("edata.mimeType", "") - - def channel: String = readOrDefault[String]("context.channel", "") - - def eid: String = readOrDefault[String]("eid", "") - - def artifactUrl: String = readOrDefault[String]("edata.artifactUrl", "") - - def identifier: String = readOrDefault[String]("edata.identifier", "") - - def eData: Map[String, AnyRef] = readOrDefault("edata", new util.HashMap[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - - def isValid: Boolean = { - StringUtils.isNotBlank(artifactUrl) && - StringUtils.isNotBlank(mimeType) && - ( - StringUtils.equalsIgnoreCase(mimeType, "video/mp4") || - StringUtils.equalsIgnoreCase(mimeType, "video/webm") - ) && - StringUtils.isNotBlank(identifier) && - StringUtils.isNotBlank(channel) - } - -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/exception/MediaServiceException.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/exception/MediaServiceException.scala deleted file mode 100644 index 906ec883b..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/exception/MediaServiceException.scala +++ /dev/null @@ -1,3 +0,0 @@ -package org.sunbird.job.livevideostream.exception - -class MediaServiceException(var errorCode: String = null, msg: String, ex: Exception = null) extends Exception(msg, ex) diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/functions/LiveVideoStreamGenerator.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/functions/LiveVideoStreamGenerator.scala deleted file mode 100644 index 247b9b267..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/functions/LiveVideoStreamGenerator.scala +++ /dev/null @@ -1,91 +0,0 @@ -package org.sunbird.job.livevideostream.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.livevideostream.domain.Event -import org.sunbird.job.livevideostream.service.LiveVideoStreamService -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} - -import java.util - -class LiveVideoStreamGenerator(config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessKeyedFunction[String, Event, Event](config) { - - implicit lazy val videoStreamConfig: LiveVideoStreamGeneratorConfig = config - private[this] lazy val logger = LoggerFactory.getLogger(classOf[LiveVideoStreamGenerator]) - private var videoStreamService: LiveVideoStreamService = _ - private lazy val timerDurationInMS: Long = config.timerDuration * 1000 - private var nextTimerTimestamp = 0L - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.skippedEventCount, config.successEventCount, config.failedEventCount, config.retryEventCount) - } - - override def open(parameters: Configuration, metrics: Metrics): Unit = { - videoStreamService = new LiveVideoStreamService()(config, httpUtil) - val processing = videoStreamService.readFromDB(Map("status" -> "PROCESSING")) ++ videoStreamService.readFromDB(Map("status" -> "FAILED", "iteration" -> Map("type" -> "lte", "value" -> 10))) - if (processing.nonEmpty) { - logger.info("Requests in queue to validate update the status: " + processing.size) - videoStreamService.processJobRequest(metrics) - } else logger.info("open() ==> There are no video streaming requests in queue to validate update the status.") - } - - override def close(): Unit = { - videoStreamService.closeConnection() - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: KeyedProcessFunction[String, Event, Event]#Context, - metrics: Metrics): Unit = { - try { - metrics.incCounter(config.totalEventsCount) - if (event.isValid) { - val eData = event.eData ++ Map("channel" -> event.channel) - videoStreamService.submitJobRequest(eData) - logger.info("Streaming job submitted for " + event.identifier + " with url: " + event.artifactUrl) - registerTimer(context) - } else metrics.incCounter(config.skippedEventCount) - } catch { - case ex: Exception => - metrics.incCounter(config.failedEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, Event, Event]#OnTimerContext, metrics: Metrics): Unit = { - unregisterTimer() - val processing = videoStreamService.readFromDB(Map("status" -> "PROCESSING")) ++ videoStreamService.readFromDB(Map("status" -> "FAILED", "iteration" -> Map("type" -> "lte", "value" -> 10))) - if (processing.nonEmpty) { - logger.info("Requests in queue to validate update the status: " + processing.size) - videoStreamService.processJobRequest(metrics) - registerTimer(ctx) - } else { - logger.info("There are no video streaming requests in queue to validate update the status.") - } - } - - private def registerTimer(context: KeyedProcessFunction[String, Event, Event]#Context): Unit = { - if (nextTimerTimestamp == 0L) { - val nextTrigger = context.timestamp() + timerDurationInMS - context.timerService().registerProcessingTimeTimer(nextTrigger) - nextTimerTimestamp = nextTrigger - logger.info("Timer registered to execute at " + nextTimerTimestamp) - } else { - logger.info("Timer already exists at: " + nextTimerTimestamp) - } - } - - private def unregisterTimer(): Unit = { - nextTimerTimestamp = 0L - } - -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsRequestBody.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsRequestBody.scala deleted file mode 100644 index c0f43059c..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsRequestBody.scala +++ /dev/null @@ -1,6 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -object AwsRequestBody { - - val submit_hls_job = "{\"Queue\": \"queueId\",\"role\":\"mediaRole\",\"Settings\": {\"OutputGroups\": [{\"CustomName\": \"hls\",\"Name\": \"Apple HLS\",\"Outputs\": [{\"ContainerSettings\": {\"Container\": \"M3U8\",\"M3u8Settings\": {\"AudioFramesPerPes\": 4,\"PcrControl\": \"PCR_EVERY_PES_PACKET\",\"PmtPid\": 480,\"PrivateMetadataPid\": 503,\"ProgramNumber\": 1,\"PatInterval\": 0,\"PmtInterval\": 0,\"Scte35Source\": \"NONE\",\"NielsenId3\": \"NONE\",\"TimedMetadata\": \"NONE\",\"VideoPid\": 481,\"AudioPids\": [482,483,484,485,486,487,488,489,490,491,492]}},\"VideoDescription\": {\"Width\": 256,\"ScalingBehavior\": \"DEFAULT\",\"Height\": 144,\"TimecodeInsertion\": \"DISABLED\",\"AntiAlias\": \"ENABLED\",\"Sharpness\": 50,\"CodecSettings\": {\"Codec\": \"H_264\",\"H264Settings\": {\"InterlaceMode\": \"PROGRESSIVE\",\"NumberReferenceFrames\": 3,\"Syntax\": \"DEFAULT\",\"Softness\": 0,\"GopClosedCadence\": 1,\"GopSize\": 90,\"Slices\": 1,\"GopBReference\": \"DISABLED\",\"SlowPal\": \"DISABLED\",\"SpatialAdaptiveQuantization\": \"ENABLED\",\"TemporalAdaptiveQuantization\": \"ENABLED\",\"FlickerAdaptiveQuantization\": \"DISABLED\",\"EntropyEncoding\": \"CABAC\",\"Bitrate\": 109000,\"FramerateControl\": \"INITIALIZE_FROM_SOURCE\",\"RateControlMode\": \"CBR\",\"CodecProfile\": \"MAIN\",\"Telecine\": \"NONE\",\"MinIInterval\": 0,\"AdaptiveQuantization\": \"HIGH\",\"CodecLevel\": \"AUTO\",\"FieldEncoding\": \"PAFF\",\"SceneChangeDetect\": \"ENABLED\",\"QualityTuningLevel\": \"SINGLE_PASS\",\"FramerateConversionAlgorithm\": \"DUPLICATE_DROP\",\"UnregisteredSeiTimecode\": \"DISABLED\",\"GopSizeUnits\": \"FRAMES\",\"ParControl\": \"INITIALIZE_FROM_SOURCE\",\"NumberBFramesBetweenReferenceFrames\": 2,\"RepeatPps\": \"DISABLED\",\"DynamicSubGop\": \"STATIC\"}},\"AfdSignaling\": \"NONE\",\"DropFrameTimecode\": \"ENABLED\",\"RespondToAfd\": \"NONE\",\"ColorMetadata\": \"INSERT\"},\"AudioDescriptions\": [{\"AudioTypeControl\": \"FOLLOW_INPUT\",\"CodecSettings\": {\"Codec\": \"AAC\",\"AacSettings\": {\"AudioDescriptionBroadcasterMix\": \"NORMAL\",\"Bitrate\": 96000,\"RateControlMode\": \"CBR\",\"CodecProfile\": \"LC\",\"CodingMode\": \"CODING_MODE_2_0\",\"RawFormat\": \"NONE\",\"SampleRate\": 48000,\"Specification\": \"MPEG4\"}},\"LanguageCodeControl\": \"FOLLOW_INPUT\"}],\"OutputSettings\": {\"HlsSettings\": {\"AudioGroupId\": \"program_audio\",\"SegmentModifier\": \"$dt$\",\"IFrameOnlyManifest\": \"EXCLUDE\"}},\"NameModifier\": \"_144\"},{\"ContainerSettings\": {\"Container\": \"M3U8\",\"M3u8Settings\": {\"AudioFramesPerPes\": 4,\"PcrControl\": \"PCR_EVERY_PES_PACKET\",\"PmtPid\": 480,\"PrivateMetadataPid\": 503,\"ProgramNumber\": 1,\"PatInterval\": 0,\"PmtInterval\": 0,\"Scte35Source\": \"NONE\",\"Scte35Pid\": 500,\"NielsenId3\": \"NONE\",\"TimedMetadata\": \"NONE\",\"TimedMetadataPid\": 502,\"VideoPid\": 481,\"AudioPids\": [482,483,484,485,486,487,488,489,490,491,492]}},\"VideoDescription\": {\"Width\": 640,\"ScalingBehavior\": \"DEFAULT\",\"Height\": 360,\"TimecodeInsertion\": \"DISABLED\",\"AntiAlias\": \"ENABLED\",\"Sharpness\": 50,\"CodecSettings\": {\"Codec\": \"H_264\",\"H264Settings\": {\"InterlaceMode\": \"PROGRESSIVE\",\"NumberReferenceFrames\": 3,\"Syntax\": \"DEFAULT\",\"Softness\": 0,\"GopClosedCadence\": 1,\"GopSize\": 90,\"Slices\": 1,\"GopBReference\": \"DISABLED\",\"SlowPal\": \"DISABLED\",\"SpatialAdaptiveQuantization\": \"ENABLED\",\"TemporalAdaptiveQuantization\": \"ENABLED\",\"FlickerAdaptiveQuantization\": \"DISABLED\",\"EntropyEncoding\": \"CABAC\",\"Bitrate\": 525000,\"FramerateControl\": \"INITIALIZE_FROM_SOURCE\",\"RateControlMode\": \"CBR\",\"CodecProfile\": \"MAIN\",\"Telecine\": \"NONE\",\"MinIInterval\": 0,\"AdaptiveQuantization\": \"HIGH\",\"CodecLevel\": \"AUTO\",\"FieldEncoding\": \"PAFF\",\"SceneChangeDetect\": \"ENABLED\",\"QualityTuningLevel\": \"SINGLE_PASS\",\"FramerateConversionAlgorithm\": \"DUPLICATE_DROP\",\"UnregisteredSeiTimecode\": \"DISABLED\",\"GopSizeUnits\": \"FRAMES\",\"ParControl\": \"INITIALIZE_FROM_SOURCE\",\"NumberBFramesBetweenReferenceFrames\": 2,\"RepeatPps\": \"DISABLED\",\"DynamicSubGop\": \"STATIC\"}},\"AfdSignaling\": \"NONE\",\"DropFrameTimecode\": \"ENABLED\",\"RespondToAfd\": \"NONE\",\"ColorMetadata\": \"INSERT\"},\"AudioDescriptions\": [{\"AudioTypeControl\": \"FOLLOW_INPUT\",\"CodecSettings\": {\"Codec\": \"AAC\",\"AacSettings\": {\"AudioDescriptionBroadcasterMix\": \"NORMAL\",\"Bitrate\": 96000,\"RateControlMode\": \"CBR\",\"CodecProfile\": \"LC\",\"CodingMode\": \"CODING_MODE_2_0\",\"RawFormat\": \"NONE\",\"SampleRate\": 48000,\"Specification\": \"MPEG4\"}},\"LanguageCodeControl\": \"FOLLOW_INPUT\"}],\"OutputSettings\": {\"HlsSettings\": {\"AudioGroupId\": \"program_audio\",\"SegmentModifier\": \"$dt$\",\"IFrameOnlyManifest\": \"EXCLUDE\"}},\"NameModifier\": \"_360\"},{\"ContainerSettings\": {\"Container\": \"M3U8\",\"M3u8Settings\": {\"AudioFramesPerPes\": 4,\"PcrControl\": \"PCR_EVERY_PES_PACKET\",\"PmtPid\": 480,\"PrivateMetadataPid\": 503,\"ProgramNumber\": 1,\"PatInterval\": 0,\"PmtInterval\": 0,\"Scte35Source\": \"NONE\",\"Scte35Pid\": 500,\"NielsenId3\": \"NONE\",\"TimedMetadata\": \"NONE\",\"TimedMetadataPid\": 502,\"VideoPid\": 481,\"AudioPids\": [482,483,484,485,486,487,488,489,490,491,492]}},\"VideoDescription\": {\"Width\": 1280,\"ScalingBehavior\": \"DEFAULT\",\"Height\": 720,\"TimecodeInsertion\": \"DISABLED\",\"AntiAlias\": \"ENABLED\",\"Sharpness\": 50,\"CodecSettings\": {\"Codec\": \"H_264\",\"H264Settings\": {\"InterlaceMode\": \"PROGRESSIVE\",\"NumberReferenceFrames\": 3,\"Syntax\": \"DEFAULT\",\"Softness\": 0,\"GopClosedCadence\": 1,\"GopSize\": 90,\"Slices\": 1,\"GopBReference\": \"DISABLED\",\"SlowPal\": \"DISABLED\",\"SpatialAdaptiveQuantization\": \"ENABLED\",\"TemporalAdaptiveQuantization\": \"ENABLED\",\"FlickerAdaptiveQuantization\": \"DISABLED\",\"EntropyEncoding\": \"CABAC\",\"Bitrate\": 1378000,\"FramerateControl\": \"INITIALIZE_FROM_SOURCE\",\"RateControlMode\": \"CBR\",\"CodecProfile\": \"MAIN\",\"Telecine\": \"NONE\",\"MinIInterval\": 0,\"AdaptiveQuantization\": \"HIGH\",\"CodecLevel\": \"AUTO\",\"FieldEncoding\": \"PAFF\",\"SceneChangeDetect\": \"ENABLED\",\"QualityTuningLevel\": \"SINGLE_PASS\",\"FramerateConversionAlgorithm\": \"DUPLICATE_DROP\",\"UnregisteredSeiTimecode\": \"DISABLED\",\"GopSizeUnits\": \"FRAMES\",\"ParControl\": \"INITIALIZE_FROM_SOURCE\",\"NumberBFramesBetweenReferenceFrames\": 2,\"RepeatPps\": \"DISABLED\",\"DynamicSubGop\": \"STATIC\"}},\"AfdSignaling\": \"NONE\",\"DropFrameTimecode\": \"ENABLED\",\"RespondToAfd\": \"NONE\",\"ColorMetadata\": \"INSERT\"},\"AudioDescriptions\": [{\"AudioTypeControl\": \"FOLLOW_INPUT\",\"CodecSettings\": {\"Codec\": \"AAC\",\"AacSettings\": {\"AudioDescriptionBroadcasterMix\": \"NORMAL\",\"Bitrate\": 96000,\"RateControlMode\": \"CBR\",\"CodecProfile\": \"LC\",\"CodingMode\": \"CODING_MODE_2_0\",\"RawFormat\": \"NONE\",\"SampleRate\": 48000,\"Specification\": \"MPEG4\"}},\"LanguageCodeControl\": \"FOLLOW_INPUT\"}],\"OutputSettings\": {\"HlsSettings\": {\"AudioGroupId\": \"program_audio\",\"IFrameOnlyManifest\": \"EXCLUDE\"}},\"NameModifier\": \"_720\"}],\"OutputGroupSettings\": {\"Type\": \"HLS_GROUP_SETTINGS\",\"HlsGroupSettings\": {\"ManifestDurationFormat\": \"INTEGER\",\"SegmentLength\": 10,\"TimedMetadataId3Period\": 10,\"CaptionLanguageSetting\": \"OMIT\",\"TimedMetadataId3Frame\": \"PRIV\",\"CodecSpecification\": \"RFC_4281\",\"OutputSelection\": \"MANIFESTS_AND_SEGMENTS\",\"ProgramDateTimePeriod\": 600,\"MinSegmentLength\": 0,\"MinFinalSegmentLength\": 0,\"DirectoryStructure\": \"SINGLE_DIRECTORY\",\"ProgramDateTime\": \"EXCLUDE\",\"SegmentControl\": \"SEGMENTED_FILES\",\"ManifestCompression\": \"NONE\",\"ClientCache\": \"ENABLED\",\"StreamInfResolution\": \"INCLUDE\",\"Destination\": \"outputLocation\"}}}],\"AdAvailOffset\": 0,\"Inputs\": [{\"AudioSelectors\": {\"Audio Selector 1\": {\"Offset\": 0,\"DefaultSelection\": \"DEFAULT\",\"ProgramSelection\": 1}},\"VideoSelector\": {\"ColorSpace\": \"FOLLOW\"},\"FilterEnable\": \"AUTO\",\"PsiControl\": \"USE_PSI\",\"FilterStrength\": 0,\"DeblockFilter\": \"DISABLED\",\"DenoiseFilter\": \"DISABLED\",\"TimecodeSource\": \"EMBEDDED\",\"FileInput\": \"inputVideoFile\"}]}}" -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsResult.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsResult.scala deleted file mode 100644 index 63854ec26..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsResult.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -import java.text.SimpleDateFormat -import java.util.{Date, TimeZone} - -import org.apache.commons.lang3.StringUtils - -import scala.collection.immutable.HashMap - -object AwsResult extends Result { - - val formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") - formatter.setTimeZone(TimeZone.getTimeZone("UTC")) - - override def getSubmitJobResult(response: MediaResponse): Map[String, AnyRef] = { - val job: Map[String, AnyRef] = response.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]] - val timing: Map[String, AnyRef] = job.getOrElse("timing", Map).asInstanceOf[Map[String, AnyRef]] - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> job.getOrElse("id", "").toString, - "status" -> job.getOrElse("status", "").toString.toUpperCase(), - "submittedOn" -> formatter.format(new Date(timing.getOrElse("submitTime", "").toString.toLong*1000)) - ) - ) - } - - override def getJobResult(response: MediaResponse): Map[String, AnyRef] = { - val job: Map[String, AnyRef] = response.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]] - val timing: Map[String, AnyRef] = job.getOrElse("timing", Map).asInstanceOf[Map[String, AnyRef]] - - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> job.getOrElse("id", "").toString, - "status" -> job.getOrElse("status", "").toString.toUpperCase(), - "submittedOn" -> formatter.format(new Date(timing.getOrElse("submitTime", "").toString.toLong*1000)), - "lastModifiedOn" -> { if(StringUtils.isNotBlank(timing.getOrElse("finishTime", "").toString)){formatter.format(new Date(timing.getOrElse("finishTime", "").toString.toLong*1000))} else {""}}, - "error" -> { - if (StringUtils.equalsIgnoreCase(job.getOrElse("status", "").toString.toUpperCase(), "ERROR")) { - HashMap[String, String]( - "errorCode" -> job.getOrElse("errorCode", "").toString, - "errorMessage" -> job.getOrElse("errorMessage", "").toString - ) - }else{ - null - } - } - ) - ) - } - - override def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] = ??? - - override def getListJobResult(response: MediaResponse): Map[String, AnyRef] = ??? -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsSignUtils.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsSignUtils.scala deleted file mode 100644 index f5628f50c..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AwsSignUtils.scala +++ /dev/null @@ -1,133 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -import java.text.SimpleDateFormat -import java.util.{Date, TimeZone} - -import javax.crypto.Mac -import javax.crypto.spec.SecretKeySpec -import javax.xml.bind.DatatypeConverter -import org.apache.commons.codec.binary.Hex -import org.apache.commons.codec.digest.DigestUtils -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig - -object AwsSignUtils { - - val dateFormat = new SimpleDateFormat("yyyyMMdd") - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")) - - def getSiginingkey()(implicit config: LiveVideoStreamGeneratorConfig): Array[Byte] = { - val date = dateFormat.format(new Date()).getBytes("UTF8") - val kSecret = ("AWS4" + config.getConfig("aws.token.access_secret")).getBytes("UTF8") - val kDate = HmacSHA256(date, kSecret) - val kRegion = HmacSHA256(config.getConfig("aws.region").getBytes("UTF8"), kDate) - val kService = HmacSHA256(config.getConfig("aws.service.name").getBytes("UTF8"), kRegion) - val kSigning = HmacSHA256("aws4_request".getBytes("UTF8"), kService) - kSigning - } - - def getStringToSign(httpMethod: String, url: String, headers: Map[String, String], payload: String)(implicit config: LiveVideoStreamGeneratorConfig): String = { - val canonicalUri = getCanonicalUri(url) - val canonicalQueryString = getCanonicalQueryString(url) - val hashedPayload = getHashedPayload(payload) - val canonicalHeaders = getCanonicalHeaders(headers, hashedPayload) - val signedHeaders = getSignedHeaders(headers.keySet) - - val canonicalRequest = httpMethod + "\n" + canonicalUri + "\n" + canonicalQueryString + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + hashedPayload - - val timeStampISO8601Format = headers.get("x-amz-date").get - val scope = dateFormat.format(new Date()) + "/" + config.getConfig("aws.region") + "/" + config.getConfig("aws.service.name") + "/aws4_request" - - val stringToSign = "AWS4-HMAC-SHA256" + "\n" + timeStampISO8601Format + "\n" + scope + "\n" + sha256Hash(canonicalRequest) - - stringToSign - } - - def generateToken(httpMethod: String, url: String, headers: Map[String, String], payload: String)(implicit config: LiveVideoStreamGeneratorConfig): String = { - val signature = new String(Hex.encodeHex(HmacSHA256(getStringToSign(httpMethod, url, headers, payload).getBytes("UTF-8"), getSiginingkey()))) - - "AWS4-HMAC-SHA256 Credential=" + config.getConfig("aws.token.access_key") + "/" + dateFormat.format(new Date()) + "/" + config.getConfig("aws.region") + "/" + config.getConfig("aws.service.name") + "/aws4_request,SignedHeaders=" + getSignedHeaders(headers.keySet) + ",Signature=" + signature - } - - - @throws[Exception] - def HmacSHA256(data: Array[Byte], key: Array[Byte]) = { - val algorithm: String = "HMacSha256" - val mac: Mac = Mac.getInstance(algorithm) - mac.init(new SecretKeySpec(key, algorithm)) - mac.doFinal(data) - } - - def uriEncode(input: Array[Char], encodeSlash: Boolean): String = { - val result: StringBuilder = new StringBuilder() - var i: Int = 0 - for (i <- 0 to input.length()) { - val ch = input.charAt(i) - if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' || ch == '~' || ch == '.') { - result.append(ch) - } else if (ch == '/') { - if (encodeSlash) result.append("%2F") else result.append(ch) - } else { - result.append(Integer.toHexString((ch.asInstanceOf[Integer]))) - } - } - result.toString() - } - - def sha256Hash(input: String): String = { - DigestUtils.sha256Hex(input) - } - - def getCanonicalUri(url: String)(implicit config: LiveVideoStreamGeneratorConfig): String = { - val version = config.getConfig("aws.api.version") - var uri = url.split(version)(1) - if (StringUtils.isBlank(uri)) "" else { - uri = "/" + version + uri; uri - } - } - - def getCanonicalQueryString(url: String): String = { - var result: String = "" - if (url.split("\\?").length > 1) { - val queryString: String = url.split("\\?")(1) - if (StringUtils.isNotBlank(queryString)) { - for (param <- queryString.split("\\&")) { - if (param.split("=").length == 2) - result += uriEncode(param.split("=")(0).toCharArray, false) + "=" + uriEncode(param.split("=")(1).toCharArray, false) + "&" - else - result += uriEncode(param.split("=")(0).toCharArray, false) + "&" - } - result = result.substring(0, result.length - 2) - } - result - } else result - } - - def getCanonicalHeaders(headers: Map[String, String], hashedPayload: String): String = { - var result: String = "" - headers.foreach(header => { - result += header._1.toLowerCase + ":" + header._2.trim + "\n" - }) - result - } - - def getSignedHeaders(keySet: scala.collection.Set[String]): String = { - var result: String = "" - keySet.foreach(key => { - result += key.toLowerCase + ";" - }) - result.substring(0, result.length - 1) - } - - def getHashedPayload(payload: String): String = { - if (StringUtils.isNotBlank(payload)) { - sha256Hash(payload) - } else { - sha256Hash("") - } - } - - def stringtoHex(str: String): String = { - DatatypeConverter.printHexBinary(str.getBytes("UTF8")) - } -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AzureRequestBody.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AzureRequestBody.scala deleted file mode 100644 index 675efb889..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AzureRequestBody.scala +++ /dev/null @@ -1,8 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -object AzureRequestBody { - - val create_asset = " {\"properties\": {\"description\": \"assetDescription\",\"alternateId\" : \"assetId\"}}" - val submit_job = "{\"properties\": {\"input\": {\"@odata.type\": \"#Microsoft.Media.JobInputHttp\",\"baseUri\": \"baseInputUrl\",\"files\": [\"inputVideoFile\"]},\"outputs\": [{\"@odata.type\": \"#Microsoft.Media.JobOutputAsset\",\"assetName\": \"assetId\"}]}}" - val create_stream_locator="{\"properties\":{\"assetName\": \"assetId\",\"streamingPolicyName\": \"policyName\"}}" -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AzureResult.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AzureResult.scala deleted file mode 100644 index 3ab1227f6..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/AzureResult.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -import org.apache.commons.lang3.StringUtils - -import scala.collection.immutable.HashMap - -object AzureResult extends Result { - - override def getSubmitJobResult(response: MediaResponse): Map[String, AnyRef] = { - val result = response.result - val output: Map[String, AnyRef] = result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("outputs", List).asInstanceOf[List[Map[String, AnyRef]]].head - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> result.getOrElse("name", "").toString, - "status" -> output.getOrElse("state", "").toString.toUpperCase(), - "submittedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("created", "").toString, - "lastModifiedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("lastModified", "").toString - ) - ) - - } - - override def getJobResult(response: MediaResponse): Map[String, AnyRef] = { - val result = response.result - val output: Map[String, AnyRef] = result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("outputs", List).asInstanceOf[List[Map[String, AnyRef]]].head - - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> result.getOrElse("name", "").toString, - "status" -> output.getOrElse("state", "").toString.toUpperCase(), - "submittedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("created", "").toString, - "lastModifiedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("lastModified", "").toString, - "error" -> { - if (StringUtils.equalsIgnoreCase(output.getOrElse("state", "").toString.toUpperCase(),"ERROR")) { - val errorMap: Map[String, AnyRef] = output.getOrElse("error", Map).asInstanceOf[Map[String, AnyRef]] - Map[String, String]( - "errorCode" -> errorMap.getOrElse("code", "").toString, - "errorMessage" -> errorMap.getOrElse("details", List).asInstanceOf[List[Map[String, AnyRef]]].head.getOrElse("message", "").toString - ) - } else { - null - } - } - ) - ) - } - - override def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - - override def getListJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/CaseClasses.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/CaseClasses.scala deleted file mode 100644 index d1a6f73b8..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/CaseClasses.scala +++ /dev/null @@ -1,24 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -import org.joda.time.DateTime -import scala.collection.immutable.HashMap - - -case class MediaRequest(id: String, params: Map[String, AnyRef] = new HashMap[String, AnyRef], request: Map[String, AnyRef] = new HashMap[String, AnyRef]); - -case class MediaResponse(id: String, ts: String, params: Map[String, Any] = new HashMap[String, AnyRef], responseCode: String, result: Map[String, AnyRef] = new HashMap[String, AnyRef]) - -object ResponseCode extends Enumeration { - type Code = Value - val OK = Value(200) - val CLIENT_ERROR = Value(400) - val SERVER_ERROR = Value(500) - val RESOURCE_NOT_FOUND = Value(404) -} - -trait AlgoOutput extends AnyRef - -case class JobRequest(client_key: String, request_id: String, job_id: Option[String], status: String, request_data: String, iteration: Int, dt_job_submitted: Option[DateTime] = None, location: Option[String] = None, dt_file_created: Option[DateTime] = None, dt_first_event: Option[DateTime] = None, dt_last_event: Option[DateTime] = None, dt_expiration: Option[DateTime] = None, dt_job_processing: Option[DateTime] = None, dt_job_completed: Option[DateTime] = None, input_events: Option[Int] = None, output_events: Option[Int] = None, file_size: Option[Long] = None, latency: Option[Int] = None, execution_time: Option[Long] = None, err_message: Option[String] = None, stage: Option[String] = None, stage_status: Option[String] = None, job_name: Option[String] = None) extends AlgoOutput - -case class JobStage(request_id: String, client_key: String, stage: String, stage_status: String, status: String, err_message: String = "", dt_job_processing: Option[DateTime] = Option(new DateTime())) -case class StreamingStage(request_id: String, client_key: String, job_id: String, stage: String, stage_status: String, status: String, iteration: Int, err_message: String = "") \ No newline at end of file diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/Response.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/Response.scala deleted file mode 100644 index a64b01d48..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/Response.scala +++ /dev/null @@ -1,65 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -import java.util.UUID -import scala.collection.immutable.HashMap -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.util.{HTTPResponse, JSONUtil} - - -object Response { - - lazy val MEDIA_SERVICE_TYPE = "" -// val MEDIA_SERVICE_TYPE = AppConfig.getConfig("media_service_type") - - def getResponse(response: HTTPResponse): MediaResponse = { - var result: Map[String, AnyRef] = new HashMap[String, AnyRef] - - try { - val body = response.body - if (StringUtils.isNotBlank(body)) - result = JSONUtil.deserialize[Map[String, AnyRef]](body) - } catch { - case e: UnsupportedOperationException => e.printStackTrace() - case e: Exception => e.printStackTrace() - } - - response.status match { - case 200 => getSuccessResponse(result) - case 201 => getSuccessResponse(result) - case 400 => getFailureResponse(result, "BAD_REQUEST", "Please Provide Correct Request Data.") - case 401 => getFailureResponse(result, "SERVER_ERROR", "Access Token Expired.") - case 404 => getFailureResponse(result, "RESOURCE_NOT_FOUND", "Resource Not Found.") - case 405 => getFailureResponse(result, "METHOD_NOT_ALLOWED", "Requested Operation Not Allowed.") - case 500 => getFailureResponse(result, "SERVER_ERROR", "Internal Server Error. Please Try Again Later!") - case _ => getFailureResponse(result, "SERVER_ERROR", "Internal Server Error. Please Try Again Later!") - } - - } - - def getSuccessResponse(result: Map[String, AnyRef]): MediaResponse = { - MediaResponse(UUID.randomUUID().toString, System.currentTimeMillis().toString, new HashMap[String, AnyRef], - ResponseCode.OK.toString, result) - } - - def getFailureResponse(result: Map[String, AnyRef], errorCode: String, errorMessage: String): MediaResponse = { - val respCode: String = errorCode match { - case "BAD_REQUEST" => ResponseCode.CLIENT_ERROR.toString - case "RESOURCE_NOT_FOUND" => ResponseCode.RESOURCE_NOT_FOUND.toString - case "METHOD_NOT_ALLOWED" => ResponseCode.CLIENT_ERROR.toString - case "SERVER_ERROR" => ResponseCode.SERVER_ERROR.toString - } - val params = HashMap[String, String]( - "err" -> errorCode, - "errMsg" -> errorMessage - ) - MediaResponse(UUID.randomUUID().toString, System.currentTimeMillis().toString, params, respCode, result) - } - - def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - - def getListJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/Result.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/Result.scala deleted file mode 100644 index eba65d267..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/helpers/Result.scala +++ /dev/null @@ -1,12 +0,0 @@ -package org.sunbird.job.livevideostream.helpers - -trait Result { - - def getSubmitJobResult(response: MediaResponse): Map[String, AnyRef] - - def getJobResult(response: MediaResponse): Map[String, AnyRef] - - def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] - - def getListJobResult(response: MediaResponse): Map[String, AnyRef] -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/AwsMediaService.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/AwsMediaService.scala deleted file mode 100644 index d6a46da53..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/AwsMediaService.scala +++ /dev/null @@ -1,81 +0,0 @@ -package org.sunbird.job.livevideostream.service - -import org.sunbird.job.livevideostream.exception.MediaServiceException -import org.sunbird.job.livevideostream.helpers.{AwsRequestBody, AwsSignUtils, MediaResponse, Response} -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig - -import java.text.SimpleDateFormat -import java.util.{Date, TimeZone} -import org.sunbird.job.util.HttpUtil - -import scala.collection.immutable.HashMap -import scala.reflect.io.File - -abstract class AwsMediaService extends IMediaService { - - protected def getApiUrl(apiName: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val host: String = config.getConfig("aws.api.endpoint") - val apiVersion: String = config.getConfig("aws.api.version") - val baseUrl: String = host + File.separator + apiVersion - apiName.toLowerCase() match { - case "job" => baseUrl + "/jobs" - case _ => throw new MediaServiceException("ERR_INVALID_API_NAME", "Please Provide Valid AWS Media Service API Name") - } - } - - protected def getJobDetails(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("job") + "/" + jobId - val header = getDefaultHeader("GET", url, null) - Response.getResponse(httpUtil.get(url, header)) - } - - protected def prepareJobRequestBody(jobRequest: Map[String, AnyRef])(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val queue = config.getConfig("aws.service.queue") - val role = config.getConfig("aws.service.role") - val streamType = config.getConfig("aws.stream.protocol").toLowerCase() - val artifactUrl = jobRequest.get("artifactUrl").mkString - val contentId = jobRequest.get("identifier").mkString - val pkgVersion = jobRequest.getOrElse("pkgVersion", "").toString - val inputFile = prepareInputUrl(artifactUrl) - val output = prepareOutputUrl(contentId, streamType, pkgVersion) - AwsRequestBody.submit_hls_job - .replace("queueId", queue) - .replace("mediaRole", role) - .replace("inputVideoFile", inputFile) - .replace("outputLocation", output) - } - - protected def prepareInputUrl(url: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val temp = url.split("content") - val bucket = config.getConfig("aws.content_bucket_name") - val separator = File.separator; - "s3:" + separator + separator + bucket + separator + "content" + temp(1) - } - - protected def prepareOutputUrl(contentId: String, streamType: String, pkgVersion: String)(implicit config: LiveVideoStreamGeneratorConfig): String = { - val bucket = config.getConfig("aws.content_bucket_name") - val output = streamType.toLowerCase + "_" + pkgVersion - val separator = File.separator; - "s3:" + separator + separator + bucket + separator + "content" + separator + contentId + separator + output + separator - } - - protected def getSignatureHeader()(implicit config: LiveVideoStreamGeneratorConfig): Map[String, String] = { - val formatter = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'") - formatter.setTimeZone(TimeZone.getTimeZone("UTC")) - val date = formatter.format(new Date()) - val host: String = config.getConfig("aws.api.endpoint").replace("https://", "") - Map[String, String]("Content-Type" -> "application/json", "host" -> host, "x-amz-date" -> date) - } - - protected def getDefaultHeader(httpMethod: String, url: String, payload: String)(implicit config: LiveVideoStreamGeneratorConfig): Map[String, String] = { - val signHeader = getSignatureHeader - val authToken = AwsSignUtils.generateToken(httpMethod, url, signHeader, payload) - val host: String = config.getConfig("aws.api.endpoint").replace("https://", "") - HashMap[String, String]( - "Content-Type" -> "application/json", - "host" -> host, - "x-amz-date" -> signHeader.get("x-amz-date").mkString, - "Authorization" -> authToken - ) - } -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/AzureMediaService.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/AzureMediaService.scala deleted file mode 100644 index 14e126712..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/AzureMediaService.scala +++ /dev/null @@ -1,149 +0,0 @@ -package org.sunbird.job.livevideostream.service - -import java.io.File -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.livevideostream.exception.MediaServiceException -import org.sunbird.job.util.{HttpUtil, JSONUtil} -import org.sunbird.job.livevideostream.helpers.{AzureRequestBody, MediaResponse, Response} -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig - -import scala.collection.immutable.HashMap - - -abstract class AzureMediaService extends IMediaService { - - private var API_ACCESS_TOKEN: String = "" - - private def getToken()(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val tenant = config.getSystemConfig("azure.tenant") - val clientKey = config.getSystemConfig("azure.token.client_key") - val clientSecret = config.getSystemConfig("azure.token.client_secret") - val loginUrl = config.getConfig("azure.login.endpoint") + "/" + tenant + "/oauth2/token" - - val data = Map[String, String]( - "grant_type" -> "client_credentials", - "client_id" -> clientKey, - "client_secret" -> clientSecret, - "resource" -> "https://management.core.windows.net/" - ) - - val header = Map[String, String]( - "Content-Type" -> "application/x-www-form-urlencoded", - "Keep-Alive" -> "true" - ) - - val response:MediaResponse = Response.getResponse(httpUtil.post_map(loginUrl, data, header)) - if(response.responseCode == "OK"){ - response.result("access_token").asInstanceOf[String] - } else { - throw new Exception("Error while getting the azure access token::"+JSONUtil.serialize(response)) - } - } - - protected def getJobDetails(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("job").replace("jobIdentifier", jobId) - val response:MediaResponse = Response.getResponse(httpUtil.get(url, getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while getting the job detail::"+JSONUtil.serialize(response)) - } - } - - protected def createAsset(assetId: String, jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("asset").replace("assetId", assetId) - val requestBody = AzureRequestBody.create_asset.replace("assetId", assetId) - .replace("assetDescription", "Output Asset for " + jobId) - val response:MediaResponse = Response.getResponse(httpUtil.put(url, requestBody, getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while creating asset::(assetId->"+assetId+", jobId->"+jobId+")::"+JSONUtil.serialize(response)) - } - } - - protected def createStreamingLocator(streamingLocatorName: String, assetName: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("stream_locator").replace("streamingLocatorName", streamingLocatorName) - val streamingPolicyName = config.getConfig("azure.stream.policy_name") - val reqBody = AzureRequestBody.create_stream_locator.replace("assetId", assetName).replace("policyName", streamingPolicyName) - Response.getResponse(httpUtil.put(url, reqBody, getDefaultHeader())) - } - - protected def getStreamingLocator(streamingLocatorName: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("stream_locator").replace("streamingLocatorName", streamingLocatorName) - val response:MediaResponse = Response.getResponse(httpUtil.get(url, getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while getStreamingLocator::(streamingLocatorName->" + streamingLocatorName + ")::"+JSONUtil.serialize(response)) - } - } - - protected def getStreamUrls(streamingLocatorName: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("list_paths").replace("streamingLocatorName", streamingLocatorName) - val response:MediaResponse = Response.getResponse(httpUtil.post(url, "{}", getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while getStreamUrls::(streamingLocatorName->" + streamingLocatorName + ")::"+JSONUtil.serialize(response)) - } - } - - protected def getApiUrl(apiName: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val subscriptionId: String = config.getSystemConfig("azure.subscription_id") - val resourceGroupName: String = config.getSystemConfig("azure.resource_group_name") - val accountName: String = config.getSystemConfig("azure.account_name") - val apiVersion: String = config.getConfig("azure.api.version") - val transformName: String = config.getConfig("azure.transform.default") - - val baseUrl: String = new StringBuilder().append(config.getConfig("azure.api.endpoint")+"/subscriptions/") - .append(subscriptionId) - .append("/resourceGroups/") - .append(resourceGroupName) - .append("/providers/Microsoft.Media/mediaServices/") - .append(accountName).mkString - - - apiName.toLowerCase() match { - case "asset" => baseUrl + "/assets/assetId?api-version=" + apiVersion - case "job" => baseUrl + "/transforms/" + transformName + "/jobs/jobIdentifier?api-version=" + apiVersion - case "stream_locator" => baseUrl + "/streamingLocators/streamingLocatorName?api-version=" + apiVersion - case "list_paths" => baseUrl + "/streamingLocators/streamingLocatorName/listPaths?api-version=" + apiVersion - case _ => throw new MediaServiceException("ERR_INVALID_API_NAME", "Please Provide Valid Media Service API Name") - } - } - - protected def getDefaultHeader()(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): Map[String, String] = { - val accessToken = if (StringUtils.isNotBlank(API_ACCESS_TOKEN)) API_ACCESS_TOKEN else getToken() - val authToken = "Bearer " + accessToken - HashMap[String, String]( - "Content-Type" -> "application/json", - "Accept" -> "application/json", - "Authorization" -> authToken - ) - } - - protected def prepareStreamingUrl(streamLocatorName: String, jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): Map[String, AnyRef] = { - val streamType = config.getConfig("azure.stream.protocol") - val streamHost = config.getConfig("azure.stream.base_url") - var url = "" - val listPathResponse = getStreamUrls(streamLocatorName) - if (listPathResponse.responseCode.equalsIgnoreCase("OK")) { - val urlList: List[Map[String, AnyRef]] = listPathResponse.result.getOrElse("streamingPaths", List).asInstanceOf[List[Map[String, AnyRef]]] - urlList.foreach(streamMap => { - if (StringUtils.equalsIgnoreCase(streamMap.getOrElse("streamingProtocol", null).toString, streamType)) { - url = streamMap("paths").asInstanceOf[List[String]].head - } - }) - val streamUrl = streamHost + url.replace("aapl", "aapl-v3") - HashMap[String, AnyRef]("streamUrl" -> streamUrl) - } else { - val getResponse: MediaResponse = getJobDetails(jobId) - val fileName: String = getResponse.result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("input", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("files", List).asInstanceOf[List[AnyRef]].head.toString - val getStreamResponse = getStreamingLocator(streamLocatorName); - val locatorId = getStreamResponse.result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("streamingLocatorId", "").toString - val streamUrl = streamHost + File.separator + locatorId + File.separator + fileName.replace(".mp4", ".ism") + "/manifest(format=m3u8-aapl-v3)" - HashMap[String, AnyRef]("streamUrl" -> streamUrl) - } - } -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/IMediaService.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/IMediaService.scala deleted file mode 100644 index 9370b9a7c..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/IMediaService.scala +++ /dev/null @@ -1,20 +0,0 @@ -package org.sunbird.job.livevideostream.service - -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.livevideostream.helpers.{MediaRequest, MediaResponse} - - -trait IMediaService { - - def submitJob(request: MediaRequest)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse - - def getJob(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse - - def getStreamingPaths(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse - - def listJobs(listJobsRequest: MediaRequest): MediaResponse - - def cancelJob(cancelJobRequest: MediaRequest): MediaResponse - -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/LiveVideoStreamService.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/LiveVideoStreamService.scala deleted file mode 100644 index d4501506e..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/LiveVideoStreamService.scala +++ /dev/null @@ -1,208 +0,0 @@ -package org.sunbird.job.livevideostream.service - -import java.util.UUID -import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} -import org.joda.time.DateTime -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.livevideostream.helpers.JobRequest -import org.sunbird.job.livevideostream.service.impl.MediaServiceFactory -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.job.livevideostream.helpers.{JobRequest, MediaRequest, MediaResponse, StreamingStage} - -import scala.collection.JavaConverters._ - -class LiveVideoStreamService(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil) { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[LiveVideoStreamService]) - private lazy val mediaService = MediaServiceFactory.getMediaService(config) - private lazy val dbKeyspace:String = config.dbKeyspace - private lazy val dbTable:String = config.dbTable - lazy val cassandraUtil:CassandraUtil = new CassandraUtil(config.lmsDbHost, config.lmsDbPort, config) - private lazy val clientKey:String = "SYSTEM_LP" - private lazy val SUBMITTED:String = "SUBMITTED" - private lazy val VIDEO_STREAMING:String = "VIDEO_STREAMING" - - def submitJobRequest(eData: Map[String, AnyRef]): Unit = { - val stageName = "STREAMING_JOB_SUBMISSION"; - val jobSubmitted = DateTime.now() - val requestId = UUID.randomUUID().toString - val jobRequest = JobRequest(clientKey, requestId, None, SUBMITTED, JSONUtil.serialize(eData), 0, Option(jobSubmitted), - Option(eData.getOrElse("artifactUrl", "").asInstanceOf[String]), None, None, None, None, None, - None, None, None, None, None, None, None, Option(stageName), Option(SUBMITTED), Option(VIDEO_STREAMING)) - - saveJobRequest(jobRequest) - submitStreamJob(jobRequest) - } - - def processJobRequest(metrics: Metrics): Unit = { - updateProcessingRequest(metrics) - resubmitFailedJob() - } - - def updateProcessingRequest(metrics: Metrics): Unit = { - val processingJobRequests = readFromDB(Map("status" -> "PROCESSING")) - val stageName = "STREAMING_JOB_COMPLETE" - - for (jobRequest <- processingJobRequests) { - val iteration = jobRequest.iteration - val streamStage = if (jobRequest.job_id != None) { - val mediaResponse:MediaResponse = mediaService.getJob(jobRequest.job_id.get) - logger.info("Get job details while saving: " + JSONUtil.serialize(mediaResponse.result)) - if(mediaResponse.responseCode.contentEquals("OK")) { - val job = mediaResponse.result.getOrElse("job", Map()).asInstanceOf[Map[String, AnyRef]] - val jobStatus = job.getOrElse("status","").asInstanceOf[String] - - if(config.jobStatus.contains(jobStatus)) { - val streamingUrl = mediaService.getStreamingPaths(jobRequest.job_id.get).result.getOrElse("streamUrl","").asInstanceOf[String] - val requestData = JSONUtil.deserialize[Map[String, AnyRef]](jobRequest.request_data) - val contentId = requestData.getOrElse("identifier", "").asInstanceOf[String] - val channel = requestData.getOrElse("channel", "").asInstanceOf[String] - - if(updatePreviewUrl(contentId, streamingUrl, channel)) { - StreamingStage(jobRequest.request_id, jobRequest.client_key, jobRequest.job_id.get, stageName, jobStatus, "FINISHED", iteration + 1); - } else { - // TODO:: Set job status to FAILED - null - } - } else if(jobStatus.equalsIgnoreCase("ERROR")){ - val errMessage = job.getOrElse("error", Map()).asInstanceOf[Map[String, AnyRef]].getOrElse("errorMessage", "No error message").asInstanceOf[String] - StreamingStage(jobRequest.request_id, jobRequest.client_key, jobRequest.job_id.get, stageName, jobStatus, "FAILED", iteration + 1, errMessage) - } else { - null - } - } else { - val errorMsg = mediaResponse.result.toString - StreamingStage(jobRequest.request_id, jobRequest.client_key, null, stageName, "FAILED", "FAILED", iteration + 1, errorMsg); - } - } else { - StreamingStage(jobRequest.request_id, jobRequest.client_key, null, stageName, "FAILED", "FAILED", iteration + 1, jobRequest.err_message.getOrElse("")); - } - - if (streamStage != null) { - val counter = if (streamStage.status.equals("FINISHED")) config.successEventCount else { - if (streamStage.iteration <= config.maxRetries) config.retryEventCount else config.failedEventCount - } - metrics.incCounter(counter) - updateJobRequestStage(streamStage) - } - } - } - - def resubmitFailedJob(): Unit = { - val failedJobRequests = readFromDB(Map("status" -> "FAILED", "iteration" -> Map("type"-> "lte", "value" -> config.maxRetries))).toArray - failedJobRequests.foreach { jobRequest => - submitStreamJob(jobRequest) - } - } - - def submitStreamJob(jobRequest: JobRequest): Unit = { - - val requestData = JSONUtil.deserialize[Map[String, AnyRef]](jobRequest.request_data) - val mediaRequest = MediaRequest(UUID.randomUUID().toString, null, requestData) - val response:MediaResponse = mediaService.submitJob(mediaRequest) - val stageName = "STREAMING_JOB_SUBMISSION" - var streamStage:Option[StreamingStage] = None - - if (response.responseCode.equals("OK")) { - val jobId = response.result.getOrElse("job", Map()).asInstanceOf[Map[String, AnyRef]].getOrElse("id","").asInstanceOf[String]; - val jobStatus = response.result.getOrElse("job", Map()).asInstanceOf[Map[String, AnyRef]].getOrElse("status","").asInstanceOf[String]; - streamStage = Option(StreamingStage(jobRequest.request_id, jobRequest.client_key, jobId, stageName, jobStatus, "PROCESSING", jobRequest.iteration + 1)) - } else { - val errorMsg = response.result.toString - - streamStage = Option(StreamingStage(jobRequest.request_id, jobRequest.client_key, null, stageName, "FAILED", "FAILED", jobRequest.iteration + 1, errorMsg)); - } - - updateJobRequestStage(streamStage.get); - } - - private def updatePreviewUrl(contentId: String, streamingUrl: String, channel: String): Boolean = { - if(streamingUrl.nonEmpty && contentId.nonEmpty) { - val requestBody = "{\"request\": {\"content\": {\"streamingUrl\":\""+ streamingUrl +"\", \"migrationVersion\":1.2}}}" - val url = config.lpURL + config.contentV4Update + contentId - val headers = Map[String, String]("X-Channel-Id" -> channel, "Content-Type"->"application/json") - val response:HTTPResponse = httpUtil.patch(url, requestBody, headers) - - if(response.status == 200){ - true - } else { - logger.error("Error while updating previewUrl for content : " + contentId + " :: "+response.body) -// throw new Exception("Error while updating previewUrl for content : " + contentId + " :: "+response.body) - false - } - } else { - false - } - } - - def readFromDB(columns: Map[String, AnyRef]): List[JobRequest] = { - val selectWhere: Select.Where = QueryBuilder.select().all() - .from(dbKeyspace, dbTable) - .allowFiltering() - .where() - - columns.map(col => { - col._2 match { - case value: List[Any] => - selectWhere.and(QueryBuilder.in(col._1, value.asJava)) - case value: Map[String, AnyRef] => - if (value("type") == "lte") { - selectWhere.and(QueryBuilder.lte(col._1, value("value"))) - } else { - selectWhere.and(QueryBuilder.gte(col._1, value("value"))) - } - case _ => - selectWhere.and(QueryBuilder.eq(col._1, col._2)) - } - }) - - selectWhere.and(QueryBuilder.eq("job_name", VIDEO_STREAMING)) - - val result = cassandraUtil.find(selectWhere.toString).asScala.toList.map { jr => - JobRequest(jr.getString("client_key"), jr.getString("request_id"), Option(jr.getString("job_id")), jr.getString("status"), jr.getString("request_data"), jr.getInt("iteration"), stage=Option(jr.getString("stage")), stage_status=Option(jr.getString("stage_status")),job_name=Option(jr.getString("job_name"))) - } - result - } - - def saveJobRequest(jobRequest: JobRequest): Boolean = { - val query = QueryBuilder.insertInto(dbKeyspace, dbTable) - .value("client_key", jobRequest.client_key) - .value("request_id", jobRequest.request_id) - .value("job_id", jobRequest.job_id.getOrElse("")) - .value("status", jobRequest.status) - .value("request_data", jobRequest.request_data) - .value("iteration", jobRequest.iteration) - .value("dt_job_submitted", setDateColumn(jobRequest.dt_job_submitted).get) - .value("location", jobRequest.location.get) - .value("stage", jobRequest.stage.get) - .value("stage_status", jobRequest.stage_status.get) - .value("job_name", jobRequest.job_name.get) - - val result = cassandraUtil.session.execute(query) - result.wasApplied() - } - - def updateJobRequestStage(streamStage: StreamingStage): Boolean = { - val query = QueryBuilder.update(dbKeyspace, dbTable) - .`with`(QueryBuilder.set("job_id", streamStage.job_id)) - .and(QueryBuilder.set("stage", streamStage.stage)) - .and(QueryBuilder.set("stage_status", streamStage.stage_status)) - .and(QueryBuilder.set("status", streamStage.status)) - .and(QueryBuilder.set("iteration", streamStage.iteration)) - .and(QueryBuilder.set("err_message", streamStage.err_message)) - .where(QueryBuilder.eq("request_id", streamStage.request_id)) - .and(QueryBuilder.eq("client_key", streamStage.client_key)) - - cassandraUtil.upsert(query.toString) - } - - def setDateColumn(date: Option[DateTime]): Option[Long] = { - val timestamp = date.orNull - if (null == timestamp) None else Option(timestamp.getMillis) - } - - def closeConnection(): Unit = { - cassandraUtil.close() - } -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/AwsMediaServiceImpl.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/AwsMediaServiceImpl.scala deleted file mode 100644 index cc4385a65..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/AwsMediaServiceImpl.scala +++ /dev/null @@ -1,47 +0,0 @@ -package org.sunbird.job.livevideostream.service.impl - -import org.sunbird.job.livevideostream.helpers.{MediaRequest, MediaResponse} -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.livevideostream.helpers.{AwsResult, MediaRequest, MediaResponse, Response} -import org.sunbird.job.livevideostream.service.AwsMediaService -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig - -import scala.collection.immutable.HashMap - -object AwsMediaServiceImpl extends AwsMediaService { - - override def submitJob(request: MediaRequest)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("job") - val reqBody = prepareJobRequestBody(request.request) - val header = getDefaultHeader("POST", url, reqBody) - val response:MediaResponse = Response.getResponse(httpUtil.post(url, reqBody, header)) - if (response.responseCode == "OK") Response.getSuccessResponse(AwsResult.getSubmitJobResult(response)) else response - } - - override def getJob(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val response = getJobDetails(jobId) - if (response.responseCode == "OK") Response.getSuccessResponse(AwsResult.getJobResult(response)) else response - } - - override def getStreamingPaths(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val region = config.getConfig("aws.region"); - val getResponse = getJobDetails(jobId) - val inputs: List[Map[String, AnyRef]] = getResponse.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("settings", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("inputs", List).asInstanceOf[List[Map[String, AnyRef]]] - val input: String = inputs.head.getOrElse("fileInput", "").toString - val outputGroups: List[Map[String, AnyRef]] = getResponse.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("settings", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("outputGroups", List).asInstanceOf[List[Map[String, AnyRef]]] - val outputGroupSettings = outputGroups.head.getOrElse("outputGroupSettings", Map).asInstanceOf[Map[String, AnyRef]] - val destination = outputGroupSettings.getOrElse("hlsGroupSettings", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("destination", "").asInstanceOf[String] - val temp = destination.split("_") - val output = config.getConfig("aws.stream.protocol").toLowerCase() + "_" + temp(temp.length-1).replace("/","").trim() - val host = "https://s3." + region + ".amazonaws.com" - val streamUrl: String = input.replace("s3:/", host) - .replace("artifact", output) - .replace(".mp4", ".m3u8") - .replace(".webm", ".m3u8") - Response.getSuccessResponse(HashMap[String, AnyRef]("streamUrl" -> streamUrl)) - } - - override def listJobs(listJobsRequest: MediaRequest): MediaResponse = ??? - - override def cancelJob(cancelJobRequest: MediaRequest): MediaResponse = ??? -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/AzureMediaServiceImpl.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/AzureMediaServiceImpl.scala deleted file mode 100644 index b93eaf020..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/AzureMediaServiceImpl.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.livevideostream.service.impl - -import org.sunbird.job.livevideostream.service.AzureMediaService -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.livevideostream.helpers.{AzureRequestBody, AzureResult, MediaRequest, MediaResponse, Response} - -import scala.collection.immutable.HashMap - - -object AzureMediaServiceImpl extends AzureMediaService { - - override def submitJob(request: MediaRequest)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val inputUrl = request.request.getOrElse("artifactUrl", "").toString - val contentId = request.request.get("identifier").mkString - val jobId = contentId + "_" + System.currentTimeMillis() - val temp = inputUrl.splitAt(inputUrl.lastIndexOf("/") + 1) - val assetId = "asset-" + jobId - - val createAssetResponse = createAsset(assetId, jobId) - - if (createAssetResponse.responseCode.equalsIgnoreCase("OK")) { - val apiUrl = getApiUrl("job").replace("jobIdentifier", jobId) - val reqBody = AzureRequestBody.submit_job.replace("assetId", assetId).replace("baseInputUrl", temp._1).replace("inputVideoFile", temp._2) - val response:MediaResponse = Response.getResponse(httpUtil.put(apiUrl, reqBody, getDefaultHeader())) - if (response.responseCode == "OK") Response.getSuccessResponse(AzureResult.getSubmitJobResult(response)) else response - } else { - Response.getFailureResponse(createAssetResponse.result, "SERVER_ERROR", "Output Asset [ " + assetId + " ] Creation Failed for Job : " + jobId) - } - } - - override def getJob(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val response = getJobDetails(jobId) - if (response.responseCode == "OK") Response.getSuccessResponse(AzureResult.getJobResult(response)) else response - } - - override def getStreamingPaths(jobId: String)(implicit config: LiveVideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val streamLocatorName = "sl-" + jobId - val assetName = "asset-" + jobId - val locatorResponse = createStreamingLocator(streamLocatorName, assetName) - if (locatorResponse.responseCode == "OK" || locatorResponse.responseCode == "CLIENT_ERROR") { - Response.getSuccessResponse(prepareStreamingUrl(streamLocatorName, jobId)) - } else { - Response.getFailureResponse(new HashMap[String, AnyRef], "SERVER_ERROR", "Streaming Locator [" + streamLocatorName + "] Creation Failed for Job : " + jobId) - } - } - - override def listJobs(listJobsRequest: MediaRequest): MediaResponse = { - null - } - - override def cancelJob(cancelJobRequest: MediaRequest): MediaResponse = { - null - } - -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/MediaServiceFactory.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/MediaServiceFactory.scala deleted file mode 100644 index f6587a451..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/service/impl/MediaServiceFactory.scala +++ /dev/null @@ -1,17 +0,0 @@ -package org.sunbird.job.livevideostream.service.impl - -import org.sunbird.job.livevideostream.exception.MediaServiceException -import org.sunbird.job.livevideostream.service.IMediaService -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig - -object MediaServiceFactory { - - def getMediaService(config: LiveVideoStreamGeneratorConfig): IMediaService = { - val serviceType: String = config.getConfig("media_service_type") - serviceType match { - case "azure" => AzureMediaServiceImpl - case "aws" => AwsMediaServiceImpl - case _ => throw new MediaServiceException("ERR_INVALID_SERVICE_TYPE", "Please Provide Valid Media Service Name") - } - } -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/task/LiveVideoStreamGeneratorConfig.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/task/LiveVideoStreamGeneratorConfig.scala deleted file mode 100644 index 0e3827be9..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/task/LiveVideoStreamGeneratorConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package org.sunbird.job.livevideostream.task - -import java.util - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.livevideostream.exception.MediaServiceException - -class LiveVideoStreamGeneratorConfig(override val config: Config) extends BaseJobConfig(config, "video-stream-generator") { - - private val serialVersionUID = 2905979434303791379L - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - - val timerDuration = config.getInt("task.timer.duration") // Timer duration in sec. - val maxRetries = if (config.hasPath("task.max.retries")) config.getInt("task.max.retries") else 10 - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val retryEventCount = "retry-events-count" - val skippedEventCount = "skipped-events-count" - - // Consumers - val videoStreamConsumer = "video-streaming-consumer" - val videoStreamGeneratorFunction = "manage-streaming-jobs" - - // Cassandra Configurations - val dbTable: String = config.getString("lms-cassandra.table") - val dbKeyspace: String = config.getString("lms-cassandra.keyspace") - val hierarchyPrimaryKey: List[String] = List("identifier") - - // LP Configurations - val lpURL: String = config.getString("service.content.basePath") - val contentV4Update = "/content/v4/system/update/" - - val jobStatus:util.List[String] = if(config.hasPath("media_service_job_success_status")) config.getStringList("media_service_job_success_status") else util.Arrays.asList("FINISHED", "COMPLETE") - - def getConfig(key: String): String = { - if (config.hasPath(key)) - config.getString(key) - else throw new MediaServiceException("CONFIG_NOT_FOUND", "Configuration for key [" + key + "] Not Found.") - } - - def getSystemConfig(key: String): String = { - val sysKey=key.replaceAll("\\.","_") - if (config.hasPath(sysKey)) - config.getString(sysKey) - else throw new MediaServiceException("CONFIG_NOT_FOUND", "Configuration for key [" + sysKey + "] Not Found.") - } -} diff --git a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/task/LiveVideoStreamGeneratorStreamTask.scala b/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/task/LiveVideoStreamGeneratorStreamTask.scala deleted file mode 100644 index 3b60870d0..000000000 --- a/live-video-stream-generator/src/main/scala/org/sunbird/job/livevideostream/task/LiveVideoStreamGeneratorStreamTask.scala +++ /dev/null @@ -1,53 +0,0 @@ -package org.sunbird.job.livevideostream.task - -import java.io.File -import java.util -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.livevideostream.domain.Event -import org.sunbird.job.livevideostream.functions.LiveVideoStreamGenerator -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - - -class LiveVideoStreamGeneratorStreamTask(config: LiveVideoStreamGeneratorConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - env.addSource(source).name(config.videoStreamConsumer) - .uid(config.videoStreamConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .keyBy(_.identifier) - .process(new LiveVideoStreamGenerator(config, httpUtil)) - .name(config.videoStreamGeneratorFunction) - .uid(config.videoStreamGeneratorFunction) - .setParallelism(config.parallelism) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object LiveVideoStreamGeneratorStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("live-video-stream-generator.conf").withFallback(ConfigFactory.systemEnvironment())) - val videoStreamConfig = new LiveVideoStreamGeneratorConfig(config) - val kafkaUtil = new FlinkKafkaConnector(videoStreamConfig) - val httpUtil = new HttpUtil - val task = new LiveVideoStreamGeneratorStreamTask(videoStreamConfig, kafkaUtil, httpUtil) - task.process() - } -} - -// $COVERAGE-ON$ diff --git a/live-video-stream-generator/src/test/resources/job_request.cql b/live-video-stream-generator/src/test/resources/job_request.cql deleted file mode 100644 index 820a372b7..000000000 --- a/live-video-stream-generator/src/test/resources/job_request.cql +++ /dev/null @@ -1,2 +0,0 @@ - -INSERT INTO local_platform_db.job_request (client_key,request_id,dt_expiration,dt_file_created,dt_first_event,dt_job_completed,dt_job_processing,dt_job_submitted,dt_last_event,err_message,execution_time,file_size,input_events,iteration,job_id,job_name,latency,location,output_events,request_data,stage,stage_status,status) values ('SYSTEM_LP', '32722dd03b737ed32014a01b8c1f7c83', null, null, null, null, '2019-01-02 22:30:08.033+0000', '2018-12-18 20:11:03.088+0000', null, '', 0, 0, 0, 1, 'do_3126597193576939521909_1605816926271', 'VIDEO_STREAMING', 0, 'https://testurl.com/test.mp4', 0, '{"content_id":"do_3126597193576939521909","channel":"01254290140407398431","artifactUrl":"https://testurl.com/test.mp4"}', 'STREAMING_JOB_SUBMISSION', 'PROCESSING', 'PROCESSING'); \ No newline at end of file diff --git a/live-video-stream-generator/src/test/resources/logback-test.xml b/live-video-stream-generator/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/live-video-stream-generator/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/live-video-stream-generator/src/test/resources/test.conf b/live-video-stream-generator/src/test/resources/test.conf deleted file mode 100644 index b1c703229..000000000 --- a/live-video-stream-generator/src/test/resources/test.conf +++ /dev/null @@ -1,74 +0,0 @@ -include "base-test.conf" - -kafka { - input.topic = "sunbirddev.live.video.stream.request" - groupId = "sunbirddev-live-video-stream-generator-group" -} - -task { - consumer.parallelism = 1 - timer.duration = 10 - max.retries = 10 -} - -lms-cassandra { - keyspace = "local_platform_db" - table = "job_request" -} - -service { - content { - basePath = "http://dev.sunbirded.org/content" - } -} - -threshold.batch.read.interval = 60 // In sec -threshold.batch.read.size = 1000 -threshold.batch.write.size = 4 - - azure { - location = "centralindia" - tenant = "tenant name" - subscription_id = "subscription id" - - login { - endpoint="https://login.microsoftonline.com" - } - - api { - endpoint="https://management.azure.com" - version = "2018-07-01" - } - - account_name = "account name" - resource_group_name = "Resource Group Name" - - transform { - default = "media_transform_default" - hls = "media_transform_hls" - } - - stream { - base_url = "https://sunbirdspikemedia-inct.streaming.media.azure.net" - endpoint_name = "default" - protocol = "Hls" - policy_name = "Predefined_ClearStreamingOnly" - } - - token { - client_key = "client key" - client_secret = "client secret" - } -} - -azure_tenant="test_tenant" -azure_subscription_id="test_id" -azure_account_name="test_account_name" -azure_resource_group_name="test_resource_group_name" -azure_token_client_key="test_client_key" -azure_token_client_secret="test_client_secret" -elasticsearch.service.endpoint="test_service_endpoint" -elasticsearch.index.compositesearch.name="test_compositesearch_name" - -media_service_type="azure" - diff --git a/live-video-stream-generator/src/test/resources/test.cql b/live-video-stream-generator/src/test/resources/test.cql deleted file mode 100644 index 6ed1eca28..000000000 --- a/live-video-stream-generator/src/test/resources/test.cql +++ /dev/null @@ -1,31 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS local_platform_db WITH replication = { - 'class': 'SimpleStrategy', - 'replication_factor': '1' -}; - -CREATE TABLE IF NOT EXISTS local_platform_db.job_request ( - client_key text, - request_id text, - job_id text, - status text, - request_data text, - location text, - dt_file_created timestamp, - dt_first_event timestamp, - dt_last_event timestamp, - dt_expiration timestamp, - iteration int, - dt_job_submitted timestamp, - dt_job_processing timestamp, - dt_job_completed timestamp, - input_events int, - output_events int, - file_size bigint, - latency int, - execution_time bigint, - err_message text, - stage text, - stage_status text, - job_name text, - PRIMARY KEY (client_key, request_id) -); \ No newline at end of file diff --git a/live-video-stream-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/live-video-stream-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index c56e49461..000000000 --- a/live-video-stream-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1598956686981,"mid":"LP.1598956686981.a260af12-cd9b-4ffd-a525-1d944df47c61","actor":{"id":"Post Publish Processor","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"01254290140407398431","env":"sunbirddev"},"object":{"ver":"1587632475439","id":"do_3126597193576939521910"},"edata":{"action":"post-publish-process","iteration":1,"identifier":"do_3126597193576939521910","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1126980548391075841140/ariel-view-of-earth.mp4","mimeType":"video/mp4","contentType":"Resource","pkgVersion":1,"status":"Live"}} - |""".stripMargin - - val EVENT_2: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1598956686981,"mid":"LP.1598956686981.a260af12-cd9b-4ffd-a525-1d944df47c61","actor":{"id":"Post Publish Processor","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"01254290140407398431","env":"sunbirddev"},"object":{"ver":"1587632475439","id":"do_3126597193576939521910"},"edata":{"action":"post-publish-process","iteration":1,"identifier":"do_3126597193576939521910","artifactUrl":"https://sunbirded.com/test.mp4","mimeType":"video","contentType":"Resource","pkgVersion":1,"status":"Live"}} - |""".stripMargin -} \ No newline at end of file diff --git a/live-video-stream-generator/src/test/scala/org/sunbird/job/spec/LiveVideoStreamGeneratorTaskTestSpec.scala b/live-video-stream-generator/src/test/scala/org/sunbird/job/spec/LiveVideoStreamGeneratorTaskTestSpec.scala deleted file mode 100644 index d64fee22b..000000000 --- a/live-video-stream-generator/src/test/scala/org/sunbird/job/spec/LiveVideoStreamGeneratorTaskTestSpec.scala +++ /dev/null @@ -1,129 +0,0 @@ -package org.sunbird.job.spec - -import com.datastax.driver.core.Row -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, anyString, contains} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.livevideostream.domain.Event -import org.sunbird.job.livevideostream.service.IMediaService -import org.sunbird.job.livevideostream.task.{LiveVideoStreamGeneratorConfig, LiveVideoStreamGeneratorStreamTask} -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.util - -class LiveVideoStreamGeneratorTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val mediaService: IMediaService = mock[IMediaService](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig: LiveVideoStreamGeneratorConfig = new LiveVideoStreamGeneratorConfig(config) - val mockHttpUtil:HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) -// val httpUtil:HttpUtil = new HttpUtil - var cassandraUtil: CassandraUtil = _ - var currentMilliSecond = 1605816926271L - - val accessTokenResp = """{"token_type":"Bearer","expires_in":"3599","ext_expires_in":"3599","expires_on":"1605789466","not_before":"1605785566","resource":"https://management.core.windows.net/","access_token":"testToken"}""" - val assetJson = """{"name":"asset-do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-aaaa-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/assets/asset-do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/assets","properties":{"assetId":"aaaaaaa-13bb-45c7-aaaa-32ac2e97cf12","created":"2020-11-19T20:16:54.463Z","lastModified":"2020-11-19T20:20:33.613Z","alternateId":"asset-do_3126597193576939521910_1605816926271","description":"Output Asset for do_3126597193576939521910_1605816926271","container":"asset-aaaaaaa-13bb-45c7-b186-32ac2e97cf12","storageAccountName":"sunbirddevmedia","storageEncryptionFormat":"None"}}""" - val submitJobJson = """{"name":"do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-8a14-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/transforms/media_transform_default/jobs/do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/transforms/jobs","properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"Queued","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getJobJson = """{"name":"do_3126597193576939521910_1605816926271","job":{"status":"Finished"},"properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"FINISHED","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getStreamUrlJson = """{"streamingPaths":[{"streamingProtocol":"Hls","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-aapl)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-cmaf)"]},{"streamingProtocol":"Dash","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-csf)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-cmaf)"]},{"streamingProtocol":"SmoothStreaming","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest"]}],"downloadPaths":[]}""" - val getStreamLocatorJson = """{"properties":{"streamingLocatorId":"adcacdd-13bb-45c7-aaaa-32ac2e97cf12"}}""" - - override protected def beforeAll(): Unit = { - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.lmsDbHost, jobConfig.lmsDbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - testCassandraUtil(cassandraUtil) - BaseMetricsReporter.gaugeMetrics.clear() - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - flinkCluster.after() - super.afterAll() - } - - override protected def afterEach():Unit = { - super.afterEach() - } - - ignore should "submit a job" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new VideoStreamGeneratorMapSource) - - when(mockHttpUtil.post_map(contains("/oauth2/token"), any[Map[String, AnyRef]](), any[Map[String, String]]())).thenReturn(HTTPResponse(200, accessTokenResp)) - when(mockHttpUtil.put(contains("/providers/Microsoft.Media/mediaServices/"+jobConfig.getSystemConfig("azure.account.name")+"/assets/asset-"), anyString(), any())).thenReturn(HTTPResponse(200, assetJson)) - when(mockHttpUtil.put(contains("transforms/media_transform_default/jobs"), anyString(), any())).thenReturn(HTTPResponse(200, submitJobJson)) - when(mockHttpUtil.get(contains("transforms/media_transform_default/jobs"), any())).thenReturn(HTTPResponse(200, getJobJson)) - - when(mockHttpUtil.post(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271/listPaths?api-version="), any(), any())).thenReturn(HTTPResponse(200, getStreamUrlJson)) - when(mockHttpUtil.put(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any(), any())).thenReturn(HTTPResponse(400, getJobJson)) - when(mockHttpUtil.get(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any())).thenReturn(HTTPResponse(200, getStreamLocatorJson)) - when(mockHttpUtil.patch(contains(jobConfig.contentV4Update), any(), any())).thenReturn(HTTPResponse(200, getJobJson)) - - new LiveVideoStreamGeneratorStreamTask(jobConfig, mockKafkaUtil, mockHttpUtil).process() - val event1Progress = readFromCassandra(EventFixture.EVENT_1) - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(1) - event1Progress.size() should be(1) - - event1Progress.forEach(col => { - col.getObject("status") should be("FINISHED") - }) - - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - } - - def readFromCassandra(event: String): util.List[Row] = { - val event1 = JSONUtil.deserialize[Map[String, Any]](event) - val contentId = event1("object").asInstanceOf[Map[String, AnyRef]]("id") - val query = s"select * from ${jobConfig.dbKeyspace}.${jobConfig.dbTable} where job_id='${contentId}_$currentMilliSecond' ALLOW FILTERING;" - cassandraUtil.find(query) - } - -} - -class VideoStreamGeneratorMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2),0, 11)) - - } - - override def cancel(): Unit = {} - -} \ No newline at end of file diff --git a/live-video-stream-generator/src/test/scala/org/sunbird/job/spec/service/LiveVideoStreamServiceTestSpec.scala b/live-video-stream-generator/src/test/scala/org/sunbird/job/spec/service/LiveVideoStreamServiceTestSpec.scala deleted file mode 100644 index b32aa9725..000000000 --- a/live-video-stream-generator/src/test/scala/org/sunbird/job/spec/service/LiveVideoStreamServiceTestSpec.scala +++ /dev/null @@ -1,96 +0,0 @@ -package org.sunbird.job.spec.service - -import java.util -import com.datastax.driver.core.Row -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.joda.time.DateTimeUtils -import org.mockito.ArgumentMatchers.{any, anyString, contains} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.livevideostream.service.LiveVideoStreamService -import org.sunbird.job.livevideostream.task.LiveVideoStreamGeneratorConfig -import org.sunbird.spec.BaseTestSpec -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.job.livevideostream.domain.Event -import org.sunbird.job.Metrics - -class LiveVideoStreamServiceTestSpec extends BaseTestSpec { - var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf") - lazy val jobConfig: LiveVideoStreamGeneratorConfig = new LiveVideoStreamGeneratorConfig(config) - val httpUtil: HttpUtil = new HttpUtil - val mockHttpUtil:HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - val metricJson = s"""{"${jobConfig.totalEventsCount}": 0, "${jobConfig.skippedEventCount}": 0}""" - val mockMetrics = mock[Metrics](Mockito.withSettings().serializable()) - - val accessTokenResp = """{"token_type":"Bearer","expires_in":"3599","ext_expires_in":"3599","expires_on":"1605789466","not_before":"1605785566","resource":"https://management.core.windows.net/","access_token":"testToken"}""" - val assetJson = """{"name":"asset-do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-aaaa-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/assets/asset-do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/assets","properties":{"assetId":"aaaaaaa-13bb-45c7-aaaa-32ac2e97cf12","created":"2020-11-19T20:16:54.463Z","lastModified":"2020-11-19T20:20:33.613Z","alternateId":"asset-do_3126597193576939521910_1605816926271","description":"Output Asset for do_3126597193576939521910_1605816926271","container":"asset-aaaaaaa-13bb-45c7-b186-32ac2e97cf12","storageAccountName":"sunbirddevmedia","storageEncryptionFormat":"None"}}""" - val submitJobJson = """{"name":"do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-8a14-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/transforms/media_transform_default/jobs/do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/transforms/jobs","properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"Queued","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getJobJson = """{"name":"do_3126597193576939521910_1605816926271","job":{"status":"Finished"},"properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"FINISHED","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getStreamUrlJson = """{"streamingPaths":[{"streamingProtocol":"Hls","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-aapl)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-cmaf)"]},{"streamingProtocol":"Dash","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-csf)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-cmaf)"]},{"streamingProtocol":"SmoothStreaming","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest"]}],"downloadPaths":[]}""" - val getStreamLocatorJson = """{"properties":{"streamingLocatorId":"adcacdd-13bb-45c7-aaaa-32ac2e97cf12"}}""" - - override protected def beforeAll(): Unit = { - DateTimeUtils.setCurrentMillisFixed(1605816926271L); - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.lmsDbHost, jobConfig.lmsDbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - testCassandraUtil(cassandraUtil) - super.beforeAll() - } - - override protected def afterAll(): Unit = { - DateTimeUtils.setCurrentMillisSystem(); - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - super.afterAll() - } - - "VideoStreamService" should "submit job request" in { - when(mockHttpUtil.post_map(contains("/oauth2/token"), any[Map[String, AnyRef]](), any[Map[String, String]]())).thenReturn(HTTPResponse(200, accessTokenResp)) - when(mockHttpUtil.put(contains("/providers/Microsoft.Media/mediaServices/"+jobConfig.getSystemConfig("azure.account.name")+"/assets/asset-"), anyString(), any())).thenReturn(HTTPResponse(200, assetJson)) - when(mockHttpUtil.put(contains("transforms/media_transform_default/jobs"), anyString(), any())).thenReturn(HTTPResponse(200, submitJobJson)) - when(mockHttpUtil.get(contains("transforms/media_transform_default/jobs"), any())).thenReturn(HTTPResponse(200, getJobJson)) - - when(mockHttpUtil.post(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271/listPaths?api-version="), any(), any())).thenReturn(HTTPResponse(200, getStreamUrlJson)) - when(mockHttpUtil.put(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any(), any())).thenReturn(HTTPResponse(400, getJobJson)) - when(mockHttpUtil.get(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any())).thenReturn(HTTPResponse(200, getStreamLocatorJson)) - when(mockHttpUtil.patch(contains(jobConfig.contentV4Update), any(), any())).thenReturn(HTTPResponse(200, getJobJson)) - doNothing().when(mockMetrics).incCounter(any()) - - val eventMap1 = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 12) - - val videoStreamService = new LiveVideoStreamService()(jobConfig, mockHttpUtil); - videoStreamService.submitJobRequest(eventMap1.eData) - videoStreamService.processJobRequest(mockMetrics) - - val event1Progress = readFromCassandra(EventFixture.EVENT_1) - event1Progress.size() should be(1) - - event1Progress.forEach(col => { - col.getObject("status") should be("FINISHED") - }) - - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - } - - def readFromCassandra(event: String): util.List[Row] = { - val event1 = JSONUtil.deserialize[Map[String, Any]](event) - val contentId = event1("object").asInstanceOf[Map[String, AnyRef]]("id") - val query = s"select * from ${jobConfig.dbKeyspace}.${jobConfig.dbTable} where job_id='${contentId}_1605816926271' ALLOW FILTERING;" - cassandraUtil.find(query) - } -} diff --git a/metrics-data-transformer/pom.xml b/metrics-data-transformer/pom.xml deleted file mode 100644 index 406d11598..000000000 --- a/metrics-data-transformer/pom.xml +++ /dev/null @@ -1,214 +0,0 @@ - - - 4.0.0 - - knowledge-platform-jobs - org.sunbird - 1.0 - - metrics-data-transformer - 1.0.0 - jar - metrics-data-transformer - - Metrics data transformer Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - com.squareup.okhttp3 - mockwebserver - 4.9.1 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.metricstransformer.task.MetricsDataTransformerStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - metrics-data-transformer-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - - \ No newline at end of file diff --git a/metrics-data-transformer/src/main/resources/log4j.properties b/metrics-data-transformer/src/main/resources/log4j.properties deleted file mode 100644 index 553e7ae03..000000000 --- a/metrics-data-transformer/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=metrics-data-transformer.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/metrics-data-transformer/src/main/resources/metrics-data-transformer.conf b/metrics-data-transformer/src/main/resources/metrics-data-transformer.conf deleted file mode 100644 index ab433eb2d..000000000 --- a/metrics-data-transformer/src/main/resources/metrics-data-transformer.conf +++ /dev/null @@ -1,26 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - groupId = "sunbirddev-metrics-data-transformer-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - window.time = 60 -} - -timezone = "IST" - -service { - content.basePath = "https://localhost:9000/action" - sourcing.content.basePath = "http://localhost/content" -} - -content_read_api = "/content/v3/read" -content_update_api= "/content/v4/system/update" - -sourcing.update.api.response.error.code = ["404"] - -data.metrics = ["me_totalRatingsCount","me_averageRating","me_totalTimeSpentInSec","me_totalPlaySessionCount"] \ No newline at end of file diff --git a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/domain/Event.scala b/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/domain/Event.scala deleted file mode 100644 index 545351f12..000000000 --- a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/domain/Event.scala +++ /dev/null @@ -1,19 +0,0 @@ -package org.sunbird.job.metricstransformer.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val jobName = "MetricsDataTransformer" - - def nodeUniqueId: String = readOrDefault("nodeUniqueId", "") - - def channel: String = readOrDefault("channel","") - - def transactionData: Map[String, AnyRef] = readOrDefault("transactionData", Map()) - - def isValidContent(originId: String, originData: Map[String,AnyRef]): Boolean = { - StringUtils.isNotBlank(originId) && !originData.isEmpty - } -} \ No newline at end of file diff --git a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/function/MetricsDataTransformerFunction.scala b/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/function/MetricsDataTransformerFunction.scala deleted file mode 100644 index 6f512f5d9..000000000 --- a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/function/MetricsDataTransformerFunction.scala +++ /dev/null @@ -1,46 +0,0 @@ -package org.sunbird.job.metricstransformer.function - -import java.util -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} -import org.sunbird.job.metricstransformer.domain.Event -import org.sunbird.job.metricstransformer.service.MetricsDataTransformerService -import org.sunbird.job.metricstransformer.task.MetricsDataTransformerConfig -import org.sunbird.job.util.HttpUtil - -class MetricsDataTransformerFunction(config: MetricsDataTransformerConfig, httpUtil: HttpUtil) - (implicit mapTypeInfo: TypeInformation[util.Map[String, Any]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessKeyedFunction[String, Event, String](config) with MetricsDataTransformerService { - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(event: Event, - context: KeyedProcessFunction[String, Event, String]#Context, - metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - val propertyMap = event.transactionData("properties").asInstanceOf[Map[String, AnyRef]] - val mapKeys = propertyMap.keySet.toList - - val eventMetrics = config.metrics.stream().toArray.map(_.asInstanceOf[String]) - val filteredKeys = eventMetrics.intersect(mapKeys) - - if(filteredKeys.length > 0) { - processEvent(event, metrics, filteredKeys)(config, httpUtil) - } else metrics.incCounter(config.skippedEventCount) - - } - -} diff --git a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/service/MetricsDataTransformerService.scala b/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/service/MetricsDataTransformerService.scala deleted file mode 100644 index ee25ac7d2..000000000 --- a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/service/MetricsDataTransformerService.scala +++ /dev/null @@ -1,126 +0,0 @@ -package org.sunbird.job.metricstransformer.service - -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.metricstransformer.domain.Event -import org.sunbird.job.metricstransformer.task.MetricsDataTransformerConfig -import org.sunbird.job.util.{HTTPResponse, HttpUtil, ScalaJsonUtil} - -case class ContentProps(nv: AnyRef, ov: AnyRef) - -trait MetricsDataTransformerService { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[MetricsDataTransformerService]) - - @throws(classOf[InvalidEventException]) - def processEvent(event: Event, metrics: Metrics, keys: Array[String])(implicit config: MetricsDataTransformerConfig, httpUtil: HttpUtil): Unit = { - val identifier = event.nodeUniqueId - val url = config.contentServiceBaseUrl + config.contentReadApi + "/" + identifier - var response: Map[String, AnyRef] = Map() - try { - response = getContent(url)(config, httpUtil) - logger.info(s"Response for content $identifier ::" + response) - } catch { - case ex: Exception => - metrics.incCounter(config.failedEventCount) - ex.printStackTrace() - throw new InvalidEventException(s"Error while getting content data from read API for :: $identifier") - } - - if (!response.isEmpty) { - val propertyMap = event.transactionData("properties").asInstanceOf[Map[String, AnyRef]] - var contentMetrics = "" - - keys.foreach(f => { - val property = ScalaJsonUtil.deserialize[ContentProps](ScalaJsonUtil.serialize(propertyMap(f))) - if (null != property.nv) { - contentMetrics = contentMetrics + s""""$f": ${property.nv},""".stripMargin - } - }) - val sourcingId = response.getOrElse("origin", "").asInstanceOf[String] - val originData = response.getOrElse("originData", Map()).asInstanceOf[Map[String, AnyRef]] - if(event.isValidContent(sourcingId, originData)) { - val hasOriginContent: Boolean = readOriginContent(sourcingId, metrics, event)(config,httpUtil) - if(hasOriginContent) { - try { - val channel = event.channel - updateContentMetrics(contentMetrics, sourcingId, channel, metrics)(config, httpUtil) - logger.info("Updated content metrics: " + identifier) - } catch { - case ex: Exception => - logger.error("Error while writing content metrics :: " + event.getJson + " :: ", ex) - metrics.incCounter(config.failedEventCount) - ex.printStackTrace() - throw new InvalidEventException(s"Error writing metrics for sourcing id: $sourcingId", Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } else { - logger.info("Sourcing Read API does not have details for the identifier: " + identifier) - metrics.incCounter(config.skippedEventCount) - } - } else { - logger.info("Origin and originData of the content is not present in the response of content-read API for id: " + identifier) - metrics.incCounter(config.skippedEventCount) - } - } - else { - logger.info(s"Learning event skipped, no sourcing identifier found: $identifier") - metrics.incCounter(config.skippedEventCount) - } - } - - def readOriginContent(sourcingId: String, metrics: Metrics, event: Event)(config: MetricsDataTransformerConfig, httpUtil: HttpUtil): Boolean = { - val sourcingReadURL = config.lpURL + config.contentReadApi + "/" + sourcingId - try { - val response = getContent(sourcingReadURL)(config,httpUtil) - if (!response.isEmpty) true else false - } catch { - case ex: Exception => - metrics.incCounter(config.failedEventCount) - ex.printStackTrace() - throw new InvalidEventException(s"Error while getting content data from sourcing content read API for :: $sourcingId", Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - def getContent(url: String)(config: MetricsDataTransformerConfig, httpUtil: HttpUtil): Map[String, AnyRef] = { - val response = httpUtil.get(url, config.defaultHeaders) - if (200 == response.status) { - ScalaJsonUtil.deserialize[Map[String, AnyRef]](response.body) - .getOrElse("result", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - .getOrElse("content", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - } else if (404 == response.status) { - Map() - } else { - throw new InvalidEventException(s"Error from content read API with response: $response") - } - } - - def updateContentMetrics(contentProperty: String, sourcingId: String, channel: String, metrics: Metrics)(config: MetricsDataTransformerConfig, httpUtil: HttpUtil): Boolean = { - val updateUrl = config.lpURL + config.contentUpdate + "/" + sourcingId - val contentMetrics = if (contentProperty.nonEmpty) contentProperty.substring(0, contentProperty.length - 1) else contentProperty - - val request = - s""" - |{ - | "request": { - | "content": { - | $contentMetrics - | } - | } - |}""".stripMargin - - val headers = config.defaultHeaders ++ Map("X-Channel-Id" -> channel) - val response: HTTPResponse = httpUtil.patch(updateUrl, request, headers) - if (response.status == 200) { - metrics.incCounter(config.successEventCount) - true - } else if (config.updateAPIErrorCodeList.contains(response.status.toString)) { - logger.error("Skipping The Event As Content Update API Response is: " + response.status + " For sourcing id: " + sourcingId) - metrics.incCounter(config.skippedEventCount) - true - } else { - logger.error("Error while updating metrics for content : " + sourcingId + " :: " + response.body) - throw new InvalidEventException("Error while updating metrics for content : " + sourcingId + " :: " + response.body) - } - } - -} diff --git a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/task/MetricsDataTransformerConfig.scala b/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/task/MetricsDataTransformerConfig.scala deleted file mode 100644 index 60ae4e32a..000000000 --- a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/task/MetricsDataTransformerConfig.scala +++ /dev/null @@ -1,34 +0,0 @@ -package org.sunbird.job.metricstransformer.task - -import com.typesafe.config.Config -import org.sunbird.job.BaseJobConfig -import java.util.{List => JList} - -class MetricsDataTransformerConfig(override val config: Config) extends BaseJobConfig(config, "metrics-data-transformer") { - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val skippedEventCount = "skipped-events-count" - - val contentServiceBaseUrl : String = config.getString("service.content.basePath") - val contentReadApi : String = config.getString("content_read_api") - - val lpURL: String = config.getString("service.sourcing.content.basePath") - val contentUpdate = config.getString("content_update_api") - - val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") - val updateAPIErrorCodeList = if(config.hasPath("sourcing.update.api.response.error.code")) config.getStringList("sourcing.update.api.response.error.code") else List("404").asInstanceOf[java.util.List[String]] - - // Consumers - val eventConsumer = "metrics-data-transformer-consumer" - val metricsDataTransformerFunction = "metrics-data-transformer-function" - - val metrics: JList[String] = config.getStringList("data.metrics") -} diff --git a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/task/MetricsDataTransformerStreamTask.scala b/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/task/MetricsDataTransformerStreamTask.scala deleted file mode 100644 index b3a90f2fe..000000000 --- a/metrics-data-transformer/src/main/scala/org/sunbird/job/metricstransformer/task/MetricsDataTransformerStreamTask.scala +++ /dev/null @@ -1,58 +0,0 @@ -package org.sunbird.job.metricstransformer.task - -import java.io.File -import java.util -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.functions.KeySelector -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.metricstransformer.domain.Event -import org.sunbird.job.metricstransformer.function.MetricsDataTransformerFunction -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.util.{ElasticSearchUtil, FlinkUtil, HttpUtil} - -class MetricsDataTransformerStreamTask(config: MetricsDataTransformerConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, Any]] = TypeExtractor.getForClass(classOf[util.Map[String, Any]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.eventConsumer) - .uid(config.eventConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .keyBy(new MetricsDataTransformerKeySelector) - .process(new MetricsDataTransformerFunction(config, httpUtil)) - .name(config.metricsDataTransformerFunction) - .uid(config.metricsDataTransformerFunction) - .setParallelism(config.parallelism) - - env.execute(config.jobName) - } -} - - // $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster - object MetricsDataTransformerStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("metrics-data-transformer.conf").withFallback(ConfigFactory.systemEnvironment())) - val metricsDataTransformerConfig = new MetricsDataTransformerConfig(config) - val kafkaUtil = new FlinkKafkaConnector(metricsDataTransformerConfig) - val httpUtil = new HttpUtil() - val task = new MetricsDataTransformerStreamTask(metricsDataTransformerConfig, kafkaUtil, httpUtil) - task.process() - } - -} - -// $COVERAGE-ON$ - -class MetricsDataTransformerKeySelector extends KeySelector[Event, String] { - override def getKey(in: Event): String = in.nodeUniqueId.replace(".img", "") -} \ No newline at end of file diff --git a/metrics-data-transformer/src/test/resources/test.conf b/metrics-data-transformer/src/test/resources/test.conf deleted file mode 100644 index ab433eb2d..000000000 --- a/metrics-data-transformer/src/test/resources/test.conf +++ /dev/null @@ -1,26 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - groupId = "sunbirddev-metrics-data-transformer-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - window.time = 60 -} - -timezone = "IST" - -service { - content.basePath = "https://localhost:9000/action" - sourcing.content.basePath = "http://localhost/content" -} - -content_read_api = "/content/v3/read" -content_update_api= "/content/v4/system/update" - -sourcing.update.api.response.error.code = ["404"] - -data.metrics = ["me_totalRatingsCount","me_averageRating","me_totalTimeSpentInSec","me_totalPlaySessionCount"] \ No newline at end of file diff --git a/metrics-data-transformer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/metrics-data-transformer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index f9511698a..000000000 --- a/metrics-data-transformer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,11 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1 = "{\"ets\":1625661905885,\"channel\":\"013085024460783616158023\",\"transactionData\":{\"properties\":{\"copyright\":{\"ov\":null,\"nv\":\"Tamilnadu\"},\"keywords\":{\"ov\":null,\"nv\":[\"AALCPV\"]},\"subject\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"description\":{\"ov\":null,\"nv\":\"B112U4P76CH\"},\"textbook_name\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"showNotification\":{\"ov\":null,\"nv\":true},\"source\":{\"ov\":null,\"nv\":\"https://dock.preprod.ntp.net.in/api/content/v1/read/do_213107906984689664110841\"},\"medium\":{\"ov\":null,\"nv\":[\"English\"]},\"posterImage\":{\"ov\":null,\"nv\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312528175619801088150332/artifact/kona_1529318313157.png\"},\"sourceURL\":{\"ov\":null,\"nv\":\"https://diksha.gov.in/play/content/do_312528171026595840249084\"},\"se_mediums\":{\"ov\":null,\"nv\":[\"English\"]},\"me_totalRatingsCount\":{\"ov\":null,\"nv\":15},\"gradeLevel\":{\"ov\":null,\"nv\":[\"Class 6\"]},\"appIcon\":{\"ov\":null,\"nv\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312528171026595840249084/artifact/kona_1529318313157.thumb.png\"},\"copyrightYear\":{\"ov\":null,\"nv\":2019},\"appId\":{\"ov\":null,\"nv\":\"preprod.dock.portal\"},\"lastUpdatedOn\":{\"ov\":\"2021-07-07T12:45:05.777+0000\",\"nv\":\"2021-07-07T12:45:05.811+0000\"},\"se_gradeLevels\":{\"ov\":null,\"nv\":[\"Class 6\"]},\"creator\":{\"ov\":null,\"nv\":\"TN SCERT SCERT\"},\"createdFor\":{\"ov\":null,\"nv\":[\"013085024460783616158023\"]},\"cloudStorageKey\":{\"ov\":null,\"nv\":\"content/do_213107906984689664110841/artifact/how-to-use-protractor.mp4\"},\"consumerId\":{\"ov\":null,\"nv\":\"89490534-126f-4f0b-82ac-3ff3e49f3468\"},\"author\":{\"ov\":null,\"nv\":\"Tamilnadu\"},\"languageCode\":{\"ov\":null,\"nv\":[\"en\"]},\"se_subjects\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"versionKey\":{\"ov\":\"1625661905777\",\"nv\":\"1625661905811\"},\"createdBy\":{\"ov\":null,\"nv\":\"032b9b66-5faa-4391-8090-ffe3c97a0811\"},\"attributions\":{\"ov\":null,\"nv\":[\"TN SCERT\"]},\"resourceType\":{\"ov\":null,\"nv\":\"Learn\"}}},\"mid\":\"9c58ad31-fd6f-486b-9415-f54477e7f83f\",\"label\":\"how to use protractor\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2021-07-07T12:45:05.885+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11309778227546521611171\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":1128433,\"graphId\":\"domain\"}" - - val EVENT_2 = "{\"ets\":1625661905885,\"channel\":\"013085024460783616158023\",\"transactionData\":{\"properties\":{\"copyright\":{\"ov\":null,\"nv\":\"Tamilnadu\"},\"keywords\":{\"ov\":null,\"nv\":[\"AALCPV\"]},\"subject\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"description\":{\"ov\":null,\"nv\":\"B112U4P76CH\"},\"textbook_name\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"showNotification\":{\"ov\":null,\"nv\":true},\"source\":{\"ov\":null,\"nv\":\"https://dock.preprod.ntp.net.in/api/content/v1/read/do_213107906984689664110841\"},\"medium\":{\"ov\":null,\"nv\":[\"English\"]},\"posterImage\":{\"ov\":null,\"nv\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312528175619801088150332/artifact/kona_1529318313157.png\"},\"sourceURL\":{\"ov\":null,\"nv\":\"https://diksha.gov.in/play/content/do_312528171026595840249084\"},\"se_mediums\":{\"ov\":null,\"nv\":[\"English\"]},\"me_totalRatingsCount\":{\"ov\":null,\"nv\":15},\"gradeLevel\":{\"ov\":null,\"nv\":[\"Class 6\"]},\"appIcon\":{\"ov\":null,\"nv\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312528171026595840249084/artifact/kona_1529318313157.thumb.png\"},\"copyrightYear\":{\"ov\":null,\"nv\":2019},\"appId\":{\"ov\":null,\"nv\":\"preprod.dock.portal\"},\"lastUpdatedOn\":{\"ov\":\"2021-07-07T12:45:05.777+0000\",\"nv\":\"2021-07-07T12:45:05.811+0000\"},\"se_gradeLevels\":{\"ov\":null,\"nv\":[\"Class 6\"]},\"creator\":{\"ov\":null,\"nv\":\"TN SCERT SCERT\"},\"createdFor\":{\"ov\":null,\"nv\":[\"013085024460783616158023\"]},\"cloudStorageKey\":{\"ov\":null,\"nv\":\"content/do_213107906984689664110841/artifact/how-to-use-protractor.mp4\"},\"consumerId\":{\"ov\":null,\"nv\":\"89490534-126f-4f0b-82ac-3ff3e49f3468\"},\"author\":{\"ov\":null,\"nv\":\"Tamilnadu\"},\"languageCode\":{\"ov\":null,\"nv\":[\"en\"]},\"se_subjects\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"versionKey\":{\"ov\":\"1625661905777\",\"nv\":\"1625661905811\"},\"createdBy\":{\"ov\":null,\"nv\":\"032b9b66-5faa-4391-8090-ffe3c97a0811\"},\"attributions\":{\"ov\":null,\"nv\":[\"TN SCERT\"]},\"resourceType\":{\"ov\":null,\"nv\":\"Learn\"}}},\"mid\":\"9c58ad31-fd6f-486b-9415-f54477e7f83f\",\"label\":\"how to use protractor\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2021-07-07T12:45:05.885+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_113097782275465216111711\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":1128433,\"graphId\":\"domain\"}" - - val SKIP_EVENT = "{\"ets\":1625661905885,\"channel\":\"013085024460783616158023\",\"transactionData\":{\"properties\":{\"copyright\":{\"ov\":null,\"nv\":\"Tamilnadu\"},\"keywords\":{\"ov\":null,\"nv\":[\"AALCPV\"]},\"subject\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"description\":{\"ov\":null,\"nv\":\"B112U4P76CH\"},\"textbook_name\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"showNotification\":{\"ov\":null,\"nv\":true},\"source\":{\"ov\":null,\"nv\":\"https://dock.preprod.ntp.net.in/api/content/v1/read/do_213107906984689664110841\"},\"medium\":{\"ov\":null,\"nv\":[\"English\"]},\"posterImage\":{\"ov\":null,\"nv\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312528175619801088150332/artifact/kona_1529318313157.png\"},\"sourceURL\":{\"ov\":null,\"nv\":\"https://diksha.gov.in/play/content/do_312528171026595840249084\"},\"se_mediums\":{\"ov\":null,\"nv\":[\"English\"]},\"ratings\":{\"ov\":null,\"nv\":15},\"gradeLevel\":{\"ov\":null,\"nv\":[\"Class 6\"]},\"appIcon\":{\"ov\":null,\"nv\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312528171026595840249084/artifact/kona_1529318313157.thumb.png\"},\"copyrightYear\":{\"ov\":null,\"nv\":2019},\"appId\":{\"ov\":null,\"nv\":\"preprod.dock.portal\"},\"lastUpdatedOn\":{\"ov\":\"2021-07-07T12:45:05.777+0000\",\"nv\":\"2021-07-07T12:45:05.811+0000\"},\"se_gradeLevels\":{\"ov\":null,\"nv\":[\"Class 6\"]},\"creator\":{\"ov\":null,\"nv\":\"TN SCERT SCERT\"},\"createdFor\":{\"ov\":null,\"nv\":[\"013085024460783616158023\"]},\"cloudStorageKey\":{\"ov\":null,\"nv\":\"content/do_213107906984689664110841/artifact/how-to-use-protractor.mp4\"},\"consumerId\":{\"ov\":null,\"nv\":\"89490534-126f-4f0b-82ac-3ff3e49f3468\"},\"author\":{\"ov\":null,\"nv\":\"Tamilnadu\"},\"languageCode\":{\"ov\":null,\"nv\":[\"en\"]},\"se_subjects\":{\"ov\":null,\"nv\":[\"Mathematics\"]},\"versionKey\":{\"ov\":\"1625661905777\",\"nv\":\"1625661905811\"},\"createdBy\":{\"ov\":null,\"nv\":\"032b9b66-5faa-4391-8090-ffe3c97a0811\"},\"attributions\":{\"ov\":null,\"nv\":[\"TN SCERT\"]},\"resourceType\":{\"ov\":null,\"nv\":\"Learn\"}}},\"mid\":\"9c58ad31-fd6f-486b-9415-f54477e7f83f\",\"label\":\"how to use protractor\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2021-07-07T12:45:05.885+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11309778227546521611171\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":1128433,\"graphId\":\"domain\"}" - -} diff --git a/metrics-data-transformer/src/test/scala/org/sunbird/job/spec/MetricsDataTransformerTaskTestSpec.scala b/metrics-data-transformer/src/test/scala/org/sunbird/job/spec/MetricsDataTransformerTaskTestSpec.scala deleted file mode 100644 index cabcce510..000000000 --- a/metrics-data-transformer/src/test/scala/org/sunbird/job/spec/MetricsDataTransformerTaskTestSpec.scala +++ /dev/null @@ -1,71 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import okhttp3.mockwebserver.{MockResponse, MockWebServer} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.{ArgumentMatchers, Mockito} -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.metricstransformer.domain.Event -import org.sunbird.job.metricstransformer.task.{MetricsDataTransformerConfig, MetricsDataTransformerStreamTask} -import org.sunbird.job.util.{HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.util - -class MetricsDataTransformerTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig: MetricsDataTransformerConfig = new MetricsDataTransformerConfig(config) - val httpUtil: HttpUtil = new HttpUtil() - val server = new MockWebServer() - - var currentMilliSecond = 1605816926271L - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - flinkCluster.before() - super.beforeAll() - } - - "MetricsDataTransformerTaskTestSpec" should "generate event" in { - server.start(9200) - server.enqueue(new MockResponse().setHeader( - "Content-Type", "application/json" - ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new MetricsDataTransformerMapSource) - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - - new MetricsDataTransformerStreamTask(jobConfig, mockKafkaUtil, httpUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - server.close() - } - -} - -class MetricsDataTransformerMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - // Event to be skipped - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.SKIP_EVENT), 0, 10)) - } - - override def cancel(): Unit = {} -} diff --git a/metrics-data-transformer/src/test/scala/org/sunbird/job/spec/service/MetricsDataTransformerServiceTestSpec.scala b/metrics-data-transformer/src/test/scala/org/sunbird/job/spec/service/MetricsDataTransformerServiceTestSpec.scala deleted file mode 100644 index 2eb0e9289..000000000 --- a/metrics-data-transformer/src/test/scala/org/sunbird/job/spec/service/MetricsDataTransformerServiceTestSpec.scala +++ /dev/null @@ -1,199 +0,0 @@ -package org.sunbird.job.spec.service - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.client.JobExecutionException -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.{ArgumentMatchers, Mockito} -import org.mockito.Mockito.when -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.util.{ElasticSearchUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} -import org.sunbird.job.metricstransformer.domain.Event -import org.sunbird.job.metricstransformer.function.MetricsDataTransformerFunction -import org.sunbird.job.metricstransformer.task.MetricsDataTransformerConfig -import java.util -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicLong - -import org.sunbird.job.Metrics - -class MetricsDataTransformerServiceTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, Any]] = TypeExtractor.getForClass(classOf[util.Map[String, Any]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - val config: Config = ConfigFactory.load("test.conf") - lazy val jobConfig: MetricsDataTransformerConfig = new MetricsDataTransformerConfig(config) - val mockElasticUtil:ElasticSearchUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - lazy val metricsDataTransformer: MetricsDataTransformerFunction = new MetricsDataTransformerFunction(jobConfig, new HttpUtil()) - - override protected def beforeAll(): Unit = { - super.beforeAll() - BaseMetricsReporter.gaugeMetrics.clear() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "MetricsDataTransformerService" should "execute for valid event" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("success-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - when(mockHttpUtil.get(ArgumentMatchers.contains("https://localhost:9000/action/content/v3/read/do_11309778227546521611171"), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_123","origin":"do_119831368202241", "programId": "program123", "originData": {"identifier": "do_2133606182319800321377","repository": "https://dock.preprod.ntp.net.in/api/content/v1/read/do_2133606182319800321377"},"mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - when(mockHttpUtil.get(ArgumentMatchers.contains("http://localhost/content/content/v3/read/do_119831368202241"), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_11983136820224112","mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - when(mockHttpUtil.patch(anyString(), any(), any())).thenReturn(HTTPResponse(200, """{"id":"api.content.hierarchy.get","ver":"3.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"rootId":"do_123"}}}""")) - - implicit val config = jobConfig - try { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(0) - } - } - - "MetricsDataTransformerService" should "throw an exception if updating fails" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("failed-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - when(mockHttpUtil.get(anyString(), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_123","origin":"do_119831368202241", "programId": "Program124","originData": {"identifier": "do_2133606182319800321377","repository": "https://dock.preprod.ntp.net.in/api/content/v1/read/do_2133606182319800321377"},"mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - implicit val config = jobConfig - the [Exception] thrownBy { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } should have message "Error writing metrics for sourcing id: do_119831368202241" - - when(mockHttpUtil.patch(anyString(), any(), any())).thenReturn(HTTPResponse(404, """{"id":"api.content.hierarchy.get","ver":"3.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{}}}""")) - the [Exception] thrownBy { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } should have message "Error writing metrics for sourcing id: do_119831368202241" - - } - - "MetricsDataTransformerService" should "skip event if response from content read not found" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("skipped-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - when(mockHttpUtil.get(ArgumentMatchers.contains("https://localhost:9000/action/content/v3/read/do_11309778227546521611171"), any())).thenReturn(HTTPResponse(404, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{}}}""")) - - implicit val config = jobConfig - try { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - } - } - - "MetricsDataTransformerService" should "throw an exception if content read fails" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("failed-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - when(mockHttpUtil.get(ArgumentMatchers.contains("https://localhost:9000/action/content/v3/read/do_11309778227546521611171"), any())).thenReturn(HTTPResponse(500, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"ERROR","errmsg":null},"responseCode":"CLIENT_ERROR","result":{"content":{}}}""")) - implicit val config = jobConfig - the [Exception] thrownBy { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } should have message "Error while getting content data from read API for :: do_113097782275465216111711" - } - - "MetricsDataTransformerService" should "skip event if sourcing id not present in content read response" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("skipped-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - when(mockHttpUtil.get(ArgumentMatchers.contains("https://localhost:9000/action/content/v3/read/do_11309778227546521611171"), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_123","mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - - implicit val config = jobConfig - try { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - } - } - - it should "throw an exception if consumption read API fails" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("failed-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - - when(mockHttpUtil.get(ArgumentMatchers.contains("https://localhost:9000/action/content/v3/read/do_11309778227546521611171"), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_123","origin":"do_119831368202241", "programId": "program123", "originData": {"identifier": "do_2133606182319800321377","repository": "https://dock.preprod.ntp.net.in/api/content/v1/read/do_2133606182319800321377"},"mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - when(mockHttpUtil.get(ArgumentMatchers.contains("http://localhost/content/content/v3/read/do_119831368202241"), any())).thenReturn(HTTPResponse(500, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"ERROR","errmsg":null},"responseCode":"CLIENT_ERROR","result":{"content":{}}}""")) - - implicit val config = jobConfig - the [Exception] thrownBy { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } should have message "Error while getting content data from sourcing content read API for :: do_119831368202241" - } - - it should "skip event if response from sourcing read API not found" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("skipped-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - - when(mockHttpUtil.get(ArgumentMatchers.contains("https://localhost:9000/action/content/v3/read/do_11309778227546521611171"), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_123","origin":"do_119831368202241", "programId": "program123", "originData": {"identifier": "do_2133606182319800321377","repository": "https://dock.preprod.ntp.net.in/api/content/v1/read/do_2133606182319800321377"},"mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - when(mockHttpUtil.get(ArgumentMatchers.contains("http://localhost/content/content/v3/read/do_119831368202241"), any())).thenReturn(HTTPResponse(404, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{}}}""")) - - implicit val config = jobConfig - try { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - } - } - - it should "skip event if response from sourcing update API not found" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - val map = new ConcurrentHashMap[String, AtomicLong]() - map.put("skipped-events-count", new AtomicLong(0)) - val metrics: Metrics = Metrics(map) - - val metricList = Array("me_totalRatingsCount") - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - - when(mockHttpUtil.get(ArgumentMatchers.contains("https://localhost:9000/action/content/v3/read/do_11309778227546521611171"), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_123","origin":"do_119831368202241", "programId": "program123", "originData": {"identifier": "do_2133606182319800321377","repository": "https://dock.preprod.ntp.net.in/api/content/v1/read/do_2133606182319800321377"},"mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - when(mockHttpUtil.get(ArgumentMatchers.contains("http://localhost/content/content/v3/read/do_119831368202241"), any())).thenReturn(HTTPResponse(200, """{"id":"api.v3.read.get","ver":"1.0","ts":"2020-08-07T15:56:47ZZ","params":{"resmsgid":"bbd98348-c70b-47bc-b9f1-396c5e803436","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"parent":"do_11983136820224112","mediaType":"content","name":"APSWREIS-TGT2-Day5","identifier":"do_234","description":"Enter description for Collection","resourceType":"Collection","mimeType":"application/vnd.ekstep.content-collection","contentType":"Collection","language":["English"],"objectType":"Content","status":"Live","idealScreenSize":"normal","contentEncoding":"gzip","osId":"org.ekstep.quiz.app","contentDisposition":"inline","childNodes":["do_345"],"visibility":"Default","pkgVersion":2,"idealScreenDensity":"hdpi"}}}""")) - when(mockHttpUtil.patch(anyString(), any(), any())).thenReturn(HTTPResponse(404, """{"id":"api.content.system.update","ver":"4.0","ts":"2021-09-16T13:35:24ZZ","params":{"resmsgid":"cc879235-14d7-4815-b5cf-413ec0b0a1a7","msgid":null,"err":"CLIENT_ERROR","status":"failed","errmsg":"Validation Errors"},"responseCode":"CLIENT_ERROR","result":{"messages":["Metadata mimeType should be one of: [application/vnd.ekstep.ecml-archive]"]}}""")) - - implicit val config = jobConfig - try { - metricsDataTransformer.processEvent(inputEvent, metrics, metricList) - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - } - } - -} \ No newline at end of file diff --git a/mvc-indexer/README.md b/mvc-indexer/README.md deleted file mode 100644 index 27f2989f1..000000000 --- a/mvc-indexer/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# MVC Processor Indexer - -MVC Processor Indexer job - -## Getting Started - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. -### Prerequisites - -1. Download flink-1.13.6-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.13.6/flink-1.13.6-bin-scala_2.12.tgz). -2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. -3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. -4. Docker installed. -5. A running yarn cluster or a kubernetes cluster. - -### Build - -mvn clean install - -## Deployment - -### Yarn - -Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. - -``` -./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /mvc-indexer/target/mvc-indexer-0.0.1.jar -``` - -### Kubernetes - -``` -# Create a single node cluster -k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 -# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. -export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" - -# Only for Mac OSX -# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl -rm /usr/local/bin/kubectl -brew link --overwrite kubernetes-cli - -# Create a configmap using the flink-configuration-configmap.yaml -kubectl create -f knowledge-platform-job/kubernetes/flink-configuration-configmap.yaml - -# Create pods for jobmanager-service, job-manager and task-manager using the yaml files -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-service.yaml -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-deployment.yaml -kubectl create -f knowledge-platform-job/kubernetes/taskmanager-deployment.yaml - -# Create a port-forwarding for accessing the job-manager UI on localhost:8081 -kubectl port-forward deployment/flink-jobmanager 8081:8081 - -# Submit the job to the Kubernetes single node cluster flink-cluster -./bin/flink run -m localhost:8081 /mvc-indexer/target/mvc-indexer-0.0.1.jar - -# Commands to delete the pods created in the cluster -kubectl delete deployment/flink-jobmanager -kubectl delete deployment/flink-taskmanager -kubectl delete service/flink-jobmanager -kubectl delete configmaps/flink-config - -# Command to stop the single-node cluster -k3d stop --name="flink-cluster" -``` diff --git a/mvc-indexer/pom.xml b/mvc-indexer/pom.xml deleted file mode 100644 index 0478d2d5d..000000000 --- a/mvc-indexer/pom.xml +++ /dev/null @@ -1,220 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - mvc-indexer - 1.0.0 - jar - mvc-indexer - - MVC Processor indexer Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - com.squareup.okhttp3 - mockwebserver - 4.9.1 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.mvcindexer.task.MVCIndexerStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - mvc-indexer-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/mvc-indexer/src/main/resources/log4j.properties b/mvc-indexer/src/main/resources/log4j.properties deleted file mode 100644 index 5fe79c8d2..000000000 --- a/mvc-indexer/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=mvc-indexer.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/mvc-indexer/src/main/resources/mvc-indexer.conf b/mvc-indexer/src/main/resources/mvc-indexer.conf deleted file mode 100644 index 126f88f3b..000000000 --- a/mvc-indexer/src/main/resources/mvc-indexer.conf +++ /dev/null @@ -1,25 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.mvc.processor.job.request" - groupId = "sunbirddev-mvc-indexer-group" - output.failed.topic = "sunbirddev.mvc.events.failed" -} - -task { - consumer.parallelism = 1 - parallelism = 1 -} - -lms-cassandra.keyspace = "dev_content_store" -lms-cassandra.table = "content_data" - -nested.fields="trackable,credentials" - -service.content.basePath = "http://11.2.6.6/content" -es.indexAlias = "mvc-content" - -ml.vector.host="11.2.4.22" -ml.vector.port=1729 -ml.keyword.host="11.2.4.22" -ml.keyword.port=3579 diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/domain/Event.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/domain/Event.scala deleted file mode 100644 index 11c235b3a..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/domain/Event.scala +++ /dev/null @@ -1,34 +0,0 @@ -package org.sunbird.job.mvcindexer.domain - -import org.apache.commons.lang3.BooleanUtils -import org.sunbird.job.domain.reader.JobRequest - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val jobName = "MVCIndexer" - - def index: AnyRef = readOrDefault("index", null) - - def identifier: String = readOrDefault("object.id", "") - - var eventData: Map[String, AnyRef] = readOrDefault[Map[String, AnyRef]]("eventData", Map[String, AnyRef]()) - - def action: String = readOrDefault("eventData.action", "") - - def mlContentText: String = readOrDefault("eventData.ml_contentText", null) - - def mlKeywords: List[String] = readOrDefault("eventData.ml_Keywords", null) - - def metadata: Map[String, AnyRef] = readOrDefault[Map[String, AnyRef]]("eventData.metadata", Map[String, AnyRef]()) - - def mlContentTextVector: List[Double] = { - val mlContentTextVectorList = readOrDefault[List[List[Double]]]("eventData.ml_contentTextVector", null) - - if (mlContentTextVectorList != null) mlContentTextVectorList.head else null - } - - def isValid: Boolean = { - BooleanUtils.toBoolean(if (null == index) "true" else index.toString) && !eventMap.containsKey("edata") && eventData.nonEmpty - } - -} diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/functions/MVCIndexer.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/functions/MVCIndexer.scala deleted file mode 100644 index f89635616..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/functions/MVCIndexer.scala +++ /dev/null @@ -1,53 +0,0 @@ -package org.sunbird.job.mvcindexer.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.mvcindexer.domain.Event -import org.sunbird.job.mvcindexer.service.MVCIndexerService -import org.sunbird.job.mvcindexer.task.MVCIndexerConfig -import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HttpUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -class MVCIndexer(config: MVCIndexerConfig, var esUtil: ElasticSearchUtil, httpUtil: HttpUtil) - (implicit eventTypeInfo: TypeInformation[Event], - @transient var cassandraUtil: CassandraUtil = null) - extends BaseProcessFunction[Event, String](config) { - - var mvcIndexerService: MVCIndexerService = _ - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.esFailedEventCount, config.skippedEventCount, config.dbUpdateCount, config.dbUpdateFailedCount, config.apiFailedEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - if (esUtil == null) { - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.mvcProcessorIndex, config.mvcProcessorIndexType) - } - cassandraUtil = new CassandraUtil(config.lmsDbHost, config.lmsDbPort, config) - mvcIndexerService = new MVCIndexerService(config, esUtil, httpUtil, cassandraUtil) - } - - override def close(): Unit = { - esUtil.close() - mvcIndexerService.closeConnection() - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - try { - metrics.incCounter(config.totalEventsCount) - if (event.isValid) mvcIndexerService.processMessage(event)(metrics) - else metrics.incCounter(config.skippedEventCount) - } catch { - case ex: Exception => - metrics.incCounter(config.failedEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } -} diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/service/MVCIndexerService.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/service/MVCIndexerService.scala deleted file mode 100644 index 3077eb101..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/service/MVCIndexerService.scala +++ /dev/null @@ -1,55 +0,0 @@ -package org.sunbird.job.mvcindexer.service - -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.exception.{APIException, CassandraException, ElasticSearchException} -import org.sunbird.job.mvcindexer.domain.Event -import org.sunbird.job.mvcindexer.task.MVCIndexerConfig -import org.sunbird.job.mvcindexer.util.{ContentUtil, MVCCassandraIndexer, MVCESIndexer} -import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HttpUtil} - - -class MVCIndexerService(config: MVCIndexerConfig, esUtil: ElasticSearchUtil, httpUtil: HttpUtil, cassandraUtil: CassandraUtil) { - - private[this] lazy val logger = LoggerFactory.getLogger(classOf[MVCIndexerService]) - private val mvcEsIndexer: MVCESIndexer = Option(mvcEsIndexer).getOrElse(new MVCESIndexer(config, esUtil)) - mvcEsIndexer.createMVCSearchIndex() - private val cassandraManager: MVCCassandraIndexer = new MVCCassandraIndexer(config, cassandraUtil, httpUtil) - - @throws[Exception] - def processMessage(message: Event)(implicit metrics: Metrics): Unit = { - try { - val objectId: String = message.identifier - if (config.csUpdateAcions.contains(message.action)) { - if (message.action.equalsIgnoreCase(config.esIndexAction)) { - message.eventData = ContentUtil.getContentMetaData(message.eventData, objectId, httpUtil, config) - } - logger.info("Upsert cassandra document for " + objectId) - cassandraManager.insertIntoCassandra(message, objectId) - metrics.incCounter(config.dbUpdateCount) - } - logger.info("Elasticsearch document updation for " + objectId) - mvcEsIndexer.upsertDocument(objectId, message) - - metrics.incCounter(config.successEventCount) - } catch { - case ex: APIException => { - metrics.incCounter(config.apiFailedEventCount) - throw ex - } - case ex: CassandraException => { - metrics.incCounter(config.dbUpdateFailedCount) - throw ex - } - case ex: ElasticSearchException => { - metrics.incCounter(config.esFailedEventCount) - throw ex - } - } - } - - def closeConnection(): Unit = { - cassandraUtil.close() - } - -} diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/task/MVCIndexerConfig.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/task/MVCIndexerConfig.scala deleted file mode 100644 index f6808b7ad..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/task/MVCIndexerConfig.scala +++ /dev/null @@ -1,66 +0,0 @@ -package org.sunbird.job.mvcindexer.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.sunbird.job.BaseJobConfig - -class MVCIndexerConfig(override val config: Config) extends BaseJobConfig(config, "mvc-indexer") { - - private val serialVersionUID = 2905979434303791379L - - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - val kafkaFailedTopic: String = config.getString("kafka.output.failed.topic") - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val esFailedEventCount = "elasticsearch-failed-events-count" - val skippedEventCount = "skipped-events-count" - val dbUpdateCount = "db-update-count" - val dbUpdateFailedCount = "db-update-failed-count" - val apiFailedEventCount = "api-failed-events-count" - - // Consumers - val eventConsumer = "mvc-indexer-consumer" - val mvcIndexerFunction = "mvc-indexer-function" - - // ES Configs - val esConnectionInfo: String = config.getString("es.basePath") - val nestedFields: Array[String] = config.getString("nested.fields").split(",") - val mvcProcessorIndex = "mvc-content-v1" - val mvcProcessorIndexType = "_doc" - val operationCreate = "CREATE" - val indexAlias: String = config.getString("es.indexAlias") - val indexSettings = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1500"}},"number_of_shards":"5","analysis":{"filter":{"mynGram":{"token_chars":["letter","digit","whitespace","punctuation","symbol"],"min_gram":"1","type":"nGram","max_gram":"30"}},"analyzer":{"cs_index_analyzer":{"filter":["lowercase","mynGram"],"type":"custom","tokenizer":"standard"},"keylower":{"filter":"lowercase","tokenizer":"keyword"},"ml_custom_analyzer":{"type":"standard","stopwords":["_english_","_hindi_"]},"cs_search_analyzer":{"filter":["lowercase"],"type":"custom","tokenizer":"standard"}}},"number_of_replicas":"1"}""" - val indexMappings = """{"dynamic":"strict","properties":{"all_fields":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"allowedContentTypes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"appIcon":{"type":"text","index":false},"appIconLabel":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"appId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"artifactUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"board":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"channel":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentEncoding":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"description":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"downloadUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"framework":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"gradeLevel":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"identifier":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"label":{"type":"text","analyzer":"cs_index_analyzer","search_analyzer":"standard"},"language":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lastUpdatedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"launchUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"level1Concept":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"level1Name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"level2Concept":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"level2Name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"level3Concept":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"level3Name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"medium":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"mimeType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"ml_Keywords":{"type":"text","analyzer":"ml_custom_analyzer","search_analyzer":"standard"},"ml_contentText":{"type":"text","analyzer":"ml_custom_analyzer","search_analyzer":"standard"},"ml_contentTextVector":{"type":"long"},"name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"nodeType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"node_id":{"type":"long","fields":{"raw":{"type":"long"}}},"objectType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"organisation":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"pkgVersion":{"type":"double","fields":{"raw":{"type":"double"}}},"posterImage":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"previewUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"resourceType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"source":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"sourceURL":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"status":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"streamingUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"subject":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"textbook_name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}}""" - val elasticSearchParamSet = Set("organisation", "channel", "framework", "board", "medium", "subject", "gradeLevel", "name", "description", "language", "appId", "appIcon", "appIconLabel", "contentEncoding", "identifier", "node_id", "nodeType", "mimeType", "resourceType", "contentType", "allowedContentTypes", "objectType", "posterImage", "artifactUrl", "launchUrl", "previewUrl", "streamingUrl", "downloadUrl", "status", "pkgVersion", "source", "lastUpdatedOn", "ml_contentText", "ml_contentTextVector", "ml_Keywords", "level1Name", "level1Concept", "level2Name", "level2Concept", "level3Name", "level3Concept", "textbook_name", "sourceURL", "label", "all_fields") - - // API Configs - val contentServiceBase: String = config.getString("service.content.basePath") - val contentReadURL = s"${contentServiceBase}/content/v3/read/" - val mlVectorAPIUrl = s"http://${config.getString("ml.vector.host")}:${config.getString("ml.vector.port")}/ml/vector/ContentText" - val mlKeywordAPIUrl = s"http://${config.getString("ml.keyword.host")}:${config.getString("ml.keyword.port")}/daggit/submit" - val keywordAPIJobname = "vidyadaan_content_keyword_tagging" - - // Cassandra Configurations - val dbTable: String = config.getString("lms-cassandra.table") - val dbKeyspace: String = config.getString("lms-cassandra.keyspace") - val csFieldMap: Map[String, String] = Map[String, String]("level1Concept" -> "level1_concept", "level2Concept" -> "level2_concept", - "level3Concept" -> "level3_concept", "textbook_name" -> "textbook_name", "level1Name" -> "level1_name", - "level2Name" -> "level2_name", "level3Name" -> "level3_name") - - - // Actions - val esIndexAction = "update-es-index" - val contentRatingAction = "update-content-rating" - val mlVectorAction = "update-ml-contenttextvector" - val mlKeywordAction = "update-ml-keywords" - val csUpdateAcions: Array[String] = Array(esIndexAction, mlVectorAction, mlKeywordAction) -} diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/task/MVCIndexerStreamTask.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/task/MVCIndexerStreamTask.scala deleted file mode 100644 index df3f60229..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/task/MVCIndexerStreamTask.scala +++ /dev/null @@ -1,51 +0,0 @@ -package org.sunbird.job.mvcindexer.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.mvcindexer.domain.Event -import org.sunbird.job.mvcindexer.functions.MVCIndexer -import org.sunbird.job.util.{ElasticSearchUtil, FlinkUtil, HttpUtil} - -import java.io.File - - -class MVCIndexerStreamTask(config: MVCIndexerConfig, kafkaConnector: FlinkKafkaConnector, esUtil: ElasticSearchUtil, httpUtil: HttpUtil) { - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.eventConsumer) - .uid(config.eventConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new MVCIndexer(config, esUtil, httpUtil)) - .name(config.mvcIndexerFunction) - .uid(config.mvcIndexerFunction) - .setParallelism(config.parallelism) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object MVCIndexerStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("mvc-indexer.conf").withFallback(ConfigFactory.systemEnvironment())) - val MVCIndexerConfig = new MVCIndexerConfig(config) - val kafkaUtil = new FlinkKafkaConnector(MVCIndexerConfig) - val esUtil: ElasticSearchUtil = null - val httpUtil = new HttpUtil - val task = new MVCIndexerStreamTask(MVCIndexerConfig, kafkaUtil, esUtil, httpUtil) - task.process() - } -} - -// $COVERAGE-ON$ diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/ContentUtil.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/ContentUtil.scala deleted file mode 100644 index 5ec99144b..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/ContentUtil.scala +++ /dev/null @@ -1,42 +0,0 @@ -package org.sunbird.job.mvcindexer.util - -import org.sunbird.job.exception.APIException -import org.sunbird.job.mvcindexer.task.MVCIndexerConfig -import org.sunbird.job.util.{HTTPResponse, HttpUtil, JSONUtil} - -object ContentUtil { - - /** - * Get the Content Metadata and return the parsed metadata map - * - * @param newmap Content metadata from event envelope - * @param identifer Content ID - * @param httpUtil HttpUil instance - * @param config Config instance - * @return parsed metadata map - */ - @throws[Exception] - def getContentMetaData(newmap: Map[String, AnyRef], identifer: String, httpUtil: HttpUtil, config: MVCIndexerConfig): Map[String, AnyRef] = { - try { - val content: HTTPResponse = httpUtil.get(config.contentReadURL + identifer) - val obj = JSONUtil.deserialize[Map[String, AnyRef]](content.body) - val contentObj = obj("result").asInstanceOf[Map[String, AnyRef]]("content").asInstanceOf[Map[String, AnyRef]] - filterData(newmap, contentObj, config) - } catch { - case e: Exception => - throw new APIException(s"Error in getContentMetaData for $identifer - ${e.getLocalizedMessage}", e) - } - } - - def filterData(obj: Map[String, AnyRef], content: Map[String, AnyRef], config: MVCIndexerConfig): Map[String, AnyRef] = { - var contentObj = JSONUtil.deserialize[Map[String, AnyRef]](JSONUtil.serialize(obj)) - - for (param <- config.elasticSearchParamSet) { - val value = content.getOrElse(param, null) - if (value != null) { - contentObj += (param -> value) - } - } - contentObj - } -} diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/MVCCassandraIndexer.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/MVCCassandraIndexer.scala deleted file mode 100644 index 1bb7fdbe2..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/MVCCassandraIndexer.scala +++ /dev/null @@ -1,171 +0,0 @@ -package org.sunbird.job.mvcindexer.util - -import com.datastax.driver.core.querybuilder.Update.Assignments -import com.datastax.driver.core.querybuilder.{QueryBuilder, Update} -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.{APIException, CassandraException} -import org.sunbird.job.mvcindexer.domain.Event -import org.sunbird.job.mvcindexer.task.MVCIndexerConfig -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.apache.commons.lang3.StringUtils - -class MVCCassandraIndexer(config: MVCIndexerConfig, cassandraUtil: CassandraUtil, httpUtil: HttpUtil) { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[MVCCassandraIndexer]) - - /** - * Based on action update content metadata in cassandra and post the keywords and vector data to ML Service - * - * @param message Event envelope - * @param identifier Content ID - */ - def insertIntoCassandra(message: Event, identifier: String): Unit = { - val obj: Map[String, AnyRef] = message.eventData - message.action match { - case config.esIndexAction => - upsertWithEsIndex(obj, identifier) - case config.mlKeywordAction => - updateWithMLKeyword(message, identifier) - case config.mlVectorAction => - updateWithMLVector(message, identifier) - } - } - - def upsertWithEsIndex(obj: Map[String, AnyRef], identifier: String) = { - val esCassandraMap = extractFieldsToBeInserted(obj) - getMLKeywords(obj) - updateContentProperties(identifier, esCassandraMap) - } - - def updateWithMLKeyword(message: Event, identifier: String) = { - getMLVectors(message.mlContentText, identifier) - val csTableCols = Map[String, AnyRef]("ml_keywords" -> message.mlKeywords, "ml_content_text" -> message.mlContentText) - updateContentProperties(identifier, csTableCols) - } - - def updateWithMLVector(message: Event, identifier: String) = { - val vectorSet = JSONUtil.deserialize[java.util.HashSet[java.lang.Double]](JSONUtil.serialize(message.mlContentTextVector)) - val csTableCols = Map[String, AnyRef]("ml_content_text_vector" -> vectorSet) - updateContentProperties(identifier, csTableCols) - } - - /** - * Parse the selected fields from eventData to update in cassandra - * - * @param contentobj Content metadata - */ - private def extractFieldsToBeInserted(contentobj: Map[String, AnyRef]): Map[String, AnyRef] = { - var esCassandraMap = Map[String, AnyRef]() - for ((fieldKey: String, fieldValue: String) <- config.csFieldMap) { - if (contentobj.contains(fieldKey)) { - esCassandraMap += (fieldValue -> contentobj(fieldKey).asInstanceOf[List[String]]) - } - } - - if (contentobj.contains("source") && contentobj("source").isInstanceOf[List[String]]) { - esCassandraMap += ("source" -> contentobj("source").asInstanceOf[List[String]].head) - } - else if (contentobj.contains("source") && contentobj("source").isInstanceOf[String]) { - esCassandraMap += ("source" -> contentobj("source")) - } - if (contentobj.contains("sourceURL")) esCassandraMap += ("sourceurl" -> contentobj("sourceURL")) - - esCassandraMap - } - - /** - * Post the keywords to ML Workbench service - * - * @param contentdef Content Metadata - */ - @throws[APIException] - private def getMLKeywords(contentdef: Map[String, AnyRef]): Unit = { - val bodyObj = Map("request" -> Map("job" -> config.keywordAPIJobname, "input" -> Map("content" -> List(contentdef)))) - - val requestBody = JSONUtil.serialize(bodyObj) - try { - val resp: HTTPResponse = httpUtil.post(config.mlKeywordAPIUrl, requestBody) - logger.info("ML keyword api response - " + resp.body) - if (!resp.isSuccess) throw new Exception("") - } catch { - case e: Exception => - throw new APIException(s"ML keyword api request failed - ${e.getMessage}", e) - } - } - - /** - * Post the content text vector to ML Workbench service - * - * @param contentText ContentText from event envelope - * @param identifier Content ID - */ - @throws[APIException] - def getMLVectors(contentText: String, identifier: String): Unit = { - val bodyObj = Map("request" -> Map("language" -> "en", "method" -> "BERT", "params" -> Map("dim" -> 768, "seq_len" -> 25), "cid" -> identifier, "text" -> List(contentText))) - - val requestBody = JSONUtil.serialize(bodyObj) - try { - val resp: HTTPResponse = httpUtil.post(config.mlVectorAPIUrl, requestBody) - logger.info("ML vector api response - " + resp.body) - if (!resp.isSuccess) throw new Exception("") - } catch { - case e: Exception => - throw new APIException(s"ML vector api failed for $identifier - ${e.getMessage}", e) - } - } - - /** - * Update cassandra record - * - * @param contentId Content ID - * @param map Content object - */ - @throws[CassandraException] - def updateContentProperties(contentId: String, map: Map[String, AnyRef]): Unit = { - try { - val updateQuery = constructUpdateQuery(contentId, map) - if (StringUtils.isNotBlank(updateQuery)) cassandraUtil.session.execute(updateQuery) - } catch { - case e: Exception => - throw new CassandraException(s"Exception while inserting data into cassandra for $contentId - ${e.getMessage}", e) - } - } - - @throws[CassandraException] - def constructUpdateQuery(contentId: String, map: Map[String, AnyRef]): String = { - if (Option(map).forall(_.isEmpty)) return "" - import scala.collection.JavaConverters._ - - try { - val query: Update = QueryBuilder.update(config.dbKeyspace, config.dbTable) - var queryAssignments: Assignments = null - var i = 0 - for ((key, value) <- map.toList) { - if (null != value && null != key) { - val querySet = value match { - case strings: List[String] => - QueryBuilder.set(key, strings.asJava) - case _ => - QueryBuilder.set(key, value) - } - - queryAssignments = if (i == 0) { - query.`with`(querySet) - } else { - queryAssignments.and(querySet) - } - } else { - return "" - } - i += 1 - } - - queryAssignments.and(QueryBuilder.set("last_updated_on", System.currentTimeMillis)) - val finalQuery = queryAssignments.where(QueryBuilder.eq("content_id", contentId)) - - finalQuery.toString - } catch { - case e: Exception => - throw new CassandraException(s"Exception while constructing query to cassandra for $contentId - ${e.getMessage}", e) - } - } -} diff --git a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/MVCESIndexer.scala b/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/MVCESIndexer.scala deleted file mode 100644 index fd7f19b39..000000000 --- a/mvc-indexer/src/main/scala/org/sunbird/job/mvcindexer/util/MVCESIndexer.scala +++ /dev/null @@ -1,99 +0,0 @@ -package org.sunbird.job.mvcindexer.util - -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.ElasticSearchException -import org.sunbird.job.mvcindexer.domain.Event -import org.sunbird.job.mvcindexer.task.MVCIndexerConfig -import org.sunbird.job.util.{ElasticSearchUtil, JSONUtil} - -import java.io.IOException - - -class MVCESIndexer(config: MVCIndexerConfig, esUtil: ElasticSearchUtil) { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[MVCESIndexer]) - private val NESTED_FIELDS = config.nestedFields - - /** - * Create mvc index in Elasticsearch if not available - */ - @throws[ElasticSearchException] - def createMVCSearchIndex(): Unit = { - try { - esUtil.addIndex(config.indexSettings, config.indexMappings, config.indexAlias) - } catch { - case e: Exception => - throw new ElasticSearchException(s"Exception while creating index into in Elasticsearch - ${e.getMessage}", e) - } - - } - - /** - * Insert or update the document in ES based on the action. content id - * - * @param uniqueId Content ID - * @param message Event envelope - */ - @throws[ElasticSearchException] - def upsertDocument(uniqueId: String, message: Event): Unit = { - try { - var jsonIndexDocument: Map[String, AnyRef] = removeExtraParams(message.eventData) - jsonIndexDocument ++= processNestedProps(jsonIndexDocument) - val jsonAsString = JSONUtil.serialize(jsonIndexDocument) - message.action match { - case config.esIndexAction => - esUtil.addDocumentWithIndex(jsonAsString, config.mvcProcessorIndex, uniqueId) - - case config.contentRatingAction => - updateContentRating(uniqueId, message) - - case config.mlVectorAction => - updateContentTextVector(jsonIndexDocument, uniqueId, message) - - case config.mlKeywordAction => - esUtil.updateDocument(uniqueId, jsonAsString) - } - } catch { - case e: Exception => - throw new ElasticSearchException(s"Exception while inserting data into ES for $uniqueId - ${e.getMessage}", e) - } - } - - def updateContentRating(uniqueId: String, message: Event): Unit = { - val resp = esUtil.getDocumentAsString(uniqueId) - if (null != resp && resp.contains(uniqueId)) { - logger.info("Updating ES document with content rating for " + uniqueId) - esUtil.updateDocument(uniqueId, JSONUtil.serialize(message.metadata)) - } - else logger.info("Skipped updating content rating since ES document not found for " + uniqueId) - } - - def updateContentTextVector(jsonIndexDocument: Map[String, AnyRef], uniqueId: String, message: Event): Unit = { - val jsonAsString = JSONUtil.serialize(jsonIndexDocument + ("ml_contentTextVector" -> message.mlContentTextVector)) - esUtil.updateDocument(uniqueId, jsonAsString) - } - - /** - * Updating nested type props as null in ES - * - * @param jsonIndexDocument Content metadata - * @return New map with available nested props as null - */ - @throws[IOException] - private def processNestedProps(jsonIndexDocument: Map[String, AnyRef]): Map[String, AnyRef] = { - var nestedProps = Map[String, AnyRef]() - for (propertyName <- jsonIndexDocument.keySet) { - if (NESTED_FIELDS.contains(propertyName)) { - val propertyNewValue = JSONUtil.deserialize[Map[String, AnyRef]](jsonIndexDocument(propertyName).asInstanceOf[String]) - nestedProps += (propertyName -> propertyNewValue) - } - } - nestedProps - } - - /** - * Remove params which should not be inserted into ES from content metadata - */ - def removeExtraParams(obj: Map[String, AnyRef]): Map[String, AnyRef] = { - obj.-("action", "stage") - } -} diff --git a/mvc-indexer/src/test/resources/logback-test.xml b/mvc-indexer/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/mvc-indexer/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/mvc-indexer/src/test/resources/test.conf b/mvc-indexer/src/test/resources/test.conf deleted file mode 100644 index c9b696765..000000000 --- a/mvc-indexer/src/test/resources/test.conf +++ /dev/null @@ -1,26 +0,0 @@ -include "base-test.conf" - -kafka { - input.topic = "sunbirddev.mvc.processor.job.request" - groupId = "sunbirddev-mvc-indexer-group" - output.failed.topic = "sunbirddev.mvc.events.failed" -} - -task { - consumer.parallelism = 1 - parallelism = 1 -} - -lms-cassandra.keyspace = "local_content_store" -lms-cassandra.table = "content_data" - -nested.fields="trackable,credentials" - -service.content.basePath = "http://localhost:8080/content" -es.indexAlias = "mvc-content" - -ml.vector.host="localhost" -ml.vector.port=1729 -ml.keyword.host="localhost" -ml.keyword.port=3579 - diff --git a/mvc-indexer/src/test/resources/test.cql b/mvc-indexer/src/test/resources/test.cql deleted file mode 100644 index 1e2ad8460..000000000 --- a/mvc-indexer/src/test/resources/test.cql +++ /dev/null @@ -1,39 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS local_content_store WITH replication = { - 'class': 'SimpleStrategy', - 'replication_factor': '1' -}; - -CREATE TABLE IF NOT EXISTS local_content_store.content_data ( - content_id text PRIMARY KEY, - body blob, - externallink text, - last_updated_on timestamp, - level1_concept list, - level1_name list, - level2_concept list, - level2_name list, - level3_concept list, - level3_name list, - ml_content_text text, - ml_content_text_vector frozen>, - ml_keywords list, - oldbody blob, - screenshots blob, - source text, - sourceurl text, - stageicons blob, - textbook_name list -) WITH bloom_filter_fp_chance = 0.01 - AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} - AND comment = '' - AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} - AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} - AND crc_check_chance = 1.0 - AND dclocal_read_repair_chance = 0.1 - AND default_time_to_live = 0 - AND gc_grace_seconds = 864000 - AND max_index_interval = 2048 - AND memtable_flush_period_in_ms = 0 - AND min_index_interval = 128 - AND read_repair_chance = 0.0 - AND speculative_retry = '99PERCENTILE'; \ No newline at end of file diff --git a/mvc-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/mvc-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index 8fc925738..000000000 --- a/mvc-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,30 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"object":{"id":"do_112806963140329472124"},"index":"true","ets":"1591949601174","eventData":{"trackable": "{\"key1\":123}", "action":"update-es-index","stage":1, "identifier":"do_112806963140329472124","channel":"in.ekstep","language":["English"],"name":"Resource Content 1","status":"Draft","level1Concept":["Addition"],"level1Name":["Math-Magic"],"textbook_name":["How Many Times?"],"sourceURL":"https://dev.sunbirded.org/play/content/do_112806963140329472124","source":["Sunbird 1"]}} - |""".stripMargin - - val EVENT_2: String = - """ - |{"object":{"id":"do_112806963140329472124"},"index":"false","ets":"1591949601174","eventData":{"action":"update-es-index","stage":1,"identifier":"do_112806963140329472124","channel":"in.ekstep","language":["English"],"name":"Resource Content 1","status":"Draft","level1Concept":["Addition"],"level1Name":["Math-Magic"],"textbook_name":["How Many Times?"],"sourceURL":"https://dev.sunbirded.org/play/content/do_112806963140329472124"}} - |""".stripMargin - - val EVENT_3: String = - """ - |{"object":{"id":"do_112806963140329472124"},"index":"true","ets":"1591949601174","eventData":{"action":"update-ml-keywords","stage":"2","ml_Keywords":["maths","addition","add"],"ml_contentText":"This is the content text for addition of two numbers."}} - |""".stripMargin - - val EVENT_4: String = - """ - |{"object":{"id":"do_112806963140329472124"},"index":"true","ets":"1591949601174","eventData":{"action":"update-ml-contenttextvector","stage":3,"ml_contentTextVector":[[1.1,2,7.4,68]]}} - |""".stripMargin - - val EVENT_5: String = - """ - |{"object":{"id":"do_112806963140329472124"},"index":"true","ets":"1591949601174","eventData":{"action":"update-content-rating","stage":4,"metadata":{"me_averageRating":"1","me_total_time_spent_in_app":"2","me_total_time_spent_in_portal":"3","me_total_time_spent_in_desktop":"4","me_total_play_sessions_in_app":"5","me_total_play_sessions_in_portal":"6","me_total_play_sessions_in_desktop":"7"}}} - |""".stripMargin - -} \ No newline at end of file diff --git a/mvc-indexer/src/test/scala/org/sunbird/job/spec/MVCProcessorIndexerTaskTestSpec.scala b/mvc-indexer/src/test/scala/org/sunbird/job/spec/MVCProcessorIndexerTaskTestSpec.scala deleted file mode 100644 index de6ad4b53..000000000 --- a/mvc-indexer/src/test/scala/org/sunbird/job/spec/MVCProcessorIndexerTaskTestSpec.scala +++ /dev/null @@ -1,153 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import okhttp3.mockwebserver.{Dispatcher, MockResponse, MockWebServer, RecordedRequest} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.client.JobExecutionException -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HttpUtil, JSONUtil} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.mvcindexer.domain.Event -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.mvcindexer.task.{MVCIndexerConfig, MVCIndexerStreamTask} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.util - -class MVCProcessorIndexerTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig: MVCIndexerConfig = new MVCIndexerConfig(config) - var cassandraUtil: CassandraUtil = null - val esUtil: ElasticSearchUtil = null - val httpUtil: HttpUtil = new HttpUtil - val esServer = new MockWebServer() - - var currentMilliSecond = 1605816926271L - - val esDispatcher: Dispatcher = new Dispatcher() { - @throws[InterruptedException] - override def dispatch(request: RecordedRequest): MockResponse = { - (request.getPath, request.getMethod) match { - case ("/mvc-content-v1", "HEAD") => - new MockResponse().setResponseCode(200) - case ("/mvc-content-v1?master_timeout=30s&timeout=30s", "PUT") => - new MockResponse().setHeader("Content-Type", "application/json").setResponseCode(200).setBody("""{"acknowledged":true,"shards_acknowledged":true,"index":"mvc-content-v1"}""") - case ("/mvc-content-v1/_doc/do_112806963140329472124?timeout=1m", "PUT") => - new MockResponse().setHeader("Content-Type", "application/json").setResponseCode(200).setBody("""{"_index":"mvc-content-v1","_type":"_doc","_id":"do_112806963140329472124","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":0,"_primary_term":1}""") - case _ => { - new MockResponse().setResponseCode(200) - } - } - } - } - - override protected def beforeAll(): Unit = { - super.beforeAll() - esServer.setDispatcher(esDispatcher) - esServer.start(9200) - - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.lmsDbHost, jobConfig.lmsDbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - testCassandraUtil(cassandraUtil) - BaseMetricsReporter.gaugeMetrics.clear() - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => ex.printStackTrace() - } - super.afterAll() - esServer.close() - flinkCluster.after() - } - - - "MVCProcessorIndexerStreamTask" should "index in ES and Cassandra" in { - val contentServer = new MockWebServer() - contentServer.start(8080) - contentServer.enqueue(new MockResponse().setHeader( - "Content-Type", "application/json" - ).setBody("""{"responseCode":"OK","result":{"content":{"channel":"in.ekstep","framework":"NCF","name":"Ecml bundle Test","language":["English"],"appId":"dev.sunbird.portal","contentEncoding":"gzip","identifier":"do_112806963140329472124","mimeType":"application/vnd.ekstep.ecml-archive","contentType":"Resource","objectType":"Content","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112806963140329472124/artifact/1563350021721_do_112806963140329472124.zip","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/ecml/do_112806963140329472124-latest","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/ecml/do_112806963140329472124-latest","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_112806963140329472124/ecml-bundle-test_1563350022377_do_112806963140329472124_1.0.ecar","status":"Live","pkgVersion":1,"lastUpdatedOn":"2019-07-17T07:53:25.618+0000"}}}""")) - - val mlVectorServer = new MockWebServer() - mlVectorServer.start(3579) - - mlVectorServer.enqueue(new MockResponse().setHeader( - "Content-Type", "application/json" - ).setBody("""{"responseCode":"OK"}""")) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new MVCProcessorIndexerMapSource) - - new MVCIndexerStreamTask(jobConfig, mockKafkaUtil, esUtil, httpUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dbUpdateCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(0) - contentServer.close() - } - - "MVCProcessorIndexerStreamTask" should "throw exception and increase error count" in { - val contentServer = new MockWebServer() - contentServer.start(8080) - contentServer.enqueue(new MockResponse().setHeader( - "Content-Type", "application/json" - ).setResponseCode(500).setBody("""{}""")) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new MVCProcessorIndexerMapSource) - - try { - new MVCIndexerStreamTask(jobConfig, mockKafkaUtil, esUtil, httpUtil).process() - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.apiFailedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.esFailedEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dbUpdateFailedCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(0) - } - contentServer.close() - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - } -} - -class MVCProcessorIndexerMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - // Valid event - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1), 0, 10)) - // Invalid event - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2), 0, 11)) - } - - override def cancel(): Unit = {} -} \ No newline at end of file diff --git a/mvc-indexer/src/test/scala/org/sunbird/job/spec/service/MVCProcessorIndexerServiceTestSpec.scala b/mvc-indexer/src/test/scala/org/sunbird/job/spec/service/MVCProcessorIndexerServiceTestSpec.scala deleted file mode 100644 index a433b34ee..000000000 --- a/mvc-indexer/src/test/scala/org/sunbird/job/spec/service/MVCProcessorIndexerServiceTestSpec.scala +++ /dev/null @@ -1,231 +0,0 @@ -package org.sunbird.job.spec.service - -import com.datastax.driver.core.Row -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.mockito.{ArgumentCaptor, Mockito} -import org.sunbird.job.mvcindexer.domain.Event -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.mvcindexer.service.MVCIndexerService -import org.sunbird.job.mvcindexer.task.MVCIndexerConfig -import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.spec.BaseTestSpec -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, endsWith} -import org.mockito.Mockito.{doNothing, times, verify, when} -import org.sunbird.job.Metrics -import org.sunbird.job.exception.{APIException, CassandraException, ElasticSearchException} - -import java.util - -class MVCProcessorIndexerServiceTestSpec extends BaseTestSpec { - implicit val mapTypeInfo: TypeInformation[util.Map[String, Any]] = TypeExtractor.getForClass(classOf[util.Map[String, Any]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val config: Config = ConfigFactory.load("test.conf") - lazy val jobConfig: MVCIndexerConfig = new MVCIndexerConfig(config) - var cassandraUtil: CassandraUtil = _ - var mockElasticUtil:ElasticSearchUtil = _ - var mockHttpUtil:HttpUtil = _ - var mockMetrics: Metrics = _ -// val mockHttpUtil:HttpUtil = new HttpUtil - var mvcProcessorIndexer: MVCIndexerService = _ - - val contentResponse = """{"responseCode":"OK","result":{"content":{"channel":"in.ekstep","framework":"NCF","name":"Ecml bundle Test","language":["English"],"appId":"dev.sunbird.portal","contentEncoding":"gzip","identifier":"do_112806963140329472124","mimeType":"application/vnd.ekstep.ecml-archive","contentType":"Resource","objectType":"Content","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112806963140329472124/artifact/1563350021721_do_112806963140329472124.zip","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/ecml/do_112806963140329472124-latest","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/ecml/do_112806963140329472124-latest","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_112806963140329472124/ecml-bundle-test_1563350022377_do_112806963140329472124_1.0.ecar","status":"Live","pkgVersion":1,"lastUpdatedOn":"2019-07-17T07:53:25.618+0000"}}}""" - - override protected def beforeAll(): Unit = { - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.lmsDbHost, jobConfig.lmsDbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - testCassandraUtil(cassandraUtil) - super.beforeAll() - } - - override def beforeEach(): Unit = { - super.beforeEach() - testCassandraUtil(cassandraUtil) - mockElasticUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - mockMetrics = mock[Metrics](Mockito.withSettings().serializable()) - mvcProcessorIndexer = new MVCIndexerService(jobConfig, mockElasticUtil, mockHttpUtil, cassandraUtil) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => ex.printStackTrace() - } - } - - "MVCProcessorIndexerService" should "generate es log" in { - val argumentCaptor = ArgumentCaptor.forClass(classOf[ElasticSearchUtil]).asInstanceOf[ArgumentCaptor[String]] - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - - when(mockHttpUtil.get(endsWith("content/v3/read/do_112806963140329472124"), any())).thenReturn(HTTPResponse(200, contentResponse)) - when(mockHttpUtil.post(endsWith("/daggit/submit"), any(), any())).thenReturn(HTTPResponse(200, """{}""")) - - mvcProcessorIndexer.processMessage(inputEvent)(mockMetrics) - - val insertedRecord = readFromCassandra(inputEvent.identifier) - insertedRecord.forEach(col => { - col.getObject("source") should be ("Sunbird 1") - col.getObject("sourceurl") should be ("https://dev.sunbirded.org/play/content/do_112806963140329472124") - col.getObject("level1_name").asInstanceOf[util.ArrayList[String]] should contain("Math-Magic") - col.getObject("textbook_name").asInstanceOf[util.ArrayList[String]] should contain("How Many Times?") - col.getObject("level1_concept").asInstanceOf[util.ArrayList[String]] should contain("Addition") - }) - - verify(mockElasticUtil, times(1)).addDocumentWithIndex(argumentCaptor.capture(), argumentCaptor.capture(), argumentCaptor.capture()); - val esRecordMap = JSONUtil.deserialize[Map[String, AnyRef]](argumentCaptor.getAllValues.get(0)) - - esRecordMap("level1Concept").asInstanceOf[List[String]] should contain("Addition") - esRecordMap("artifactUrl") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112806963140329472124/artifact/1563350021721_do_112806963140329472124.zip") - esRecordMap("downloadUrl") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_112806963140329472124/ecml-bundle-test_1563350022377_do_112806963140329472124_1.0.ecar") - esRecordMap("pkgVersion") should be(1) - esRecordMap("previewUrl") should be("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/ecml/do_112806963140329472124-latest") - esRecordMap("level1Name").asInstanceOf[List[String]] should contain("Math-Magic") - esRecordMap("identifier") should be("do_112806963140329472124") - esRecordMap("appId") should be("dev.sunbird.portal") - esRecordMap("objectType") should be("Content") - esRecordMap("name") should be("Ecml bundle Test") - esRecordMap("lastUpdatedOn") should be("2019-07-17T07:53:25.618+0000") - esRecordMap("status") should be("Live") - esRecordMap("source").asInstanceOf[List[String]] should contain("Sunbird 1") - esRecordMap("contentEncoding") should be("gzip") - esRecordMap("contentType") should be("Resource") - esRecordMap("channel") should be("in.ekstep") - esRecordMap("textbook_name").asInstanceOf[List[String]] should contain("How Many Times?") - esRecordMap("mimeType") should be("application/vnd.ekstep.ecml-archive") - esRecordMap("language").asInstanceOf[List[String]] should contain("English") - esRecordMap("sourceURL") should be("https://dev.sunbirded.org/play/content/do_112806963140329472124") - argumentCaptor.getAllValues.get(2) should be(inputEvent.identifier) - } - - "MVCProcessorIndexerService" should "update ml-keywords" in { - val argumentCaptor = ArgumentCaptor.forClass(classOf[ElasticSearchUtil]).asInstanceOf[ArgumentCaptor[String]] - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3), 0, 11) - - when(mockHttpUtil.post(endsWith("/ml/vector/ContentText"), any(), any())).thenReturn(HTTPResponse(200, """{}""")) - - mvcProcessorIndexer.processMessage(inputEvent)(mockMetrics) - - verify(mockHttpUtil, times(1)).post(argumentCaptor.capture(), argumentCaptor.capture(), any()); - val esRecordMap = JSONUtil.deserialize[Map[String, AnyRef]](argumentCaptor.getAllValues.get(1)) - - esRecordMap("request").asInstanceOf[Map[String, AnyRef]]("text").asInstanceOf[List[String]] should contain(inputEvent.mlContentText) - esRecordMap("request").asInstanceOf[Map[String, AnyRef]]("cid") should be(inputEvent.identifier) - - val insertedRecord = readFromCassandra(inputEvent.identifier) - insertedRecord.forEach(col => { - val mlKeywords = col.getObject("ml_keywords").asInstanceOf[util.ArrayList[String]] - mlKeywords should contain(inputEvent.mlKeywords(0)) - mlKeywords should contain(inputEvent.mlKeywords(1)) - mlKeywords should contain(inputEvent.mlKeywords(2)) - col.getObject("ml_content_text") should be(inputEvent.mlContentText) - }) - } - - "MVCProcessorIndexerService" should "update ml-contenttextvector" in { - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_4), 0, 11) - - when(mockHttpUtil.get(endsWith("/ml/vector/ContentText"), any())).thenReturn(HTTPResponse(200, contentResponse)) - - mvcProcessorIndexer.processMessage(inputEvent)(mockMetrics) - - val insertedRecord = readFromCassandra(inputEvent.identifier) - insertedRecord.forEach(col => { - val mlContentTextVector = col.getObject("ml_content_text_vector").asInstanceOf[util.LinkedHashSet[Double]] - mlContentTextVector should contain(inputEvent.mlContentTextVector(0)) - mlContentTextVector should contain(inputEvent.mlContentTextVector(1)) - mlContentTextVector should contain(inputEvent.mlContentTextVector(2)) - mlContentTextVector should contain(inputEvent.mlContentTextVector(3)) - }) - } - - "MVCProcessorIndexerService" should "update content-ratings" in { - val argumentCaptor = ArgumentCaptor.forClass(classOf[ElasticSearchUtil]).asInstanceOf[ArgumentCaptor[String]] - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_5), 0, 11) - - when(mockElasticUtil.getDocumentAsString(any())).thenReturn("""{"_id":"do_112806963140329472124"}""") - - mvcProcessorIndexer.processMessage(inputEvent)(mockMetrics) - - verify(mockElasticUtil, times(1)).updateDocument(argumentCaptor.capture(), argumentCaptor.capture()); - val esRecordMap = JSONUtil.deserialize[Map[String, AnyRef]](argumentCaptor.getAllValues.get(1)) - - esRecordMap("me_total_time_spent_in_portal") should be(inputEvent.metadata("me_total_time_spent_in_portal")) - esRecordMap("me_averageRating") should be(inputEvent.metadata("me_averageRating")) - esRecordMap("me_total_time_spent_in_app") should be(inputEvent.metadata("me_total_time_spent_in_app")) - esRecordMap("me_total_play_sessions_in_portal") should be(inputEvent.metadata("me_total_play_sessions_in_portal")) - esRecordMap("me_total_play_sessions_in_desktop") should be(inputEvent.metadata("me_total_play_sessions_in_desktop")) - esRecordMap("me_total_play_sessions_in_app") should be(inputEvent.metadata("me_total_play_sessions_in_app")) - esRecordMap("me_total_time_spent_in_desktop") should be(inputEvent.metadata("me_total_time_spent_in_desktop")) - } - - "MVCProcessorIndexerService" should "increase Es failed metric on when exception in ElasticSearch" in { - val argumentCaptor = ArgumentCaptor.forClass(classOf[ElasticSearchUtil]).asInstanceOf[ArgumentCaptor[String]] - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10) - - when(mockHttpUtil.get(endsWith("content/v3/read/do_112806963140329472124"), any())).thenReturn(HTTPResponse(200, contentResponse)) - when(mockHttpUtil.post(endsWith("/daggit/submit"), any(), any())).thenReturn(HTTPResponse(200, """{}""")) - - try { - mvcProcessorIndexer.processMessage(inputEvent)(mockMetrics) - } catch { - case ex: ElasticSearchException => { - verify(mockMetrics, times(2)).incCounter(argumentCaptor.capture()); - argumentCaptor.getAllValues.get(0) should be(jobConfig.dbUpdateCount) - argumentCaptor.getAllValues.get(1) should be(jobConfig.esFailedEventCount) - } - } - } - - "MVCProcessorIndexerService" should "increase DB failed metric on while exception in Cassandra" in { - val argumentCaptor = ArgumentCaptor.forClass(classOf[ElasticSearchUtil]).asInstanceOf[ArgumentCaptor[String]] - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3), 0, 11) - - when(mockHttpUtil.post(endsWith("/ml/vector/ContentText"), any(), any())).thenReturn(HTTPResponse(200, """{}""")) - - try { - cassandraUtil.close() - mvcProcessorIndexer.processMessage(inputEvent)(mockMetrics) - } catch { - case ex: CassandraException => { - verify(mockMetrics, times(1)).incCounter(argumentCaptor.capture()); - argumentCaptor.getAllValues.get(0) should be(jobConfig.dbUpdateFailedCount) - } - } - } - - "MVCProcessorIndexerService" should "increase api failed metric on while exception in API" in { - val argumentCaptor = ArgumentCaptor.forClass(classOf[ElasticSearchUtil]).asInstanceOf[ArgumentCaptor[String]] - val inputEvent:Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3), 0, 11) - - when(mockHttpUtil.post(endsWith("/ml/vector/ContentText"), any(), any())).thenReturn(HTTPResponse(500, """{}""")) - - try { - mvcProcessorIndexer.processMessage(inputEvent)(mockMetrics) - } catch { - case ex: APIException => { - verify(mockMetrics, times(1)).incCounter(argumentCaptor.capture()); - argumentCaptor.getAllValues.get(0) should be(jobConfig.apiFailedEventCount) - } - } - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - } - - def readFromCassandra(contentId: String): util.List[Row] = { - val query = s"select * from ${jobConfig.dbKeyspace}.${jobConfig.dbTable} where content_id='${contentId}' ALLOW FILTERING;" - cassandraUtil.find(query) - } -} diff --git a/pom.xml b/pom.xml index f466c82aa..b4816859d 100644 --- a/pom.xml +++ b/pom.xml @@ -27,19 +27,8 @@ jobs-core - post-publish-processor - publish-pipeline - search-indexer auto-creator-v2 content-auto-creator - transaction-event-processor - qrcode-image-generator - dialcode-context-updater - cassandra-data-migration - live-video-stream-generator - csp-migrator - asset-enrichment - video-stream-generator @@ -49,7 +38,6 @@ jobs-core - publish-pipeline/publish-core diff --git a/post-publish-processor/pom.xml b/post-publish-processor/pom.xml deleted file mode 100644 index 688970c17..000000000 --- a/post-publish-processor/pom.xml +++ /dev/null @@ -1,214 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - post-publish-processor - 1.0.0 - jar - post-publish-processor - - Post Publish Processor Flink Job - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.postpublish.task.PostPublishProcessorStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - post-publish-processor-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/post-publish-processor/src/main/resources/log4j.properties b/post-publish-processor/src/main/resources/log4j.properties deleted file mode 100644 index 4a2ed773e..000000000 --- a/post-publish-processor/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=post-publish-processor.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/post-publish-processor/src/main/resources/post-publish-processor.conf b/post-publish-processor/src/main/resources/post-publish-processor.conf deleted file mode 100644 index d7a09751f..000000000 --- a/post-publish-processor/src/main/resources/post-publish-processor.conf +++ /dev/null @@ -1,54 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.content.postpublish.request" - groupId = "local-post-publish-processor-group" - publish.topic = "sunbirddev.learning.job.request" - qrimage.topic = "sunbirddev.qrimage.request" - dialcode.context.topic = "sunbirddev.dialcode.context.job.request" -} - -task { - consumer.parallelism = 1 - router.parallelism = 1 - shallow_copy.parallelism = 1 - link_dialcode.parallelism = 1 - batch_create.parallelism = 1 - dialcode_context_updater.parallelism = 1 -} - -lms-cassandra { - keyspace = "sunbird_courses" - batchTable = "course_batch" -} - -dialcode-cassandra { - keyspace = "dialcodes" - imageTable = "dialcode_images" -} - -service { - search.basePath = "http://11.2.6.6/search" - lms.basePath = "http://11.2.6.6/lms" - learning_service.basePath = "http://11.2.4.22:8080/learning-service" - dial.basePath = "https://dev.sunbirded.org/dial/" -} - -dialcode { - linkable.primaryCategory = ["Course"] -} - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"] -cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] - -cloud_storage_type="azure" -cloud_storage_key="" -cloud_storage_secret="" -cloud_storage_container="" -cloud_storage_endpoint="" \ No newline at end of file diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/domain/Event.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/domain/Event.scala deleted file mode 100644 index 77018e4d4..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/domain/Event.scala +++ /dev/null @@ -1,24 +0,0 @@ -package org.sunbird.job.postpublish.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest - -import java.util - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val jobName = "PostPublishProcessor" - def action: String = readOrDefault[String]("edata.action", "") - - def mimeType: String = readOrDefault[String]("edata.mimeType", "") - - def collectionId: String = readOrDefault[String]("edata.identifier", "") - - def eData: Map[String, AnyRef] = readOrDefault("edata", new util.HashMap[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - - def validEvent(): Boolean = { - StringUtils.equals("post-publish-process", action) && - StringUtils.equals("application/vnd.ekstep.content-collection", mimeType) - } - -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/BatchCreateFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/BatchCreateFunction.scala deleted file mode 100644 index 7bc852292..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/BatchCreateFunction.scala +++ /dev/null @@ -1,48 +0,0 @@ -package org.sunbird.job.postpublish.functions - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.postpublish.helpers.BatchCreation -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.time.format.DateTimeFormatter -import java.time.{ZoneId, ZonedDateTime} - -class BatchCreateFunction(config: PostPublishProcessorConfig, httpUtil: HttpUtil) - extends BaseProcessFunction[java.util.Map[String, AnyRef], String](config) with BatchCreation { - - private[this] val logger = LoggerFactory.getLogger(classOf[BatchCreateFunction]) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(eData: java.util.Map[String, AnyRef], context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, metrics: Metrics): Unit = { - val collectionId = eData.getOrDefault("identifier", "") - metrics.incCounter(config.batchCreationCount) - val startDate = ZonedDateTime.now(ZoneId.of("Asia/Kolkata")).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) - logger.info("Creating Batch for " + collectionId + " with start date:" + startDate) - try { - createBatch(eData, startDate)(config, httpUtil) - metrics.incCounter(config.batchCreationSuccessCount) - logger.info("Batch created for " + collectionId) - } catch { - case ex: Throwable => - logger.error(s"Error while processing message for identifier : ${collectionId}.", ex) - metrics.incCounter(config.batchCreationFailedCount) - throw ex - } - } - - override def metricsList(): List[String] = { - List(config.batchCreationCount, config.batchCreationSuccessCount, config.batchCreationFailedCount) - } - -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/DIALCodeLinkFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/DIALCodeLinkFunction.scala deleted file mode 100644 index fadb467c8..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/DIALCodeLinkFunction.scala +++ /dev/null @@ -1,80 +0,0 @@ -package org.sunbird.job.postpublish.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.postpublish.helpers.DialHelper -import org.sunbird.job.postpublish.models.ExtDataConfig -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil, Neo4JUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.util -import scala.collection.JavaConverters._ - -class DIALCodeLinkFunction(config: PostPublishProcessorConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[java.util.Map[String, AnyRef], String](config) - with DialHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeLinkFunction]) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - } - - override def close(): Unit = { - cassandraUtil.close() - super.close() - } - - override def processElement(edata: java.util.Map[String, AnyRef], context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, metrics: Metrics): Unit = { - logger.info(s"Link DIAL Code operation triggered with object : ${edata}") - metrics.incCounter(config.dialLinkCount) - try { - val dialcode: String = getDialcode(edata) - metrics.incCounter(config.dialLinkSuccessCount) - if (!dialcode.isEmpty) - createQRGeneratorEvent(edata, dialcode, context, config)(metrics, ExtDataConfig(config.dialcodeKeyspaceName, config.dialcodeTableName), cassandraUtil) - } catch { - case ex: Throwable => - logger.error(s"Error while processing message for identifier : ${edata.get("identifier").asInstanceOf[String]}.", ex) - metrics.incCounter(config.dialLinkFailedCount) - throw ex - } - } - - def getDialcode(edata: java.util.Map[String, AnyRef]): String = { - - val identifier = edata.get("identifier").asInstanceOf[String] - val dialcodes = fetchExistingDialcodes(edata) - logger.info(s"Dialcodes fetched: ${dialcodes}") // temp - if (dialcodes.isEmpty) { - logger.info(s"No Dial Code found. Checking for Reserved Dialcodes.") - var reservedDialCode: util.Map[String, Integer] = fetchExistingReservedDialcodes(edata) - if (reservedDialCode.isEmpty) { - logger.info(s"No Reserved Dial Code found. Sending request for Reserving Dialcode.") - reservedDialCode = reserveDialCodes(edata, config)(httpUtil) - } - reservedDialCode.asScala.keys.headOption match { - case Some(dialcode: String) => { - updateDIALToObject(identifier, dialcode)(neo4JUtil) - dialcode - } - case _ => { - logger.info(s"Couldn't reserve any dialcodes for object with identifier:${identifier}") - throw new Exception(s"Failed to Reserve dialcode for object with identifier:${identifier}.") - } - } - } else if (validateQR(dialcodes.get(0))(ExtDataConfig(config.dialcodeKeyspaceName, config.dialcodeTableName), cassandraUtil)) "" else dialcodes.get(0) - } - - override def metricsList(): List[String] = { - List(config.dialLinkCount, config.qrImageGeneratorEventCount, config.dialLinkSuccessCount, config.dialLinkFailedCount) - } -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/DialCodeContextUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/DialCodeContextUpdaterFunction.scala deleted file mode 100644 index 024c92bd7..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/DialCodeContextUpdaterFunction.scala +++ /dev/null @@ -1,49 +0,0 @@ -package org.sunbird.job.postpublish.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.postpublish.helpers.DialHelper -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.ScalaJsonUtil -import org.sunbird.job.{BaseProcessFunction, Metrics} - - -class DialCodeContextUpdaterFunction(config: PostPublishProcessorConfig) (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[java.util.Map[String, AnyRef], String](config) - with DialHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[DialCodeContextUpdaterFunction]) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(edata: java.util.Map[String, AnyRef], context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, metrics: Metrics): Unit = { - logger.info(s"DIAL Code Context Updater operation triggered with object : $edata") - metrics.incCounter(config.dialcodeContextUpdaterCount) - try { - val addContextDialCodes: Map[List[String],String] = edata.getOrDefault("addContextDialCodes", Map.empty[String,String]).asInstanceOf[Map[String,String]].map(rec => (ScalaJsonUtil.deserialize[List[String]](rec._1)->rec._2)) - val removeContextDialCodes: Map[List[String],String] = edata.getOrDefault("removeContextDialCodes", Map.empty[String,String]).asInstanceOf[Map[String,String]].map(rec => (ScalaJsonUtil.deserialize[List[String]](rec._1)->rec._2)) - val channel: String = edata.getOrDefault("channel", "").asInstanceOf[String] - generateDialcodeContextUpdaterEvent(channel, addContextDialCodes, removeContextDialCodes, context, metrics)(config) - metrics.incCounter(config.dialcodeContextUpdaterSuccessCount) - } catch { - case ex: Throwable => - logger.error(s"Error while processing message for DIAL Code Context Updater : $edata.", ex) - metrics.incCounter(config.dialcodeContextUpdaterFailedCount) - throw ex - } - } - - - - override def metricsList(): List[String] = { - List(config.dialcodeContextUpdaterCount, config.dialcodeContextUpdaterSuccessCount, config.dialcodeContextUpdaterFailedCount) - } -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala deleted file mode 100644 index 2c69ef8ca..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala +++ /dev/null @@ -1,74 +0,0 @@ -package org.sunbird.job.postpublish.functions - -import com.google.gson.reflect.TypeToken -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.postpublish.domain.Event -import org.sunbird.job.postpublish.helpers.{BatchCreation, DialHelper, ShallowCopyPublishing} -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil, Neo4JUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type - -case class PublishMetadata(identifier: String, contentType: String, mimeType: String, pkgVersion: Int) - -class PostPublishEventRouter(config: PostPublishProcessorConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null) - extends BaseProcessFunction[Event, String](config) with ShallowCopyPublishing with BatchCreation with DialHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[PostPublishEventRouter]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - val contentTypes = List("Course") - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - } - - override def close(): Unit = { - cassandraUtil.close() - super.close() - } - - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - logger.info("Processed event using JobRequest-SerDe: " + event) - metrics.incCounter(config.totalEventsCount) - if (event.validEvent()) { - val identifier = event.collectionId - - // Process Shallow Copy Content - getShallowCopiedContents(identifier)(config, httpUtil).foreach(metadata => context.output(config.shallowContentPublishOutTag, metadata)) - - // Process Batch Creation - val batchDetails = getBatchDetails(identifier)(neo4JUtil, cassandraUtil, config) - if (!batchDetails.isEmpty) - context.output(config.batchCreateOutTag, batchDetails) - - // Process Dialcode link - val dialCodeDetails = getDialCodeDetails(identifier, event)(neo4JUtil, config) - if (!dialCodeDetails.isEmpty) - context.output(config.linkDIALCodeOutTag, dialCodeDetails) - - val dialcodeContextMap = getDialCodeContextMap(event) - if(!dialcodeContextMap.isEmpty) - context.output(config.dialcodeContextOutTag, dialcodeContextMap) - } - else if (event.action.equals("post-publish-process") && event.eData.contains("addContextDialCodes") && event.eData.contains("removeContextDialCodes")) { - val dialcodeContextMap = getDialCodeContextMap(event) - if(!dialcodeContextMap.isEmpty) - context.output(config.dialcodeContextOutTag, dialcodeContextMap) - } - else { - metrics.incCounter(config.skippedEventCount) - logger.info(s"Event not qualified for publishing for Identifier : ${event.collectionId}.") - } - } - - override def metricsList(): List[String] = { - List(config.skippedEventCount, config.totalEventsCount) - } -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/ShallowCopyPublishFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/ShallowCopyPublishFunction.scala deleted file mode 100644 index 5566674e9..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/ShallowCopyPublishFunction.scala +++ /dev/null @@ -1,38 +0,0 @@ -package org.sunbird.job.postpublish.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.util.UUID - -class ShallowCopyPublishFunction(config: PostPublishProcessorConfig) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[PublishMetadata, String](config) { - - private[this] val logger = LoggerFactory.getLogger(classOf[ShallowCopyPublishFunction]) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(metadata: PublishMetadata, context: ProcessFunction[PublishMetadata, String]#Context, metrics: Metrics): Unit = { - val epochTime = System.currentTimeMillis - val event = s"""{"eid":"BE_JOB_REQUEST","ets":${epochTime},"mid":"LP.${epochTime}.${UUID.randomUUID()}","actor":{"id":"collection-publish","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"sunbird","env":"sunbirddev"},"object":{"ver":"${metadata.pkgVersion}","id":"${metadata.identifier}"},"edata":{"publish_type":"public","metadata":{"identifier":"${metadata.identifier}", "mimeType":"${metadata.mimeType}","objectType":"Collection","lastPublishedBy":"System","pkgVersion":${metadata.pkgVersion}},"action":"publish","iteration":1,"contentType":"${metadata.contentType}"}}""" - context.output(config.publishEventOutTag, event) - metrics.incCounter(config.shallowCopyCount) - logger.info("Shallow copy content publish triggered for " + metadata.identifier) - logger.info("Shallow copy content publish event: " + event) - } - - override def metricsList(): List[String] = { - List(config.shallowCopyCount) - } -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala deleted file mode 100644 index b7f0921d1..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala +++ /dev/null @@ -1,103 +0,0 @@ -package org.sunbird.job.postpublish.helpers - -import com.datastax.driver.core.querybuilder.QueryBuilder -import org.apache.commons.collections.{CollectionUtils, MapUtils} -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil, Neo4JUtil} - -import java.util -import scala.collection.JavaConverters._ - -trait BatchCreation { - - private[this] val logger = LoggerFactory.getLogger(classOf[BatchCreation]) - - def createBatch(eData: java.util.Map[String, AnyRef], startDate: String)(implicit config: PostPublishProcessorConfig, httpUtil: HttpUtil) = { - val request = new java.util.HashMap[String, AnyRef]() { - { - put("request", new java.util.HashMap[String, AnyRef]() { - { - put("courseId", eData.get("identifier")) - put("name", eData.get("name")) - if (eData.containsKey("createdBy")) - put("createdBy", eData.get("createdBy")) - if (eData.containsKey("createdFor")) - put("createdFor", eData.get("createdFor")) - put("enrollmentType", "open") - put("startDate", startDate) - } - }) - } - } - val httpRequest = JSONUtil.serialize(request) - val httpResponse = httpUtil.post(config.batchCreateAPIPath, httpRequest) - if (httpResponse.status == 200) { - logger.info("Batch create success: " + httpResponse.body) - } else { - logger.error("Batch create failed: " + httpResponse.status + " :: " + httpResponse.body) - throw new Exception("Batch creation failed for " + eData.get("identifier")) - } - } - - - def batchRequired(metadata: java.util.Map[String, AnyRef], identifier: String)(implicit config: PostPublishProcessorConfig, cassandraUtil: CassandraUtil): Boolean = { - val trackable = isTrackable(metadata, identifier) - if (trackable) { - !isBatchExists(identifier) - } else false - } - - def isTrackable(metadata: java.util.Map[String, AnyRef], identifier: String): Boolean = { - if (MapUtils.isNotEmpty(metadata)) { - val trackableStr = metadata.getOrDefault("trackable", "{}").asInstanceOf[String] - val trackableObj = JSONUtil.deserialize[java.util.Map[String, AnyRef]](trackableStr) - val trackingEnabled = trackableObj.getOrDefault("enabled", "No").asInstanceOf[String] - val autoBatchCreateEnabled = trackableObj.getOrDefault("autoBatch", "No").asInstanceOf[String] - val trackable = (StringUtils.equalsIgnoreCase(trackingEnabled, "Yes") && StringUtils.equalsIgnoreCase(autoBatchCreateEnabled, "Yes")) - logger.info("Trackable for " + identifier + " : " + trackable) - trackable - } else { - throw new Exception("Metadata [isTrackable] is not found for object: " + identifier) - } - } - - def isBatchExists(identifier: String)(implicit config: PostPublishProcessorConfig, cassandraUtil: CassandraUtil): Boolean = { - val selectQuery = QueryBuilder.select().all().from(config.lmsKeyspaceName, config.batchTableName) - selectQuery.where.and(QueryBuilder.eq("courseid", identifier)) - val rows = cassandraUtil.find(selectQuery.toString) - if (CollectionUtils.isNotEmpty(rows)) { - val activeBatches = rows.asScala.filter(row => { - val enrolmentType = row.getString("enrollmenttype") - val status = row.getInt("status") - (StringUtils.equalsIgnoreCase(enrolmentType, "Open") && (0 == status || 1 == status)) - }).toList - if (activeBatches.nonEmpty) - logger.info("Collection has a active batch: " + activeBatches.head.toString) - activeBatches.nonEmpty - } else false - } - - def getBatchDetails(identifier: String)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, config: PostPublishProcessorConfig): util.Map[String, AnyRef] = { - logger.info("Process Batch Creation for content: " + identifier) - val metadata = neo4JUtil.getNodeProperties(identifier) - - // Validate and trigger batch creation. - if (batchRequired(metadata, identifier)(config, cassandraUtil)) { - val createdFor = metadata.get("createdFor").asInstanceOf[java.util.List[String]] - new util.HashMap[String, AnyRef]() { - { - put("identifier", identifier) - put("name", metadata.get("name")) - put("createdBy", metadata.get("createdBy")) - if (CollectionUtils.isNotEmpty(createdFor)) - put("createdFor", new util.ArrayList[String](createdFor)) - } - } - } else { - new util.HashMap[String, AnyRef]() - } - } - -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/DialHelper.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/DialHelper.scala deleted file mode 100644 index 686746186..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/DialHelper.scala +++ /dev/null @@ -1,158 +0,0 @@ -package org.sunbird.job.postpublish.helpers - -import com.google.gson.Gson -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.postpublish.domain.Event -import org.sunbird.job.postpublish.models.ExtDataConfig -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil, Neo4JUtil, ScalaJsonUtil} - -import java.util -import java.util.UUID -import scala.collection.JavaConverters._ - -trait DialHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[DialHelper]) - val graphId = "domain" - lazy private val gson = new Gson() - - - def fetchExistingReservedDialcodes(edata: java.util.Map[String, AnyRef]): util.Map[String, Integer] = { - val reservedDialcodes = edata.getOrDefault("reservedDialcodes", "{}").asInstanceOf[String] - JSONUtil.deserialize[util.Map[String, Integer]](reservedDialcodes) - } - - def reserveDialCodes(edata: java.util.Map[String, AnyRef], config: PostPublishProcessorConfig)(implicit httpUtil: HttpUtil): java.util.Map[String, Integer] = { - val identifier = edata.get("identifier").asInstanceOf[String] - val request = s"""{"request": { "dialcodes": {"count": 1, "qrCodeSpec": {"errorCorrectionLevel": "H"}}}}""" - val headers = Map[String, String]("X-Channel-Id" -> edata.getOrDefault("channel", "").asInstanceOf[String], - "Content-Type" -> "application/json") - - logger.info(s"Reserved Dialcode Api request body : ${request}") - logger.info(s"Reserved Dialcode Api header : ${headers}") - logger.info(s"Reserved Dialcode Api url : " + config.reserveDialCodeAPIPath + identifier) - - val response = httpUtil.post(config.reserveDialCodeAPIPath + "/" + identifier, request, headers) - logger.info(s"Reserved Dialcode Api response status : ${response.status} and response body : ${response.body}") - if (response.status == 200) { - val responseBody = gson.fromJson(response.body, classOf[java.util.Map[String, AnyRef]]) - val reservedDialcodes: util.Map[String, Integer] = responseBody.getOrDefault("result", new util.HashMap[String, AnyRef]) - .asInstanceOf[util.Map[String, AnyRef]].getOrDefault("reservedDialcodes", new util.HashMap[String, Integer]()) - .asInstanceOf[util.Map[String, Integer]] - reservedDialcodes - } else throw new Exception(s"Couldn't reserve dialcodes for identifier: $identifier") - } - - def updateDIALToObject(identifier: String, dialCode: String)(implicit neo4JUtil: Neo4JUtil) = { - neo4JUtil.updateNodeProperty(identifier, "dialcodes", (s"""["${dialCode}"]""")) - logger.info(s"Added Reserved Dialcode to node.") - } - - def fetchExistingDialcodes(edata: java.util.Map[String, AnyRef]): util.List[String] = { - edata.getOrDefault("dialcodes", new util.ArrayList()).asInstanceOf[util.List[String]] - } - - def validatePrimaryCategory(edata: java.util.Map[String, AnyRef])(implicit config: PostPublishProcessorConfig) = { - val primaryCategory = edata.get("primaryCategory").asInstanceOf[String] - config.primaryCategories.contains(primaryCategory) - } - - def validateQR(dialcode: String)(implicit extConfig: ExtDataConfig, cassandraUtil: CassandraUtil): Boolean = { - getQRImageRecord(dialcode) match { - case Some(url: String) => true - case _ => false - } - } - - def getQRImageRecord(dialcode: String)(implicit extConfig: ExtDataConfig, cassandraUtil: CassandraUtil): Option[String] = { - if (dialcode.isEmpty) throw new Exception("Invalid dialcode to read") - val fileName = s"0_$dialcode" - val query = s"select url from ${extConfig.keyspace}.${extConfig.table} where filename = '$fileName';" - val result = cassandraUtil.findOne(query) - if (result == null) None else Some(result.getString("url")) - } - - def updateDialcodeRecord(dialcode: String, channel: String, ets: Long)(implicit extConfig: ExtDataConfig, cassandraUtil: CassandraUtil): Boolean = { - if (dialcode.isEmpty) throw new Exception("Invalid dialcode to update") - val query: String = s"insert into ${extConfig.keyspace}.${extConfig.table} (filename,channel,created_on,dialcode,status) values ('0_${dialcode}', '${channel}', ${ets}, '${dialcode}', 0);" - if (cassandraUtil.upsert(query)) { - logger.info(s"Added Dialcode to the table.") - true - } else { - logger.error("There was an issue while inserting the dialcode details into table") - throw new Exception("There was an issue while inserting the dialcode details into table") - } - } - - def createQRGeneratorEvent(edata: java.util.Map[String, AnyRef], dialcode: String, context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, config: PostPublishProcessorConfig)(implicit metrics: Metrics, extConfig: ExtDataConfig, cassandraUtil: CassandraUtil): Unit = { - logger.info("Generating event for QR Image Generation.") - val ets = System.currentTimeMillis - val identifier = edata.get("identifier").asInstanceOf[String] - val channelId = edata.getOrDefault("channel", "").asInstanceOf[String] - - if (updateDialcodeRecord(dialcode, channelId, ets)) { - val event = s"""{"eid":"BE_QR_IMAGE_GENERATOR", "objectId": "${identifier}", "dialcodes": [{"data": "${config.dialBaseUrl}${dialcode}", "text": "${dialcode}", "id": "0_${dialcode}"}], "storage": {"container": "dial", "path": "${channelId}/", "fileName": "${identifier}_${ets}"}, "config": {"errorCorrectionLevel": "H", "pixelsPerBlock": 2, "qrCodeMargin": 3, "textFontName": "Verdana", "textFontSize": 11, "textCharacterSpacing": 0.1, "imageFormat": "png", "colourModel": "Grayscale", "imageBorderSize": 1}}""".stripMargin - logger.info(s"QR Image Generator Event Object : ${event}") - context.output(config.generateQRImageOutTag, event) - metrics.incCounter(config.qrImageGeneratorEventCount) - } - - } - - def getDialCodeDetails(identifier: String, event: Event)(implicit neo4JUtil: Neo4JUtil, config: PostPublishProcessorConfig): util.Map[String, AnyRef] = { - logger.info("Process Dialcode Link for content: " + identifier) - val metadata = neo4JUtil.getNodeProperties(identifier) - - if (validatePrimaryCategory(metadata)(config)) { - logger.info(s"Primary Category match found. Starting the process for Dial Code Generation.") - Map[String, AnyRef]("identifier" -> identifier, - "primaryCategory" -> metadata.getOrDefault("primaryCategory", ""), - "contentType" -> metadata.getOrDefault("contentType", ""), - "channel" -> metadata.getOrDefault("channel", ""), - "dialcodes" -> metadata.getOrDefault("dialcodes", new util.ArrayList[String]()), - "reservedDialcodes" -> metadata.getOrDefault("reservedDialcodes", "{}")).asJava - } else { - logger.info(s"Primary Category does not match. Skipping the process for Dial Code Generation.") - new util.HashMap[String, AnyRef]() - } - } - - def getDialCodeContextMap(event: Event): util.Map[String, AnyRef] = { - val dialcodeContextMap = new util.HashMap[String, AnyRef]() - - if(event.eData.contains("addContextDialCodes")) dialcodeContextMap.put("addContextDialCodes", event.eData.getOrElse("addContextDialCodes", Map.empty)) - if(event.eData.contains("removeContextDialCodes")) dialcodeContextMap.put("removeContextDialCodes", event.eData.getOrElse("removeContextDialCodes", Map.empty)) - if(event.eData.contains("channel")) dialcodeContextMap.put("channel", event.eData.getOrElse("channel", "").asInstanceOf[String]) - - dialcodeContextMap - } - - def generateDialcodeContextUpdaterEvent(channel: String, addContextDialCodes: Map[List[String],String], removeContextDialCodes: Map[List[String],String], context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, metrics: Metrics)(implicit config: PostPublishProcessorConfig) = { - if(removeContextDialCodes.nonEmpty) { - removeContextDialCodes.foreach(rec => { - dialcodeContextUpdaterEvent(channel, rec._1, rec._2, context, "dialcode-context-delete")(metrics, config) - }) - } - if(addContextDialCodes.nonEmpty) { - addContextDialCodes.foreach(rec => { - dialcodeContextUpdaterEvent(channel, rec._1, rec._2, context, "dialcode-context-update")(metrics, config) - }) - } - - } - - def dialcodeContextUpdaterEvent(channel: String, dialcodes: List[String], contentId: String, context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, action: String)(implicit metrics: Metrics, config: PostPublishProcessorConfig) = { - dialcodes.foreach(dialcode => { - val epochTime = System.currentTimeMillis - val event = s"""{"eid":"BE_JOB_REQUEST","ets":$epochTime,"mid":"LP.$epochTime.${UUID.randomUUID()}","actor":{"id":"DIAL code context update Job","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"$channel","env":"dev"},"object":{"ver":"1.0","id":"$dialcode"},"edata":{"action":"$action","iteration":1,"dialcode":"$dialcode","identifier": "$contentId"},"identifier": "$contentId"}""" - context.output(config.dialcodeContextUpdaterOutTag, event) - metrics.incCounter(config.dialcodeContextUpdaterCount) - logger.info("DialHelper:: dialcodeContextUpdaterEvent:: Dial code context update event: " + event) - }) - - } - -} \ No newline at end of file diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/ShallowCopyPublishing.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/ShallowCopyPublishing.scala deleted file mode 100644 index 783fc8983..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/ShallowCopyPublishing.scala +++ /dev/null @@ -1,38 +0,0 @@ -package org.sunbird.job.postpublish.helpers - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.postpublish.functions.PublishMetadata -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.{HttpUtil, JSONUtil} - -trait ShallowCopyPublishing{ - - private[this] val logger = LoggerFactory.getLogger(classOf[ShallowCopyPublishing]) - - def getShallowCopiedContents(identifier: String)(implicit config: PostPublishProcessorConfig, httpUtil: HttpUtil): List[PublishMetadata] = { - logger.info("Process Shallow Copy for content: " + identifier) - val httpRequest = s"""{"request":{"filters":{"status":["Draft","Review","Live","Unlisted","Failed"],"origin":"${identifier}"},"fields":["identifier","mimeType","contentType","versionKey","channel","status","pkgVersion","lastPublishedBy","origin","originData"]}}""" - val httpResponse = httpUtil.post(config.searchAPIPath, httpRequest) - if (httpResponse.status == 200) { - val response = JSONUtil.deserialize[Map[String, AnyRef]](httpResponse.body) - val result = response.getOrElse("result", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - val contents = result.getOrElse("content", List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String,AnyRef]]] - contents.filter(c => c.contains("originData")) - .filter(content => { - val originDataStr = content.getOrElse("originData", "{}").asInstanceOf[String] - val originData = JSONUtil.deserialize[Map[String, AnyRef]](originDataStr) - val copyType = originData.getOrElse("copyType", "deep").asInstanceOf[String] - (StringUtils.equalsIgnoreCase(copyType, "shallow")) - }).map(content => { - val copiedId = content("identifier").asInstanceOf[String] - val copiedMimeType = content("mimeType").asInstanceOf[String] - val copiedPKGVersion = content.getOrElse("pkgVersion", 0.asInstanceOf[AnyRef]).asInstanceOf[Number] - val copiedContentType = content("contentType").asInstanceOf[String] - PublishMetadata(copiedId, copiedContentType, copiedMimeType, copiedPKGVersion.intValue()) - }) - } else { - throw new Exception("Content search failed for shallow copy check:" + identifier) - } - } -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/models/Models.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/models/Models.scala deleted file mode 100644 index d89279d9a..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/models/Models.scala +++ /dev/null @@ -1,12 +0,0 @@ -package org.sunbird.job.postpublish.models - -class ObjectData(val identifier: String, val metadata: Map[String, AnyRef], val extData: Option[Map[String, AnyRef]] = None, val hierarchy: Option[Map[String, AnyRef]] = None) { - - val dbId = metadata.getOrElse("IL_UNIQUE_ID", identifier).asInstanceOf[String] - - - val pkgVersion = metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number].intValue() - -} - -case class ExtDataConfig(keyspace: String, table: String) \ No newline at end of file diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala deleted file mode 100644 index b5f46abdd..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala +++ /dev/null @@ -1,87 +0,0 @@ -package org.sunbird.job.postpublish.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.postpublish.functions.PublishMetadata - -import java.util - -class PostPublishProcessorConfig(override val config: Config) extends BaseJobConfig(config, "post-publish-processor") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val publishMetaTypeInfo: TypeInformation[PublishMetadata] = TypeExtractor.getForClass(classOf[PublishMetadata]) - - // Job Configuration - val jobEnv: String = config.getString("job.env") - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val contentPublishTopic: String = config.getString("kafka.publish.topic") - - val inputConsumerName = "post-publish-event-consumer" - - // Parallelism - val eventRouterParallelism: Int = config.getInt("task.router.parallelism") - val shallowCopyParallelism: Int = config.getInt("task.shallow_copy.parallelism") - val linkDialCodeParallelism: Int = config.getInt("task.link_dialcode.parallelism") - val batchCreateParallelism: Int = config.getInt("task.batch_create.parallelism") - val dialcodeContextUpdaterParallelism: Int = config.getInt("task.dialcode_context_updater.parallelism") - - // Metric List - val totalEventsCount = "total-events-count" - val skippedEventCount = "skipped-events-count" - val batchCreationCount = "batch-creation-count" - val batchCreationSuccessCount = "batch-creation-success-count" - val batchCreationFailedCount = "batch-creation-failed-count" - val shallowCopyCount = "shallow-copy-count" - val dialLinkCount = "dial-link-count" - val dialLinkSuccessCount = "dial-link-success-count" - val dialLinkFailedCount = "dial-link-failed-count" - val qrImageGeneratorEventCount = "qr-image-event-count" - val dialcodeContextUpdaterCount = "dial-context-count" - val dialcodeContextUpdaterSuccessCount = "dial-context-success-count" - val dialcodeContextUpdaterFailedCount = "dial-context-failed-count" - - // Cassandra Configurations - val dbHost: String = config.getString("lms-cassandra.host") - val dbPort: Int = config.getInt("lms-cassandra.port") - val lmsKeyspaceName: String = config.getString("lms-cassandra.keyspace") - val batchTableName: String = config.getString("lms-cassandra.batchTable") - val dialcodeKeyspaceName: String = config.getString("dialcode-cassandra.keyspace") - val dialcodeTableName: String = config.getString("dialcode-cassandra.imageTable") - - // Neo4J Configurations - val graphRoutePath: String = config.getString("neo4j.routePath") - val graphName: String = config.getString("neo4j.graph") - - - // Tags - val batchCreateOutTag: OutputTag[java.util.Map[String, AnyRef]] = OutputTag[java.util.Map[String, AnyRef]]("batch-create") - val linkDIALCodeOutTag: OutputTag[java.util.Map[String, AnyRef]] = OutputTag[java.util.Map[String, AnyRef]]("dialcode-link") - val dialcodeContextOutTag: OutputTag[java.util.Map[String, AnyRef]] = OutputTag[java.util.Map[String, AnyRef]]("dialcode-context") - val shallowContentPublishOutTag: OutputTag[PublishMetadata] = OutputTag[PublishMetadata]("shallow-copied-content-publish") - val publishEventOutTag: OutputTag[String] = OutputTag[String]("content-publish-request") - val generateQRImageOutTag: OutputTag[String] = OutputTag[String]("qr-image-generator-request") - val dialcodeContextUpdaterOutTag: OutputTag[String] = OutputTag[String]("dialcode-context-updater") - - val searchBaseUrl: String = config.getString("service.search.basePath") - val lmsBaseUrl: String = config.getString("service.lms.basePath") - val learningBaseUrl: String = config.getString("service.learning_service.basePath") - val dialBaseUrl: String = config.getString("service.dial.basePath") - - // API URLs - val batchCreateAPIPath: String = lmsBaseUrl + "/private/v1/course/batch/create" - val searchAPIPath: String = searchBaseUrl + "/v3/search" - val reserveDialCodeAPIPath: String = learningBaseUrl + "/content/v3/dialcode/reserve" - - // QR Image Generator - val QRImageGeneratorTopic: String = config.getString("kafka.qrimage.topic") - val primaryCategories: util.List[String] = if (config.hasPath("dialcode.linkable.primaryCategory")) config.getStringList("dialcode.linkable.primaryCategory") else util.Arrays.asList("Course") //List[String]("Course") - val dialcodeContextUpdaterTopic: String = config.getString("kafka.dialcode.context.topic") - -} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala deleted file mode 100644 index 582c1dd98..000000000 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala +++ /dev/null @@ -1,78 +0,0 @@ -package org.sunbird.job.postpublish.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.postpublish.domain.Event -import org.sunbird.job.postpublish.functions._ -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - -import java.io.File -import java.util - -class PostPublishProcessorStreamTask(config: PostPublishProcessorConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val publishMetaTypeInfo: TypeInformation[PublishMetadata] = TypeExtractor.getForClass(classOf[PublishMetadata]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - - val processStreamTask = env.addSource(source).name(config.inputConsumerName) - .uid(config.inputConsumerName).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new PostPublishEventRouter(config, httpUtil)) - .name("post-publish-event-router").uid("post-publish-event-router") - .setParallelism(config.eventRouterParallelism) - - processStreamTask.getSideOutput(config.batchCreateOutTag).process(new BatchCreateFunction(config, httpUtil)) - .name("batch-create-process").uid("batch-create-process").setParallelism(config.batchCreateParallelism) - - val shallowCopyPublishStream = processStreamTask.getSideOutput(config.shallowContentPublishOutTag) - .process(new ShallowCopyPublishFunction(config)) - .name("shallow-content-publish").uid("shallow-content-publish") - .setParallelism(config.shallowCopyParallelism) - - shallowCopyPublishStream.getSideOutput(config.publishEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.contentPublishTopic)) - .name("shallow-content-publish-producer").uid("shallow-content-publish-producer") - - val linkDialCodeStream = processStreamTask.getSideOutput(config.linkDIALCodeOutTag) - .process(new DIALCodeLinkFunction(config, httpUtil)) - .name("dialcode-link-process").uid("dialcode-link-process").setParallelism(config.linkDialCodeParallelism) - - linkDialCodeStream.getSideOutput(config.generateQRImageOutTag).addSink(kafkaConnector.kafkaStringSink(config.QRImageGeneratorTopic)) - - val dialcodeContextUpdaterStream = processStreamTask.getSideOutput(config.dialcodeContextOutTag) - .process(new DialCodeContextUpdaterFunction(config)) - .name("dialcode-context-updater-process").uid("dialcode-context-updater-process").setParallelism(config.dialcodeContextUpdaterParallelism) - - dialcodeContextUpdaterStream.getSideOutput(config.dialcodeContextUpdaterOutTag).addSink(kafkaConnector.kafkaStringSink(config.dialcodeContextUpdaterTopic)) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object PostPublishProcessorStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("post-publish-processor.conf").withFallback(ConfigFactory.systemEnvironment())) - val pppConfig = new PostPublishProcessorConfig(config) - val kafkaUtil = new FlinkKafkaConnector(pppConfig) - val httpUtil = new HttpUtil - val task = new PostPublishProcessorStreamTask(pppConfig, kafkaUtil, httpUtil) - task.process() - } - -} - -// $COVERAGE-ON$ \ No newline at end of file diff --git a/post-publish-processor/src/test/resources/logback-test.xml b/post-publish-processor/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/post-publish-processor/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/post-publish-processor/src/test/resources/test.conf b/post-publish-processor/src/test/resources/test.conf deleted file mode 100644 index f7e316490..000000000 --- a/post-publish-processor/src/test/resources/test.conf +++ /dev/null @@ -1,39 +0,0 @@ -include "base-test.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.content.postpublish.request" - groupId = "local-post-publish-processor-group" - publish.topic = "sunbirddev.learning.job.request" - qrimage.topic = "sunbirddev.qrimage.request" - dialcode.context.topic = "sunbirddev.dialcode.context.job.request" -} - -task { - consumer.parallelism = 1 - router.parallelism = 1 - shallow_copy.parallelism = 1 - link_dialcode.parallelism = 1 - batch_create.parallelism = 1 - dialcode_context_updater.parallelism = 1 -} - -lms-cassandra { - keyspace = "sunbird_courses" - batchTable = "course_batch" -} - -dialcode-cassandra { - keyspace = "dialcodes" - imageTable = "dialcode_images" -} - -service { - search.basePath = "http://localhost/search" - lms.basePath = "http://localhost/lms" - learning_service.basePath = "https://localhost" - dial.basePath = "https://localhost/dial/" -} diff --git a/post-publish-processor/src/test/resources/test.cql b/post-publish-processor/src/test/resources/test.cql deleted file mode 100644 index ae21fcf64..000000000 --- a/post-publish-processor/src/test/resources/test.cql +++ /dev/null @@ -1,35 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; - -CREATE TABLE IF NOT EXISTS sunbird_courses.course_batch ( - courseid text, - batchid text, - cert_templates map>>, - createdby text, - createddate text, - createdfor list, - description text, - enddate text, - enrollmentenddate text, - enrollmenttype text, - mentors list, - name text, - startdate text, - status int, - updateddate text, - PRIMARY KEY (courseid, batchid) -); - -CREATE KEYSPACE IF NOT EXISTS dialcodes with replication = {'class':'SimpleStrategy','replication_factor':1}; - -CREATE TABLE IF NOT EXISTS dialcodes.dialcode_images ( - filename text PRIMARY KEY, - channel text, - config map, - created_on timestamp, - dialcode text, - publisher text, - status int, - url text -); - -INSERT INTO dialcodes.dialcode_images(filename, channel, dialcode, url) VALUES ('0_Q1I5I3', 'b00bc992ef25f1a9a8d63291e20efc8d', 'Q1I5I3', 'https://sunbirddev.blob.core.windows.net/sunbird-content-dev/in.ekstep/0_Q1I5I3.png') ; diff --git a/post-publish-processor/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/post-publish-processor/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index 887b4f237..000000000 --- a/post-publish-processor/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,29 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1599417126869,"mid":"LP.1599417126869.b5a197ab-4111-4230-936b-39c1c67dbab0","actor":{"id":"Post Publish Processor","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"b00bc992ef25f1a9a8d63291e20efc8d","env":"sunbirddev"},"object":{"ver":"1599417119724","id":"do_11300581751853056018"},"edata":{"trackable":"{\"enabled\":\"Yes\",\"autoBatch\":\"Yes\"}","identifier":"do_11300581751853056018","createdFor":["ORG_001"],"createdBy":"874ed8a5-782e-4f6c-8f36-e0288455901e","name":"Origin Content","action":"post-publish-process","iteration":1,"id":"do_11300581751853056018","mimeType":"application/vnd.ekstep.content-collection","contentType":"TextBook","pkgVersion":19.0,"status":"Live"}} - |""".stripMargin - - val QREVENT_1: String = - """ - |{"eid": "BE_JOB_REQUEST","ets": 1613105152461,"mid": "LP.1613105152461.044714c5-a4db-4e98-9b52-609aa199cde6","actor": {"id": "Post Publish Processor","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.ekstep.platform"},"channel": "b00bc992ef25f1a9a8d63291e20efc8d","env": "sunbirddev"},"object": {"ver": "1613105149590","id": "do_113214556543234"},"edata": {"trackable": "{\"enabled\":\"No\",\"autoBatch\":\"No\"}","identifier": "do_113214556543234","createdFor": ["ORG_001"],"createdBy": "874ed8a5-782e-4f6c-8f36-e0288455901e","name": "FITB","action": "post-publish-process","iteration": 1,"id": "do_113214556543234","mimeType": "application/vnd.ekstep.content-collection","contentType": "Course","pkgVersion": 1,"status": "Live"}} - |""".stripMargin - - val QREVENT_2: String = - """ - |{"eid": "BE_JOB_REQUEST","ets": 1613105152461,"mid": "LP.1613105152461.044714c5-a4db-4e98-9b52-609aa199cde6","actor": {"id": "Post Publish Processor","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.ekstep.platform"},"channel": "b00bc992ef25f1a9a8d63291e20efc8d","env": "sunbirddev"},"object": {"ver": "1613105149590","id": "do_113214556543235"},"edata": {"trackable": "{\"enabled\":\"No\",\"autoBatch\":\"No\"}","identifier": "do_113214556543235","createdFor": ["ORG_001"],"createdBy": "874ed8a5-782e-4f6c-8f36-e0288455901e","name": "FITB","action": "post-publish-process","iteration": 1,"id": "do_113214556543235","mimeType": "application/vnd.ekstep.content-collection","contentType": "Course","pkgVersion": 1,"status": "Live"}} - |""".stripMargin - - val QREVENT_3: String = - """ - |{"eid": "BE_JOB_REQUEST","ets": 1613105152461,"mid": "LP.1613105152461.044714c5-a4db-4e98-9b52-609aa199cde6","actor": {"id": "Post Publish Processor","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.ekstep.platform"},"channel": "b00bc992ef25f1a9a8d63291e20efc8d","env": "sunbirddev"},"object": {"ver": "1613105149590","id": "do_113214556543236"},"edata": {"trackable": "{\"enabled\":\"No\",\"autoBatch\":\"No\"}","identifier": "do_113214556543236","createdFor": ["ORG_001"],"createdBy": "874ed8a5-782e-4f6c-8f36-e0288455901e","name": "FITB","action": "post-publish-process","iteration": 1,"id": "do_113214556543236","mimeType": "application/vnd.ekstep.content-collection","contentType": "Course","pkgVersion": 1,"status": "Live"}} - |""".stripMargin - - val QREVENT_4: String = - """ - |{"eid": "BE_JOB_REQUEST","ets": 1613105152461,"mid": "LP.1613105152461.044714c5-a4db-4e98-9b52-609aa199cde6","actor": {"id": "Post Publish Processor","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.ekstep.platform"},"channel": "b00bc992ef25f1a9a8d63291e20efc8d","env": "sunbirddev"},"object": {"ver": "1613105149590","id": "do_113214556543237"},"edata": {"trackable": "{\"enabled\":\"No\",\"autoBatch\":\"No\"}","identifier": "do_113214556543237","createdFor": ["ORG_001"],"createdBy": "874ed8a5-782e-4f6c-8f36-e0288455901e","name": "FITB","action": "post-publish-process","iteration": 1,"id": "do_113214556543237","mimeType": "application/vnd.ekstep.content-collection","contentType": "Course","pkgVersion": 1,"status": "Live"}} - |""".stripMargin -} diff --git a/post-publish-processor/src/test/scala/org/sunbird/job/postpublish/helpers/DialHelperTest.scala b/post-publish-processor/src/test/scala/org/sunbird/job/postpublish/helpers/DialHelperTest.scala deleted file mode 100644 index aecae381e..000000000 --- a/post-publish-processor/src/test/scala/org/sunbird/job/postpublish/helpers/DialHelperTest.scala +++ /dev/null @@ -1,200 +0,0 @@ -package org.sunbird.job.postpublish.helpers - -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito.{doNothing, when} -import org.mockito.{ArgumentMatchers, Mockito} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.Metrics -import org.sunbird.job.postpublish.domain.Event -import org.sunbird.job.postpublish.models.ExtDataConfig -import org.sunbird.job.postpublish.task.PostPublishProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil, Neo4JUtil} - -import java.util -import scala.collection.JavaConverters._ - -class DialHelperTest extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val neo4jUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val httpUtil: HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val jobConfig: PostPublishProcessorConfig = new PostPublishProcessorConfig(config) - implicit val extConfig: ExtDataConfig = new ExtDataConfig(jobConfig.dialcodeKeyspaceName, jobConfig.dialcodeTableName) - implicit val metrics: Metrics = new Metrics(null) - - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.dbHost, jobConfig.dbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => { - } - } - } - - - "fetchExistingReservedDialcodes" should "return map of dialcodes that is reserved" in { - val dialUtility = new TestDialHelper() - when(neo4jUtil.getNodeProperties(ArgumentMatchers.anyString())).thenReturn(getNeo4jData()) - val dialcodes: util.Map[String, Integer] = dialUtility.fetchExistingReservedDialcodes(getEvent()) - dialcodes.isEmpty should be(false) - dialcodes.getOrDefault("Q1I5I3", -1) shouldEqual (0) - } - - "reserveDialCodes" should "reserve dialcodes and return a map of the same " in { - val dialUtility = new TestDialHelper() - when(httpUtil.post(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.any())).thenReturn(getReserveDialcodeResponse()) - val reserved = dialUtility.reserveDialCodes(getEvent(), jobConfig) - reserved.isEmpty should be(false) - reserved.getOrDefault("V9E4D9", -1) shouldEqual (0) - } - - "reserveDialCodes with error response" should "throw an exception " in { - val dialUtility = new TestDialHelper() - when(httpUtil.post(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.any())).thenReturn(HTTPResponse(400, "")) - intercept[Exception] { - dialUtility.reserveDialCodes(getEvent(), jobConfig) - } - } - - "updateDIALToObject" should "update the neo4j data" in { - doNothing().when(neo4jUtil).updateNodeProperty(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()) - val dialUtility = new TestDialHelper() - dialUtility.updateDIALToObject("do_234", "Q1I5I3") - } - - "fetchExistingDialcodes" should "return map of dialcodes that is reserved" in { - val dialcodes: util.List[String] = new TestDialHelper().fetchExistingDialcodes(getNeo4jData()) - dialcodes.isEmpty should be(false) - dialcodes should contain("Q1I5I3") - } - - "validatePrimaryCategory" should "validate the PrimaryCategory" in { - new TestDialHelper().validatePrimaryCategory(new util.HashMap[String, AnyRef]() { - { - put("primaryCategory", "Course") - } - }) should be(true) - } - - "validateQR" should "throw exception" in { - assertThrows[Exception] { - new TestDialHelper().validateQR("") - } - } - - "validateQR" should "validate the QR code image with valid Dialcode" in { - new TestDialHelper().validateQR("Q1I5I3") should be(true) - } - - "validateQR" should "invalidate the QR code image with invalid Dialcode" in { - new TestDialHelper().validateQR("Q1I5I2") should be(false) - } - - "updateDialcodeRecord" should "throw exception because of invalid channel" in { - assertThrows[Exception] { - new TestDialHelper().updateDialcodeRecord("", "channel1", System.currentTimeMillis()) - } - } - - "updateDialcodeRecord" should "throw exception because of passing invalid keyspace" in { - val extConfig = new ExtDataConfig("invalidKeyspace", jobConfig.dialcodeTableName) - assertThrows[Exception] { - new TestDialHelper().updateDialcodeRecord("123", "channel1", 123)(extConfig, cassandraUtil) - } - } - - "updateDialcodeRecord" should "pass by inserting record into table" in { - (new TestDialHelper().updateDialcodeRecord("Q1I5I2", "channel1", System.currentTimeMillis())) should be(true) - } - - "createQRGeneratorEvent" should " throw Exception because of passing invalid keyspaces" in { - val extConfig = new ExtDataConfig("invalidKeyspace", jobConfig.dialcodeTableName) - assertThrows[Exception] { - new TestDialHelper().createQRGeneratorEvent(new util.HashMap[String, AnyRef](), "", null, jobConfig)(metrics, extConfig, cassandraUtil) - } - - } - - "getDialCodeContextMap" should "return map of add and remove dial codes for context update" in { - val DIALCODE_CONTEXT_EVENT: String = """{"eid":"BE_JOB_REQUEST", "ets": 1655115290761, "mid": "LP.1655115290761.3d71f870-0f87-4a92-b1cd-96ff4fd977e8", "actor": {"id": "Post Publish Processor", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}, "channel":"01309282781705830427","env":"sunbirddev"},"object":{"ver":"1655115235618","id":"do_113556562556149760174"},"edata": {"action":"post-publish-process","iteration":1,"identifier":"do_113556562556149760174","channel":"01309282781705830427","mimeType":"application/vnd.ekstep.content-collection","contentType":"TextBook","pkgVersion":3,"status":"Live","name":"DIAL Context 7","trackable":{"enabled":"No","autoBatch":"No"}, "addContextDialCodes": {"[\"D4G2L3\"]":"do_113556562556149760174","[\"G3L9S2\"]":"do_113556563202981888177"}, "removeContextDialCodes": {"[\"C9N8K3\"]":"do_113556563202891776175"} }}""".stripMargin - val event = new Event(JSONUtil.deserialize[java.util.Map[String, Any]](DIALCODE_CONTEXT_EVENT),0,1) - val dialcodeMap = new TestDialHelper().getDialCodeContextMap(event) - assert(dialcodeMap.containsKey("removeContextDialCodes")) - assert(dialcodeMap.containsKey("addContextDialCodes")) - assert(dialcodeMap.containsKey("channel")) - } - - - - def getEvent(): util.Map[String, AnyRef] = { - Map[String, AnyRef]("reservedDialcodes" -> "{\"Q1I5I3\":0}", "identifier" -> "do_234", "channel" -> "in.ekstep").asJava - } - - def getInvalidEvent(): util.Map[String, AnyRef] = { - Map[String, AnyRef]("reservedDialcodes" -> "{\"Q1I5I3\":0}", "channel" -> "in.ekstep").asJava - } - - def getNeo4jData(): util.Map[String, AnyRef] = { - Map[String, AnyRef]("reservedDialcodes" -> "{\"Q1I5I3\":0}", "identifier" -> "do_234", "dialcodes" -> (List[String]("Q1I5I3")).asJava).asJava - } - - def getEmptyNeo4jData(): util.Map[String, AnyRef] = Map[String, AnyRef]().asJava - - def getReserveDialcodeResponse() = { - val body = - """ - |{ - | "id": "ekstep.learning.content.dialcode.reserve", - | "ver": "3.0", - | "ts": "2021-02-03T10:11:05ZZ", - | "params": { - | "resmsgid": "4f83734c-7b05-4ebb-b01e-121a365d4399", - | "msgid": null, - | "err": null, - | "status": "successful", - | "errmsg": null - | }, - | "responseCode": "OK", - | "result": { - | "count": 1, - | "reservedDialcodes": { - | "V9E4D9": 0 - | }, - | "node_id": "do_234", - | "versionKey": "1612347048689" - | } - |} - """.stripMargin - HTTPResponse(200, body) - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } -} - - -class TestDialHelper extends DialHelper { - -} \ No newline at end of file diff --git a/post-publish-processor/src/test/scala/org/sunbird/job/spec/PostPublishProcessorTaskTestSpec.scala b/post-publish-processor/src/test/scala/org/sunbird/job/spec/PostPublishProcessorTaskTestSpec.scala deleted file mode 100644 index 0723ec59f..000000000 --- a/post-publish-processor/src/test/scala/org/sunbird/job/spec/PostPublishProcessorTaskTestSpec.scala +++ /dev/null @@ -1,355 +0,0 @@ -package org.sunbird.job.spec - -import com.google.gson.Gson -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.sink.SinkFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, anyString, endsWith} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.neo4j.driver.v1.StatementResult -import org.sunbird.job.Metrics -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.postpublish.domain.Event -import org.sunbird.job.postpublish.functions.{DIALCodeLinkFunction, DialCodeContextUpdaterFunction, PostPublishEventRouter} -import org.sunbird.job.postpublish.helpers.TestDialHelper -import org.sunbird.job.postpublish.task.{PostPublishProcessorConfig, PostPublishProcessorStreamTask} -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil, Neo4JUtil, ScalaJsonUtil} -import org.sunbird.spec.BaseTestSpec - -import java.time.format.DateTimeFormatter -import java.time.{ZoneId, ZonedDateTime} -import java.util -import scala.collection.JavaConverters._ - -class PostPublishProcessorTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - implicit val jobConfig: PostPublishProcessorConfig = new PostPublishProcessorConfig(config) - - implicit val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - // For Shallow Copy and Batch Creation - val startDate = ZonedDateTime.now(ZoneId.of("Asia/Kolkata")).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) - val searchRequestBody = s"""{"request":{"filters":{"status":["Draft","Review","Live","Unlisted","Failed"],"origin":"do_11300581751853056018"},"fields":["identifier","mimeType","contentType","versionKey","channel","status","pkgVersion","lastPublishedBy","origin","originData"]}}""" - val batchRequestBody = s"""{"request":{"createdFor":["ORG_001"],"createdBy":"874ed8a5-782e-4f6c-8f36-e0288455901e","name":"Origin Content","enrollmentType":"open","courseId":"do_11300581751853056018","startDate":"${startDate}"}}""" - // END - // For Dial Codes - val searchRequestForQRImageEvent1 = s"""{"request":{"filters":{"status":["Draft","Review","Live","Unlisted","Failed"],"origin":"do_113214556543234"},"fields":["identifier","mimeType","contentType","versionKey","channel","status","pkgVersion","lastPublishedBy","origin","originData"]}}""" - val searchRequestForQRImageEvent2 = s"""{"request":{"filters":{"status":["Draft","Review","Live","Unlisted","Failed"],"origin":"do_113214556543235"},"fields":["identifier","mimeType","contentType","versionKey","channel","status","pkgVersion","lastPublishedBy","origin","originData"]}}""" - val searchRequestForQRImageEvent3 = s"""{"request":{"filters":{"status":["Draft","Review","Live","Unlisted","Failed"],"origin":"do_113214556543236"},"fields":["identifier","mimeType","contentType","versionKey","channel","status","pkgVersion","lastPublishedBy","origin","originData"]}}""" - val searchRequestForQRImageEvent4 = s"""{"request":{"filters":{"status":["Draft","Review","Live","Unlisted","Failed"],"origin":"do_113214556543237"},"fields":["identifier","mimeType","contentType","versionKey","channel","status","pkgVersion","lastPublishedBy","origin","originData"]}}""" - val qrRequestUrl = "https://localhost/action/content/v3/dialcode/reserve/do_113214556543237" - val qrRequestBody = s"""{"request":{"dialcodes":{"count":1,"qrCodeSpec":{"errorCorrectionLevel":"H"}}}}""" - val qrRequestHeaders: Map[String, String] = Map[String, String]("X-Channel-Id" -> "b00bc992ef25f1a9a8d63291e20efc8d", - "Content-Type" -> "application/json") - // END - val gson = new Gson() - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.dbHost, jobConfig.dbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - flinkCluster.after() - } - - "Post Publish Processor" should "process and find shallow copied contents" in { - when(mockHttpUtil.post(endsWith("/v3/search"), anyString(), any())).thenReturn(HTTPResponse(200, """{"id":"api.search-service.search","ver":"3.0","ts":"2020-08-31T22:09:07ZZ","params":{"resmsgid":"bc9a8ac0-f67d-47d5-b093-2077191bf93b","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"count":5,"content":[{"identifier":"do_11301367667942195211854","origin":"do_11300581751853056018","channel":"b00bc992ef25f1a9a8d63291e20efc8d","originData":"{\"name\":\"Origin Content\",\"copyType\":\"deep\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"]}","mimeType":"application/vnd.ekstep.content-collection","contentType":"TextBook","objectType":"Content","status":"Draft","versionKey":"1588583579763"},{"identifier":"do_113005885057662976128","origin":"do_11300581751853056018","pkgVersion": 2,"channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632481597"},{"identifier":"do_113005885161611264130","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632475439"},{"identifier":"do_113005882957578240124","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632233649"},{"identifier":"do_113005820474007552111","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587624624051"}]}}""")) - val identifier = "do_11300581751853056018" - val list = new PostPublishEventRouter(jobConfig, mockHttpUtil).getShallowCopiedContents(identifier) - list.size should be(4) - list.map(c => c.identifier) should contain allOf("do_113005885057662976128", "do_113005885161611264130", "do_113005882957578240124", "do_113005820474007552111") - } - - "Post Publish Processor" should "process and return the metadata for batch " in { - val metaData = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_11300581751853056018") - put("identifier", "do_11300581751853056018") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"Yes\",\"autoBatch\":\"Yes\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - } - } - - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(metaData) - val identifier = "do_11300581751853056018" - val batchMetadata = new PostPublishEventRouter(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getBatchDetails(identifier)(mockNeo4JUtil, cassandraUtil, jobConfig) - batchMetadata.size() should be(4) - batchMetadata.get("identifier") should be("do_11300581751853056018") - } - - "Post Publish Processor" should "process and return the empty metadata for batch" in { - val metaData = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_11300581751853056018") - put("identifier", "do_11300581751853056018") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"false\",\"autoBatch\":\"false\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - } - } - - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(metaData) - val identifier = "do_11300581751853056018" - val batchMetadata = new PostPublishEventRouter(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getBatchDetails(identifier)(mockNeo4JUtil, cassandraUtil, jobConfig) - batchMetadata.isEmpty() should be(true) - } - - "Post Publish Processor" should "process and return the metadata for dialcode generation" in { - val metadata = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_113214556543234") - put("identifier", "do_113214556543234") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"No\",\"autoBatch\":\"No\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("reservedDialcodes", "{\"Q1I5I3\": 0}") - put("dialcodes", util.Arrays.asList("Q1I5I3")) - put("primaryCategory", "Course") - } - } - - val identifier = "do_113214556543234" - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(metadata) - val qrEventMap1 = gson.fromJson(EventFixture.QREVENT_1, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]].asScala ++ Map("partition" -> 0.asInstanceOf[Any]) - val event = new Event(qrEventMap1.asJava,0, 10) - val dialcodeMetadata = new PostPublishEventRouter(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getDialCodeDetails(identifier, event)(mockNeo4JUtil, jobConfig) - dialcodeMetadata.isEmpty() should be(false) - dialcodeMetadata.get("dialcodes").asInstanceOf[util.List[String]] should contain("Q1I5I3") - } - - "Post Publish Processor" should "process and return the empty metadata for dialcode generation" in { - val metadata = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_113214556543234") - put("identifier", "do_113214556543234") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"No\",\"autoBatch\":\"No\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("reservedDialcodes", "{\"Q1I5I3\": 0}") - put("dialcodes", util.Arrays.asList("Q1I5I3")) - put("primaryCategory", "Textbook") - } - } - - val identifier = "do_113214556543234" - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(metadata) - val qrEventMap1 = gson.fromJson(EventFixture.QREVENT_1, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]].asScala ++ Map("partition" -> 0.asInstanceOf[Any]) - val event = new Event(qrEventMap1.asJava, 0, 11) - val dialcodeMetadata = new PostPublishEventRouter(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getDialCodeDetails(identifier, event)(mockNeo4JUtil, jobConfig) - dialcodeMetadata.isEmpty() should be(true) - } - - - "Post Publish Processor" should "process request for dialcode generation return the nothing for QR Image " in { - val edata = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_113214556543234") - put("identifier", "do_113214556543234") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"No\",\"autoBatch\":\"No\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("reservedDialcodes", "{\"Q1I5I3\": 0}") - put("dialcodes", util.Arrays.asList("Q1I5I3")) - put("primaryCategory", "Course") - } - } - val dialcode = new DIALCodeLinkFunction(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getDialcode(edata) - dialcode should be("") - } - - "Post Publish Processor" should "process request for dialcode generation and return the dialcode for QR Image " in { - val edata = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_113214556543235") - put("identifier", "do_113214556543235") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"No\",\"autoBatch\":\"No\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("reservedDialcodes", "{\"Q1I5I4\": 0}") - put("dialcodes", util.Arrays.asList("Q1I5I4")) - put("primaryCategory", "Course") - } - } - val dialcode = new DIALCodeLinkFunction(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getDialcode(edata) - dialcode should be("Q1I5I4") - } - - "Post Publish Processor" should " use the existing reserved dialcode and return that dialcode for QR Image " in { - val edata = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_113214556543236") - put("identifier", "do_113214556543236") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"No\",\"autoBatch\":\"No\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("reservedDialcodes", "{\"Q1I5I5\": 0}") - put("primaryCategory", "Course") - } - } - doNothing().when(mockNeo4JUtil).updateNodeProperty(anyString, anyString, anyString) - val dialcode = new DIALCodeLinkFunction(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getDialcode(edata) - dialcode should be("Q1I5I5") - } - - - "Post Publish Processor" should " reserve a dialcode and return that dialcode for QR Image " in { - val edata = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_113214556543237") - put("identifier", "do_113214556543237") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"No\",\"autoBatch\":\"No\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("primaryCategory", "Course") - } - } - doNothing().when(mockNeo4JUtil).updateNodeProperty(anyString, anyString, anyString) - when(mockHttpUtil.post(anyString, anyString, any())).thenReturn(HTTPResponse(200, """{"result": {"reservedDialcodes": {"Q2I5I9" : 0}}}""")) - - val dialcode = new DIALCodeLinkFunction(jobConfig, mockHttpUtil, mockNeo4JUtil, cassandraUtil).getDialcode(edata) - dialcode should be("Q2I5I9") - } - - "getDialCodeContextMap" should "return map of add and remove dial codes for context update" in { - implicit val metrics: Metrics = new Metrics(null) - val DIALCODE_CONTEXT_EVENT: String = """{"eid":"BE_JOB_REQUEST", "ets": 1653307293906, "mid": "LP.1653307293906.c4ebcb3b-4828-4bf1-99fa-bd363f4ceae9", "actor": {"id": "Post Publish Processor", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}, "channel":"0126825293972439041","env":"sunbirddev"},"object":{"ver":"1652871771396","id":"do_21354027142511820812318.img"},"edata": {"action":"post-publish-process","iteration":1,"identifier":"do_21354027142511820812318.img","channel":"0126825293972439041","mimeType":"","contentType":"TextBook","pkgVersion":1,"status":"Draft","name":"DialCodeHierarchy","trackable":{}, "addContextDialCodes": {"[\"V2A8N1\"]":"do_21354027142511820812318","[\"L5L9A2\"]":"do_21354031968955596812322","[\"F6I6S6\"]":"do_21354032495652044812326","[\"R8C7G2\"]":"do_21354031968951500812320","[\"D1D5H9\"]":"do_21354032495648768012324"}, "removeContextDialCodes": {"[\"A3S8L6\"]":"do_21354031968955596812322"} }}""".stripMargin - val event = new Event(JSONUtil.deserialize[java.util.Map[String, Any]](DIALCODE_CONTEXT_EVENT),0,1) - val dialcodeContextMap = new DialCodeContextUpdaterFunction(jobConfig).getDialCodeContextMap(event) - - val addContextDialCodes: Map[List[String],String] = dialcodeContextMap.getOrDefault("addContextDialCodes", Map.empty[String,String]).asInstanceOf[Map[String,String]].map(rec => (ScalaJsonUtil.deserialize[List[String]](rec._1)->rec._2)) - val removeContextDialCodes: Map[List[String],String] = dialcodeContextMap.getOrDefault("removeContextDialCodes", Map.empty[String,String]).asInstanceOf[Map[String,String]].map(rec => (ScalaJsonUtil.deserialize[List[String]](rec._1)->rec._2)) - val channel: String = dialcodeContextMap.getOrDefault("channel","").asInstanceOf[String] - assertThrows[Exception] { - new DialCodeContextUpdaterFunction(jobConfig).generateDialcodeContextUpdaterEvent(channel, addContextDialCodes, removeContextDialCodes, null, metrics) - } - } - - ignore should "run all the scenarios for a given event" in { - when(mockHttpUtil.post(jobConfig.searchAPIPath, searchRequestBody)).thenReturn(HTTPResponse(200, """{"id":"api.search-service.search","ver":"3.0","ts":"2020-08-31T22:09:07ZZ","params":{"resmsgid":"bc9a8ac0-f67d-47d5-b093-2077191bf93b","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"count":5,"content":[{"identifier":"do_11301367667942195211854","origin":"do_11300581751853056018","channel":"b00bc992ef25f1a9a8d63291e20efc8d","originData":"{\"name\":\"Origin Content\",\"copyType\":\"deep\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"]}","mimeType":"application/vnd.ekstep.content-collection","contentType":"TextBook","objectType":"Content","status":"Draft","versionKey":"1588583579763"},{"identifier":"do_113005885057662976128","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632481597"},{"identifier":"do_113005885161611264130","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632475439"},{"identifier":"do_113005882957578240124","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632233649"},{"identifier":"do_113005820474007552111","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587624624051"}]}}""")) - when(mockHttpUtil.post(jobConfig.batchCreateAPIPath, batchRequestBody)).thenReturn(HTTPResponse(200, """{}""")) - when(mockKafkaUtil.kafkaStringSink(jobConfig.contentPublishTopic)).thenReturn(new PublishEventSink) - when(mockHttpUtil.post(qrRequestUrl, qrRequestBody, qrRequestHeaders)).thenReturn(HTTPResponse(200, """{"result": {"reservedDialcodes": {"Q2I5I9" : 0}}}""")) - when(mockHttpUtil.post(jobConfig.searchAPIPath, searchRequestForQRImageEvent1)).thenReturn(HTTPResponse(200, """{"responseCode": "OK","result": {"count": 5,"content": []}}""")) - when(mockHttpUtil.post(jobConfig.searchAPIPath, searchRequestForQRImageEvent2)).thenReturn(HTTPResponse(200, """{"responseCode": "OK","result": {"count": 5,"content": []}}""")) - when(mockHttpUtil.post(jobConfig.searchAPIPath, searchRequestForQRImageEvent3)).thenReturn(HTTPResponse(200, """{"responseCode": "OK","result": {"count": 5,"content": []}}""")) - when(mockHttpUtil.post(jobConfig.searchAPIPath, searchRequestForQRImageEvent4)).thenReturn(HTTPResponse(200, """{"responseCode": "OK","result": {"count": 5,"content": []}}""")) - when(mockKafkaUtil.kafkaStringSink(jobConfig.QRImageGeneratorTopic)).thenReturn(new QRImageEventSink) - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new PostPublishEventSource) - when(mockHttpUtil.post(endsWith("/v3/search"), anyString(), any())).thenReturn(HTTPResponse(200, """{"id":"api.search-service.search","ver":"3.0","ts":"2020-08-31T22:09:07ZZ","params":{"resmsgid":"bc9a8ac0-f67d-47d5-b093-2077191bf93b","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"count":5,"content":[{"identifier":"do_11301367667942195211854","origin":"do_11300581751853056018","channel":"b00bc992ef25f1a9a8d63291e20efc8d","originData":"{\"name\":\"Origin Content\",\"copyType\":\"deep\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"]}","mimeType":"application/vnd.ekstep.content-collection","contentType":"TextBook","objectType":"Content","status":"Draft","versionKey":"1588583579763"},{"identifier":"do_113005885057662976128","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632481597"},{"identifier":"do_113005885161611264130","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632475439"},{"identifier":"do_113005882957578240124","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587632233649"},{"identifier":"do_113005820474007552111","origin":"do_11300581751853056018","channel":"sunbird","originData":"{\"name\":\"Origin Content\",\"copyType\":\"shallow\",\"license\":\"CC BY 4.0\",\"organisation\":[\"Sunbird\"],\"pkgVersion\":2.0}","mimeType":"application/vnd.ekstep.content-collection","lastPublishedBy":"Ekstep","contentType":"TextBook","objectType":"Content","status":"Live","versionKey":"1587624624051"}]}}""")) - when(mockHttpUtil.post(endsWith("/private/v1/course/batch/create"), anyString(), any())).thenReturn(HTTPResponse(200, """{}""")) - val trackable = """{"enabled":"Yes","autoBatch":"Yes"}""" - when(mockNeo4JUtil.getNodeProperties("do_11300581751853056018")).thenReturn(Map("name" -> "Origin Content", "createdBy" -> "874ed8a5-782e-4f6c-8f36-e0288455901e", "createdFor" -> util.Arrays.asList("ORG_001"), "channel" -> "b00bc992ef25f1a9a8d63291e20efc8d", "trackable" -> trackable).asJava) - val mockResult = mock[StatementResult](Mockito.withSettings().serializable()) - when(mockNeo4JUtil.executeQuery(anyString())).thenReturn(mockResult) - val streamTask = new PostPublishProcessorStreamTask(jobConfig, mockKafkaUtil, mockHttpUtil) - streamTask.process() - - PublishEventSink.values.size() should be(4) - PublishEventSink.values.forEach(event => { - println("PUBLISH_EVENT: " + event) - }) - - QRImageEventSink.values.size() should be(3) - QRImageEventSink.values.forEach(event => { - println("QR_IMAGE_EVENT: " + event) - }) - } -} - -class PostPublishEventSource extends SourceFunction[Event] { - override def run(ctx: SourceContext[Event]): Unit = { - val gson = new Gson() - // Event for Batch Creation and ShallowCopy - val eventMap1 = gson.fromJson(EventFixture.EVENT_1, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]].asScala ++ Map("partition" -> 0.asInstanceOf[Any]) - ctx.collect(new Event(eventMap1.asJava,0, 10)) - // Event for Dial Codes - val qrEventMap1 = gson.fromJson(EventFixture.QREVENT_1, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]].asScala ++ Map("partition" -> 0.asInstanceOf[Any]) - ctx.collect(new Event(qrEventMap1.asJava,0, 11)) - val qrEventMap2 = gson.fromJson(EventFixture.QREVENT_2, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]].asScala ++ Map("partition" -> 0.asInstanceOf[Any]) - ctx.collect(new Event(qrEventMap2.asJava,0, 12)) - val qrEventMap3 = gson.fromJson(EventFixture.QREVENT_3, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]].asScala ++ Map("partition" -> 0.asInstanceOf[Any]) - ctx.collect(new Event(qrEventMap3.asJava,0, 13)) - val qrEventMap4 = gson.fromJson(EventFixture.QREVENT_4, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]].asScala ++ Map("partition" -> 0.asInstanceOf[Any]) - ctx.collect(new Event(qrEventMap4.asJava,0, 14)) - } - - override def cancel() = {} -} - -class PublishEventSink extends SinkFunction[String] { - - override def invoke(value: String): Unit = { - synchronized { - PublishEventSink.values.add(value) - } - } -} - -object PublishEventSink { - val values: util.List[String] = new util.ArrayList() -} - -class QRImageEventSink extends SinkFunction[String] { - - override def invoke(value: String): Unit = { - synchronized { - QRImageEventSink.values.add(value) - } - } -} - -object QRImageEventSink { - val values: util.List[String] = new util.ArrayList() -} diff --git a/publish-pipeline/content-publish/pom.xml b/publish-pipeline/content-publish/pom.xml deleted file mode 100644 index 83e74dfea..000000000 --- a/publish-pipeline/content-publish/pom.xml +++ /dev/null @@ -1,212 +0,0 @@ - - - - publish-pipeline - org.sunbird - 1.0 - - 4.0.0 - - content-publish - 1.0.0 - jar - content-publish - - Content publish job - - - - - - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - publish-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - jackson-module-scala_${scala.version} - com.fasterxml.jackson.module - - - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.sunbird.job.content.task.ContentPublishStreamTask - - - - reference.conf - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - content-publish-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/publish-pipeline/content-publish/src/main/resources/content-publish.conf b/publish-pipeline/content-publish/src/main/resources/content-publish.conf deleted file mode 100644 index 4f41bef70..000000000 --- a/publish-pipeline/content-publish/src/main/resources/content-publish.conf +++ /dev/null @@ -1,175 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.publish.job.request" - post_publish.topic = "sunbirddev.content.postpublish.request" - mvc.topic = "sunbirddev.mvc.processor.job.request" - error.topic = "sunbirddev.learning.events.failed" - groupId = "local-content-publish-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - router.parallelism = 1 -} - -redis { - database { - contentCache.id = 0 - } -} - -content { - bundleLocation = "/tmp/contentBundle" - isECARExtractionEnabled = true - retry_asset_download_count = 1 - keyspace = "dev_content_store" - table = "content_data" - tmp_file_location = "/tmp" - objectType = ["Content", "ContentImage","Collection","CollectionImage"] - mimeType = ["application/pdf", - "application/vnd.ekstep.ecml-archive", - "application/vnd.ekstep.html-archive", - "application/vnd.android.package-archive", - "application/vnd.ekstep.content-archive", - "application/epub", - "application/msword", - "application/vnd.ekstep.h5p-archive", - "video/webm", - "video/mp4", - "application/vnd.ekstep.content-collection", - "video/quicktime", - "application/octet-stream", - "application/json", - "application/javascript", - "application/xml", - "text/plain", - "text/html", - "text/javascript", - "text/xml", - "text/css", - "image/jpeg", - "image/jpg", - "image/png", - "image/tiff", - "image/bmp", - "image/gif", - "image/svg+xml", - "image/x-quicktime", - "video/avi", - "video/mpeg", - "video/quicktime", - "video/3gpp", - "video/mp4", - "video/ogg", - "video/webm", - "video/msvideo", - "video/x-msvideo", - "video/x-qtc", - "video/x-mpeg", - "audio/mp3", - "audio/mp4", - "audio/mpeg", - "audio/ogg", - "audio/webm", - "audio/x-wav", - "audio/wav", - "audio/mpeg3", - "audio/x-mpeg-3", - "audio/vorbis", - "application/x-font-ttf", - "application/vnd.ekstep.plugin-archive", - "video/x-youtube", - "video/youtube", - "text/x-url"] - asset_download_duration = "60 seconds" - - stream { - enabled = true - mimeType = ["video/mp4", "video/webm"] - } - artifact.size.for_online=209715200 - - downloadFiles { - spine = ["appIcon"] - full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"] - } - - nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions"] - -} - -hierarchy { - keyspace = "dev_hierarchy_store" - table = "content_hierarchy" -} - -cloud_storage { - folder { - content = "content" - artifact = "artifact" - } -} - -service { - print.basePath = "http://11.2.6.6/print" -} - - -contentTypeToPrimaryCategory { - ClassroomTeachingVideo: "Explanation Content" - ConceptMap: "Learning Resource" - Course: "Course" - CuriosityQuestionSet: "Practice Question Set" - eTextBook: "eTextbook" - Event: "Event" - EventSet: "Event Set" - ExperientialResource: "Learning Resource" - ExplanationResource: "Explanation Content" - ExplanationVideo: "Explanation Content" - FocusSpot: "Teacher Resource" - LearningOutcomeDefinition: "Teacher Resource" - MarkingSchemeRubric: "Teacher Resource" - PedagogyFlow: "Teacher Resource" - PracticeQuestionSet: "Practice Question Set" - PracticeResource: "Practice Question Set" - SelfAssess: "Course Assessment" - TeachingMethod: "Teacher Resource" - TextBook: "Digital Textbook" - Collection: "Content Playlist" - ExplanationReadingMaterial: "Learning Resource" - LearningActivity: "Learning Resource" - LessonPlan: "Content Playlist" - LessonPlanResource: "Teacher Resource" - PreviousBoardExamPapers: "Learning Resource" - TVLesson: "Explanation Content" - OnboardingResource: "Learning Resource" - ReadingMaterial: "Learning Resource" - Template: "Template" - Asset: "Asset" - Plugin: "Plugin" - LessonPlanUnit: "Lesson Plan Unit" - CourseUnit: "Course Unit" - TextBookUnit: "Textbook Unit" - Asset: "Certificate Template" -} - -max_allowed_content_name = 120 -enableDIALContextUpdate = "Yes" - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "CLOUD_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"] -cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] - -cloud_storage_type="azure" -cloud_storage_key="" -cloud_storage_secret="" -cloud_storage_container="" -cloud_storage_endpoint="" \ No newline at end of file diff --git a/publish-pipeline/content-publish/src/main/resources/log4j.properties b/publish-pipeline/content-publish/src/main/resources/log4j.properties deleted file mode 100644 index 602c83eb7..000000000 --- a/publish-pipeline/content-publish/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=-content-publish.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala deleted file mode 100644 index 5dd7e0f63..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala +++ /dev/null @@ -1,196 +0,0 @@ -package org.sunbird.job.content.function - -import akka.dispatch.ExecutionContexts -import com.google.gson.reflect.TypeToken -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.neo4j.driver.v1.exceptions.ClientException -import org.slf4j.LoggerFactory -import org.sunbird.job.cache.{DataCache, RedisConnect} -import org.sunbird.job.content.publish.domain.Event -import org.sunbird.job.content.publish.helpers.CollectionPublisher -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util._ -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type -import java.util.UUID -import scala.collection.mutable -import scala.concurrent.ExecutionContext - -class CollectionPublishFunction(config: ContentPublishConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var esUtil: ElasticSearchUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var definitionCache: DefinitionCache = null, - @transient var definitionConfig: DefinitionConfig = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with CollectionPublisher with FailedEventHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[CollectionPublishFunction]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - private var cache: DataCache = _ - private val COLLECTION_CACHE_KEY_PREFIX = "hierarchy_" - private val COLLECTION_CACHE_KEY_SUFFIX = ":leafnodes" - - @transient var ec: ExecutionContext = _ - private val pkgTypes = List(EcarPackageType.SPINE, EcarPackageType.ONLINE) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType) - cloudStorageUtil = new CloudStorageUtil(config) - ec = ExecutionContexts.global - definitionCache = new DefinitionCache() - definitionConfig = DefinitionConfig(config.schemaSupportVersionMap, config.definitionBasePath) - cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) - cache.init() - } - - override def close(): Unit = { - super.close() - cassandraUtil.close() - cache.close() - } - - override def metricsList(): List[String] = { - List(config.collectionPublishEventCount, config.collectionPublishSuccessEventCount, config.collectionPublishFailedEventCount, config.skippedEventCount, config.collectionPostPublishProcessEventCount) - } - - override def processElement(data: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - val definition: ObjectDefinition = definitionCache.getDefinition(data.objectType, config.schemaSupportVersionMap.getOrElse(data.objectType.toLowerCase(), "1.0").asInstanceOf[String], config.definitionBasePath) - val readerConfig = ExtDataConfig(config.hierarchyKeyspaceName, config.hierarchyTableName, definition.getExternalPrimaryKey, definition.getExternalProps) - logger.info("Collection publishing started for : " + data.identifier) - metrics.incCounter(config.collectionPublishEventCount) - val obj: ObjectData = getObject(data.identifier, data.pkgVersion, data.mimeType, data.publishType, readerConfig)(neo4JUtil, cassandraUtil, config) - logger.info(s"KN-856: Step:1 - From DB Collection: ${obj.identifier} | Hierarchy: ${obj.hierarchy}"); - try { - if (obj.pkgVersion > data.pkgVersion) { - metrics.incCounter(config.skippedEventCount) - logger.info(s"""pkgVersion should be greater than or equal to the obj.pkgVersion for : ${obj.identifier}""") - } else { - val updObj = new ObjectData(obj.identifier, obj.metadata ++ Map("lastPublishedBy" -> data.lastPublishedBy, "dialcodes" -> obj.metadata.getOrElse("dialcodes",null)), obj.extData, obj.hierarchy) - val messages: List[String] = List.empty[String] // validate(obj, obj.identifier, validateMetadata) - if (messages.isEmpty) { - // Pre-publish update - updateProcessingNode(updObj)(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig) - logger.info(s"KN-856: Step:2 - After updating processing status Collection: ${updObj.identifier} | Hierarchy: ${updObj.hierarchy}"); - val isCollectionShallowCopy = isContentShallowCopy(updObj) - val updatedObj = if (isCollectionShallowCopy) updateOriginPkgVersion(updObj)(neo4JUtil) else updObj - logger.info(s"KN-856: Step:3 - After shallow copy status check and update Collection: ${updatedObj.identifier} | isCollectionShallowCopy: $isCollectionShallowCopy | Hierarchy: ${updatedObj.hierarchy}"); - - // Clear redis cache - cache.del(data.identifier) - cache.del(data.identifier + COLLECTION_CACHE_KEY_SUFFIX) - cache.del(COLLECTION_CACHE_KEY_PREFIX + data.identifier) - - // Collection - add step to remove units of already Live content from redis - line 243 in PublishFinalizer - val unitNodes = if (obj.metadata("identifier").asInstanceOf[String].endsWith(".img")) { - val childNodes = getUnitsFromLiveContent(updatedObj)(cassandraUtil, readerConfig, config) - childNodes.filter(rec => rec.nonEmpty).foreach(childId => cache.del(COLLECTION_CACHE_KEY_PREFIX + childId)) - childNodes.filter(rec => rec.nonEmpty) - } else List.empty - - logger.info("CollectionPublishFunction:: Live unitNodes: " + unitNodes) - val enrichedObj = enrichObject(updatedObj)(neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig) - logger.info(s"KN-856: Step:4 - After enriching the object Collection: ${enrichedObj.identifier} | Hierarchy: ${enrichedObj.hierarchy}"); - logger.info("CollectionPublishFunction:: Collection Object Enriched: " + enrichedObj.identifier) - val objWithEcar = getObjectWithEcar(enrichedObj, pkgTypes)(ec, neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig, httpUtil) - logger.info("CollectionPublishFunction:: ECAR generation completed for Collection Object: " + objWithEcar.identifier) - logger.info(s"KN-856: Step:5 - After WithEcar Collection: ${objWithEcar.identifier} | Hierarchy: ${objWithEcar.hierarchy}"); - - val collRelationalMetadata = getRelationalMetadata(obj.identifier, obj.pkgVersion-1, readerConfig)(cassandraUtil, config).getOrElse(Map.empty[String, AnyRef]) - - val dialContextMap = if(config.enableDIALContextUpdate.equalsIgnoreCase("Yes")) fetchDialListForContextUpdate(obj)(neo4JUtil, cassandraUtil, readerConfig, config) else Map.empty[String, AnyRef] - logger.info("CollectionPublishFunction:: dialContextMap: " + dialContextMap) - - saveOnSuccess(new ObjectData(objWithEcar.identifier, objWithEcar.metadata.-("children"), objWithEcar.extData, objWithEcar.hierarchy))(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig, config) - logger.info("CollectionPublishFunction:: Published Collection Object metadata saved successfully to graph DB: " + objWithEcar.identifier) - - val variantsJsonString = ScalaJsonUtil.serialize(objWithEcar.metadata("variants")) - val publishType = objWithEcar.getString("publish_type", "Public") - val successObj = new ObjectData(objWithEcar.identifier, objWithEcar.metadata + ("status" -> (if (publishType.equalsIgnoreCase("Unlisted")) "Unlisted" else "Live"), "variants" -> variantsJsonString, "identifier" -> objWithEcar.identifier), objWithEcar.extData, objWithEcar.hierarchy) - val children = successObj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - logger.info(s"KN-856: Step:6 - After saveOnSuccess(Neo4J Save) Collection: ${successObj.identifier} | Hierarchy: $children"); - // Collection - update and publish children - line 418 in PublishFinalizer - val updatedChildren = updateHierarchyMetadata(children, successObj.metadata, collRelationalMetadata)(config) - logger.info(s"KN-856: Step:7 - After updateHierarchyMetadata Collection: ${successObj.identifier} | Hierarchy: $updatedChildren"); - logger.info("CollectionPublishFunction:: Hierarchy Metadata updated for Collection Object: " + successObj.identifier + " || updatedChildren:: " + updatedChildren) - publishHierarchy(updatedChildren, successObj, readerConfig, config)(cassandraUtil) - logger.info(s"KN-856: Step:8 - After publishHierarchy Collection: ${successObj.identifier} | Hierarchy: $updatedChildren"); - //TODO: Save IMAGE Object with enrichedObj children and collRelationalMetadata when pkgVersion is 1 - verify with MaheshG - if(data.pkgVersion == 1) { - saveImageHierarchy(enrichedObj, readerConfig, collRelationalMetadata)(cassandraUtil) - logger.info(s"KN-856: Step:8.1 - After saveImageHierarchy Collection: ${enrichedObj.identifier} | Hierarchy: ${enrichedObj.hierarchy}"); - } - - if (!isCollectionShallowCopy) syncNodes(successObj, updatedChildren, unitNodes)(esUtil, neo4JUtil, cassandraUtil, readerConfig, definition, config) - pushPostProcessEvent(successObj, dialContextMap, context)(metrics) - logger.info(s"KN-856: Step:9 - After pushPostProcessEvent Collection: ${successObj.identifier} | Hierarchy: ${successObj.hierarchy}"); - metrics.incCounter(config.collectionPublishSuccessEventCount) - logger.info("CollectionPublishFunction:: Collection publishing completed successfully for : " + data.identifier) - } else { - saveOnFailure(obj, messages, data.pkgVersion)(neo4JUtil) - val errorMessages = messages.mkString("; ") - pushFailedEvent(data, errorMessages, null, context)(metrics) - logger.info("CollectionPublishFunction:: Collection publishing failed for : " + data.identifier) - } - } - } catch { - case ex@(_: InvalidInputException | _: ClientException) => // ClientException - Invalid input exception. - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - pushFailedEvent(data, null, ex, context)(metrics) - logger.error(s"CollectionPublishFunction::Error while publishing collection :: ${data.partition} and Offset: ${data.offset}. Error : ${ex.getMessage}", ex) - case ex: Exception => - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - logger.error(s"CollectionPublishFunction::Error while processing message for Partition: ${data.partition} and Offset: ${data.offset}. Error : ${ex.getMessage}", ex) - throw ex - } - } - - private def pushPostProcessEvent(obj: ObjectData, dialContextMap: Map[String, AnyRef] ,context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - try { - val event = getPostProcessEvent(obj, dialContextMap) - context.output(config.generatePostPublishProcessTag, event) - metrics.incCounter(config.collectionPostPublishProcessEventCount) - } catch { - case ex: Exception => ex.printStackTrace() - throw new InvalidInputException("CollectionPublisher:: pushPostProcessEvent:: Error while pushing post process event.", ex) - } - } - - def getPostProcessEvent(obj: ObjectData, dialContextMap: Map[String, AnyRef]): String = { - val ets = System.currentTimeMillis - val mid = s"""LP.$ets.${UUID.randomUUID}""" - val channelId = obj.metadata("channel") - val ver = obj.metadata("versionKey") - val contentType = obj.metadata("contentType") - val status = obj.metadata("status") - - val serAddContextDialCode = ScalaJsonUtil.serialize(dialContextMap.getOrElse("addContextDialCodes", mutable.Map.empty).asInstanceOf[mutable.Map[List[String], String]].map(rec => (ScalaJsonUtil.serialize(rec._1) -> rec._2))) - val serRemoveContextDialCode = ScalaJsonUtil.serialize(dialContextMap.getOrElse("removeContextDialCodes", mutable.Map.empty).asInstanceOf[mutable.Map[List[String], String]].map(rec => (ScalaJsonUtil.serialize(rec._1) -> rec._2))) - - //TODO: deprecate using contentType in the event. - val event = s"""{"eid":"BE_JOB_REQUEST", "ets": $ets, "mid": "$mid", "actor": {"id": "Post Publish Processor", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}, "channel":"$channelId","env":"${config.jobEnv}"},"object":{"ver":"$ver","id":"${obj.identifier}"},"edata": {"action":"post-publish-process","iteration":1,"identifier":"${obj.identifier}","channel":"$channelId","mimeType":"${obj.mimeType}","contentType":"$contentType","pkgVersion":${obj.pkgVersion},"status":"$status","name":"${obj.metadata("name")}","trackable":${obj.metadata.getOrElse("trackable",ScalaJsonUtil.serialize(Map.empty))}, "addContextDialCodes": ${serAddContextDialCode}, "removeContextDialCodes": ${serRemoveContextDialCode} }}""".stripMargin - logger.info(s"Post Publish Process Event for identifier ${obj.identifier} is : $event") - event - } - - private def pushFailedEvent(event: Event, errorMessage: String, error: Throwable, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - val failedEvent = if (error == null) getFailedEvent(event.jobName, event.getMap(), errorMessage) else getFailedEvent(event.jobName, event.getMap(), error) - context.output(config.failedEventOutTag, failedEvent) - metrics.incCounter(config.collectionPublishFailedEventCount) - } - -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/ContentPublishFunction.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/ContentPublishFunction.scala deleted file mode 100644 index be975fffc..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/ContentPublishFunction.scala +++ /dev/null @@ -1,200 +0,0 @@ -package org.sunbird.job.content.function - -import akka.dispatch.ExecutionContexts -import com.google.gson.reflect.TypeToken -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.neo4j.driver.v1.exceptions.ClientException -import org.slf4j.LoggerFactory -import org.sunbird.job.cache.{DataCache, RedisConnect} -import org.sunbird.job.content.publish.domain.Event -import org.sunbird.job.content.publish.helpers.{ContentPublisher, ExtractableMimeTypeHelper} -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type -import java.util.UUID -import scala.concurrent.ExecutionContext - -class ContentPublishFunction(config: ContentPublishConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var definitionCache: DefinitionCache = null, - @transient var definitionConfig: DefinitionConfig = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with ContentPublisher with FailedEventHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[ContentPublishFunction]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - private var cache: DataCache = _ - private val readerConfig = ExtDataConfig(config.contentKeyspaceName, config.contentTableName) - - @transient var ec: ExecutionContext = _ - private val pkgTypes = List(EcarPackageType.FULL, EcarPackageType.SPINE) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - cloudStorageUtil = new CloudStorageUtil(config) - ec = ExecutionContexts.global - definitionCache = new DefinitionCache() - definitionConfig = DefinitionConfig(config.schemaSupportVersionMap, config.definitionBasePath) - cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) - cache.init() - } - - override def close(): Unit = { - super.close() - cassandraUtil.close() - cache.close() - } - - override def metricsList(): List[String] = { - List(config.contentPublishEventCount, config.contentPublishSuccessEventCount, config.contentPublishFailedEventCount, - config.videoStreamingGeneratorEventCount, config.skippedEventCount, config.mvProcessorEventCount) - } - - override def processElement(data: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - logger.info("Content publishing started for : " + data.identifier) - metrics.incCounter(config.contentPublishEventCount) - val obj: ObjectData = getObject(data.identifier, data.pkgVersion, data.mimeType, data.publishType, readerConfig)(neo4JUtil, cassandraUtil, config) - try { - if (obj.pkgVersion > data.pkgVersion) { - metrics.incCounter(config.skippedEventCount) - logger.info(s"""pkgVersion should be greater than or equal to the obj.pkgVersion for : ${obj.identifier}""") - } else { - val messages: List[String] = validate(obj, obj.identifier, config, validateMetadata) - if (messages.isEmpty) { - // Pre-publish update - updateProcessingNode(new ObjectData(obj.identifier, obj.metadata ++ Map("lastPublishedBy" -> data.lastPublishedBy), obj.extData, obj.hierarchy))(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig) - - val ecmlVerifiedObj = if (obj.mimeType.equalsIgnoreCase("application/vnd.ekstep.ecml-archive")) { - val ecarEnhancedObj = ExtractableMimeTypeHelper.processECMLBody(obj, config)(ec, cloudStorageUtil) - new ObjectData(obj.identifier, ecarEnhancedObj, obj.extData, obj.hierarchy) - } else obj - - // Clear redis cache - cache.del(data.identifier) - val enrichedObj = enrichObject(ecmlVerifiedObj)(neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig) - val objWithEcar = getObjectWithEcar(enrichedObj, if (enrichedObj.getString("contentDisposition", "").equalsIgnoreCase("online-only")) List(EcarPackageType.SPINE) else pkgTypes)(ec, neo4JUtil, cloudStorageUtil, config, definitionCache, definitionConfig, httpUtil) - logger.info("Ecar generation done for Content: " + objWithEcar.identifier) - saveOnSuccess(objWithEcar)(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig, config) - pushStreamingUrlEvent(enrichedObj, context)(metrics) - pushMVCProcessorEvent(enrichedObj, context)(metrics) - - if(obj.metadata.contains("dialcodes") && config.enableDIALContextUpdate.equalsIgnoreCase("Yes")) { - //pushDIALcodeContextUpdaterEvent for linked and delinked DIAL codes - pushDIALcodeContextUpdaterEvent(obj, context)(neo4JUtil, metrics) - } - - metrics.incCounter(config.contentPublishSuccessEventCount) - logger.info("Content publishing completed successfully for : " + data.identifier) - } else { - saveOnFailure(obj, messages, data.pkgVersion)(neo4JUtil) - val errorMessages = messages.mkString("; ") - pushFailedEvent(data, errorMessages, null, context)(metrics) - logger.info("Content publishing failed for : " + data.identifier) - } - } - } catch { - case ex@(_: InvalidInputException | _: ClientException | _:java.lang.IllegalArgumentException) => // ClientException - Invalid input exception. - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - pushFailedEvent(data, null, ex, context)(metrics) - logger.error("Error while publishing content :: " + ex.getMessage) - case ex: Exception => - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - logger.error(s"Error while processing message for Partition: ${data.partition} and Offset: ${data.offset}. Error : ${ex.getMessage}", ex) - throw ex - } - } - - private def pushStreamingUrlEvent(obj: ObjectData, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - if (config.isStreamingEnabled && config.streamableMimeType.contains(obj.mimeType)) { - val event = getStreamingEvent(obj) - context.output(config.generateVideoStreamingOutTag, event) - metrics.incCounter(config.videoStreamingGeneratorEventCount) - } - } - - def getStreamingEvent(obj: ObjectData): String = { - val ets = System.currentTimeMillis - val mid = s"""LP.$ets.${UUID.randomUUID}""" - val channelId = obj.getString("channel", "") - val ver = obj.getString("versionKey", "") - val artifactUrl = obj.getString("artifactUrl", "") - val contentType = obj.getString("contentType", "") - val status = obj.getString("status", "") - //TODO: deprecate using contentType in the event. - val event = s"""{"eid":"BE_JOB_REQUEST", "ets": $ets, "mid": "$mid", "actor": {"id": "Post Publish Processor", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"}, "channel":"$channelId","env":"${config.jobEnv}"},"object":{"ver":"$ver","id":"${obj.identifier}"},"edata": {"action":"post-publish-process","iteration":1,"identifier":"${obj.identifier}","channel":"$channelId","artifactUrl":"$artifactUrl","mimeType":"${obj.mimeType}","contentType":"$contentType","pkgVersion":${obj.pkgVersion},"status":"$status"}}""".stripMargin - logger.info(s"Video Streaming Event for identifier ${obj.identifier} is : $event") - event - } - - private def pushMVCProcessorEvent(obj: ObjectData, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - if (obj.getString("sourceURL", "").nonEmpty) { - val event = getMVCProcessorEvent(obj) - context.output(config.mvcProcessorTag, event) - metrics.incCounter(config.mvProcessorEventCount) - } - } - - def getMVCProcessorEvent(obj: ObjectData): String = { - val ets = System.currentTimeMillis - val mid = s"""LP.$ets.${UUID.randomUUID}""" - val channelId = obj.getString("channel", "") - val ver = obj.getString("versionKey", "") - val artifactUrl = obj.getString("artifactUrl", "") - val name = obj.getString("name", "") - //TODO: deprecate using contentType in the event. - val event = s"""{"eid":"MVC_JOB_PROCESSOR", "ets": $ets, "mid": "$mid", "actor": {"id": "Post Publish Processor", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}, "channel":"$channelId","env":"${config.jobEnv}"},"object":{"ver":"$ver","id":"${obj.identifier}"},"eventData": {"action":"update-es-index","stage":1,"identifier":"${obj.identifier}","channel":"$channelId","artifactUrl":"$artifactUrl","name":"$name"}}""".stripMargin - logger.info(s"MVC Processor Event for identifier ${obj.identifier} is : $event") - event - } - - private def pushFailedEvent(event: Event, errorMessage: String, error: Throwable, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - val failedEvent = if (error == null) getFailedEvent(event.jobName, event.getMap(), errorMessage) else getFailedEvent(event.jobName, event.getMap(), error) - context.output(config.failedEventOutTag, failedEvent) - metrics.incCounter(config.contentPublishFailedEventCount) - } - - - - private def pushDIALcodeContextUpdaterEvent(obj: ObjectData, context: ProcessFunction[Event, String]#Context)(implicit neo4JUtil: Neo4JUtil, metrics: Metrics): Unit = { - val nodeId: String = obj.identifier.replaceAll(".img","") - val draftDialcode = obj.metadata.getOrElse("dialcodes", List.empty[String]).asInstanceOf[List[String]] - val publishedDialCode = if(obj.pkgVersion>0) neo4JUtil.getNodeProperties(nodeId).getOrDefault("dialcodes",List.empty[String]).asInstanceOf[List[String]] else List.empty[String] - - val addContextDialCodes: Map[List[String], String] = if(draftDialcode!=null && draftDialcode.nonEmpty) Map(draftDialcode -> nodeId) else Map.empty[List[String], String] - val removeContextDialCodes: Map[List[String], String] = if(publishedDialCode!=null && publishedDialCode.nonEmpty && publishedDialCode.exists(dialcode => !draftDialcode.contains(dialcode))) Map(publishedDialCode.filter(dialcode => !draftDialcode.contains(dialcode)) -> nodeId) else Map.empty[List[String], String] - - if(addContextDialCodes.nonEmpty || removeContextDialCodes.nonEmpty) { - val event = getDIALcodeContextUpdaterEvent(obj, addContextDialCodes, removeContextDialCodes)(neo4JUtil) - context.output(config.dialcodeContextUpdaterOutTag, event) - metrics.incCounter(config.dialcodeContextUpdaterEventCount) - } - } - - def getDIALcodeContextUpdaterEvent(obj: ObjectData, addContextDialCodes: Map[List[String], String], removeContextDialCodes: Map[List[String], String])(implicit neo4JUtil: Neo4JUtil): String = { - val ets = System.currentTimeMillis - val mid = s"""LP.$ets.${UUID.randomUUID}""" - val channelId = obj.getString("channel", "") - val ver = obj.getString("versionKey", "") - val contentType = obj.getString("contentType", "") - val status = obj.getString("status", "") - //TODO: deprecate using contentType in the event. - val event = s"""{"eid":"BE_JOB_REQUEST", "ets": $ets, "mid": "$mid", "actor": {"id": "Post Publish Processor", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"}, "channel":"$channelId","env":"${config.jobEnv}"},"object":{"ver":"$ver","id":"${obj.identifier}"},"edata": {"action":"post-publish-process","iteration":1,"identifier":"${obj.identifier}","channel":"$channelId","contentType":"$contentType","status":"$status", "addContextDialCodes": ${addContextDialCodes}, "removeContextDialCodes": ${removeContextDialCodes} }}""".stripMargin - logger.info(s"Video Streaming Event for identifier ${obj.identifier} is : $event") - event - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/PublishEventRouter.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/PublishEventRouter.scala deleted file mode 100644 index fc57b370f..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/PublishEventRouter.scala +++ /dev/null @@ -1,55 +0,0 @@ -package org.sunbird.job.content.function - -import com.google.gson.reflect.TypeToken -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.content.publish.domain.Event -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type - -class PublishEventRouter(config: ContentPublishConfig) extends BaseProcessFunction[Event, String](config) { - - - private[this] val logger = LoggerFactory.getLogger(classOf[PublishEventRouter]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def metricsList(): List[String] = { - List(config.skippedEventCount, config.totalEventsCount) - } - - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - logger.info("PublishEventRouter :: Event: " + event) - // Event validation - if (event.validEvent(config)) { - event.objectType match { - case "Content" | "ContentImage" => { - logger.info("PublishEventRouter :: Sending Content For Publish Having Identifier: " + event.identifier) - context.output(config.contentPublishOutTag, event) - } - case "Collection" | "CollectionImage" => { - logger.info("PublishEventRouter :: Sending Collection For Publish Having Identifier: " + event.identifier) - context.output(config.collectionPublishOutTag, event) - } - case _ => { - metrics.incCounter(config.skippedEventCount) - logger.info("Invalid Object Type Received For Publish.| Identifier : " + event.identifier + " , objectType : " + event.objectType) - } - } - } else { - logger.warn("Event skipped for identifier: " + event.identifier + " objectType: " + event.objectType) - metrics.incCounter(config.skippedEventCount) - } - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/domain/Event.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/domain/Event.scala deleted file mode 100644 index 64661bafa..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/domain/Event.scala +++ /dev/null @@ -1,40 +0,0 @@ -package org.sunbird.job.content.publish.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.domain.reader.JobRequest - -import java.util -import scala.collection.JavaConverters._ - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "content-publish" - - def eData: Map[String, AnyRef] = readOrDefault("edata", new util.HashMap[String, AnyRef]()).asScala.toMap - - def action: String = readOrDefault[String]("edata.action", "") - - def mimeType: String = readOrDefault[String]("edata.metadata.mimeType", "") - - def identifier: String = readOrDefault[String]("edata.metadata.identifier", "") - - def objectType: String = readOrDefault[String]("edata.metadata.objectType", "") - - def contentType: String = readOrDefault[String]("edata.contentType", "") - - def publishType: String = readOrDefault[String]("edata.publish_type", "") - - def lastPublishedBy: String = readOrDefault[String]("edata.metadata.lastPublishedBy", "") - - def pkgVersion: Double = { - val pkgVersion: Number = readOrDefault[Number]("edata.metadata.pkgVersion", 0) - pkgVersion.doubleValue() - } - - def validEvent(config: ContentPublishConfig): Boolean = { - ((StringUtils.equals("publish", action) && StringUtils.isNotBlank(identifier)) - && (config.supportedObjectType.contains(objectType) && config.supportedMimeType.contains(mimeType)) - && !StringUtils.equalsIgnoreCase("Asset", contentType)) - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/CollectionPublisher.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/CollectionPublisher.scala deleted file mode 100644 index 9e7ef8582..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/CollectionPublisher.scala +++ /dev/null @@ -1,700 +0,0 @@ -package org.sunbird.job.content.publish.helpers - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.{Insert, QueryBuilder, Select} -import com.fasterxml.jackson.core.JsonProcessingException -import org.apache.commons.io.FileUtils -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers._ -import org.sunbird.job.util._ - -import java.io.{File, IOException} -import java.text.{DecimalFormat, DecimalFormatSymbols, SimpleDateFormat} -import java.util -import java.util.{Date, Locale} -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.collection.mutable.ListBuffer -import scala.concurrent.ExecutionContext - -trait CollectionPublisher extends ObjectReader with SyncMessagesGenerator with ObjectValidator with ObjectEnrichment with EcarGenerator with ObjectUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[CollectionPublisher]) - private val level4ContentTypes = List("Course", "CourseUnit", "LessonPlan", "LessonPlanUnit") - private val EXPANDABLE_OBJECTS = List("Collection", "QuestionSet") - private val EXCLUDE_LEAFNODE_OBJECTS = List("Collection", "Question") - private val INCLUDE_LEAFNODE_OBJECTS = List("QuestionSet") - private val INCLUDE_CHILDNODE_OBJECTS = List("Collection") - private val PUBLISHED_STATUS_LIST = List("Live", "Unlisted") - private val COLLECTION_MIME_TYPE = "application/vnd.ekstep.content-collection" - - override def getExtData(identifier: String, pkgVersion: Double, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] = None - - override def getHierarchy(identifier: String, pkgVersion: Double, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = { - val row: Row = Option(getCollectionHierarchy(getEditableObjId(identifier, pkgVersion), readerConfig)).getOrElse(getCollectionHierarchy(identifier, readerConfig)) - if (null != row) { - val hierarchy = row.getString("hierarchy") - val updatedHierarchy = if (config.asInstanceOf[ContentPublishConfig].isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(hierarchy) else hierarchy - val data: Map[String, AnyRef] = if(updatedHierarchy.nonEmpty) ScalaJsonUtil.deserialize[Map[String, AnyRef]](updatedHierarchy) else Map.empty[String, AnyRef] - Option(data) - } else Option(Map.empty[String, AnyRef]) - } - - private def getLiveHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = { - val row: Row = getCollectionHierarchy(identifier, readerConfig) - if (null != row) { - val hierarchy = row.getString("hierarchy") - val updatedHierarchy = if(config.asInstanceOf[ContentPublishConfig].isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(hierarchy) else hierarchy - val data: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](updatedHierarchy) - Option(data) - } else Option(Map.empty[String, AnyRef]) - } - - private def getCollectionHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Row = { - val selectWhere: Select.Where = QueryBuilder.select().all() - .from(readerConfig.keyspace, readerConfig.table). - where() - selectWhere.and(QueryBuilder.eq("identifier", identifier)) - cassandraUtil.findOne(selectWhere.toString) - } - - def getRelationalMetadata(identifier: String, pkgVersion: Double, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: ContentPublishConfig): Option[Map[String, AnyRef]] = { - val row: Row = Option(getCollectionHierarchy(getEditableObjId(identifier, pkgVersion), readerConfig)).getOrElse(getCollectionHierarchy(identifier, readerConfig)) - if (null != row && row.getString("relational_metadata") != null && row.getString("relational_metadata").nonEmpty) { - val relationalMetadata = row.getString("relational_metadata") - val updatedRelationalMetadata = if (config.isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(relationalMetadata) else relationalMetadata - val data: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](updatedRelationalMetadata) - Option(data) - } else Option(Map.empty[String, AnyRef]) - } - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig): Option[ObjectData] = { - val contentConfig = config.asInstanceOf[ContentPublishConfig] - val extraMeta = Map("pkgVersion" -> (obj.pkgVersion + 1).asInstanceOf[AnyRef], "lastPublishedOn" -> getTimeStamp, - "flagReasons" -> null, "body" -> null, "publishError" -> null, "variants" -> null, "downloadUrl" -> null) - val contentSize = obj.metadata.getOrElse("size", 0).toString.toDouble - val configSize = contentConfig.artifactSizeForOnline - val updatedMeta: Map[String, AnyRef] = if (contentSize > configSize) obj.metadata ++ extraMeta ++ Map("contentDisposition" -> "online-only") else obj.metadata ++ extraMeta - - val publishType = obj.getString("publish_type", "Public") - val status = if (StringUtils.equalsIgnoreCase("Unlisted", publishType)) "Unlisted" else "Live" - val updatedCompatibilityLevelMeta: Map[String, AnyRef] = setCompatibilityLevel(obj, updatedMeta).get + ("status" -> status) - - val isCollectionShallowCopy = isContentShallowCopy(obj) - - // Collection - Enrich Children - line 345 - val collectionHierarchy: Map[String, AnyRef] = if (isCollectionShallowCopy) { - val originData: Map[String, AnyRef] = obj.metadata.getOrElse("originData", "").asInstanceOf[Map[String, AnyRef]] - getHierarchy(obj.metadata("origin").asInstanceOf[String], originData.getOrElse("pkgVersion", 0).asInstanceOf[Number].doubleValue(), readerConfig).get - } else getHierarchy(obj.identifier, obj.pkgVersion, readerConfig).get - logger.info("CollectionPublisher:: enrichObjectMetadata:: collectionHierarchy:: " + collectionHierarchy) - val children = if (collectionHierarchy.nonEmpty) { - collectionHierarchy.getOrElse("children", List.empty[Map[String, AnyRef]]).asInstanceOf[List[Map[String, AnyRef]]] - } else List.empty[Map[String, AnyRef]] - val toEnrichChildren = children.to[ListBuffer] - val enrichedObj: ObjectData = if (collectionHierarchy.nonEmpty && !isCollectionShallowCopy) { - val childNodesToRemove = ListBuffer.empty[String] - val collectionResourceChildNodes: mutable.HashSet[String] = mutable.HashSet.empty[String] - val enrichedChildrenData = enrichChildren(toEnrichChildren, collectionResourceChildNodes, childNodesToRemove) - val collectionChildNodes = (updatedCompatibilityLevelMeta.getOrElse("childNodes", new java.util.ArrayList()).asInstanceOf[java.util.List[String]].asScala.toList ++ collectionResourceChildNodes).distinct - new ObjectData(obj.identifier, updatedCompatibilityLevelMeta ++ Map("childNodes" -> collectionChildNodes.filter(rec => !childNodesToRemove.contains(rec))), obj.extData, Option(collectionHierarchy + ("children" -> enrichedChildrenData.toList))) - } else new ObjectData(obj.identifier, updatedCompatibilityLevelMeta, obj.extData, Option(collectionHierarchy ++ Map("children" -> toEnrichChildren.toList))) - - logger.info("CollectionPublisher:: enrichObjectMetadata:: Collection data after processing for : " + enrichedObj.identifier + " | Metadata : " + enrichedObj.metadata) - logger.debug("CollectionPublisher:: enrichObjectMetadata:: Collection children data after processing : " + enrichedObj.hierarchy.get("children")) - - Some(enrichedObj) - } - - override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = { - val hChildren: List[Map[String, AnyRef]] = obj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - Some(getFlatStructure(List(obj.metadata ++ obj.extData.getOrElse(Map()) ++ Map("children" -> hChildren)), List())) - } - - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val identifier = obj.identifier.replace(".img", "") - val query: Insert = QueryBuilder.insertInto(readerConfig.keyspace, readerConfig.table) - query.value(readerConfig.primaryKey(0), identifier) - query.value("relational_metadata", null) - val result = cassandraUtil.upsert(query.toString) - if (result) { - logger.info(s"relational_metadata emptied successfully for ${identifier}") - } else { - val msg = s"relational_metadata emptying Failed For $identifier" - logger.error(msg) - } - } - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - def getObjectWithEcar(obj: ObjectData, pkgTypes: List[String])(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: ContentPublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig, httpUtil: HttpUtil): ObjectData = { - val collRelationalMetadata = getRelationalMetadata(obj.identifier, obj.pkgVersion-1, readerConfig).getOrElse(Map.empty[String, AnyRef]) - // Line 1107 in PublishFinalizer - val children = obj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - val updatedChildren = updateHierarchyMetadata(children, obj.metadata, collRelationalMetadata)(config) - val enrichedObj = processCollection(obj, updatedChildren) - val updatedObj = updateRootChildrenList(enrichedObj, updatedChildren) - val nodes = ListBuffer.empty[ObjectData] - val nodeIds = ListBuffer.empty[String] - nodes += obj - nodeIds += obj.identifier - - val ecarMap: Map[String, String] = generateEcar(updatedObj, pkgTypes) - val variants: java.util.Map[String, java.util.Map[String, String]] = ecarMap.map { case (key, value) => key.toLowerCase -> Map[String, String]("ecarUrl" -> value, "size" -> httpUtil.getSize(value).toString).asJava }.asJava - logger.info("CollectionPulisher ::: getObjectWithEcar ::: variants ::: " + variants) - - val meta: Map[String, AnyRef] = Map("downloadUrl" -> ecarMap.getOrElse(EcarPackageType.SPINE, ""), "variants" -> variants, "size" -> httpUtil.getSize(ecarMap.getOrElse(EcarPackageType.SPINE, "")).asInstanceOf[AnyRef]) - new ObjectData(updatedObj.identifier, updatedObj.metadata ++ meta, updatedObj.extData, updatedObj.hierarchy) - } - - private def setCompatibilityLevel(obj: ObjectData, updatedMeta: Map[String, AnyRef]): Option[Map[String, AnyRef]] = { - if (level4ContentTypes.contains(obj.getString("contentType", ""))) { - logger.info("CollectionPublisher:: setCompatibilityLevel:: setting compatibility level for content id : " + obj.identifier + " as 4.") - Some(updatedMeta ++ Map("compatibilityLevel" -> 4.asInstanceOf[AnyRef])) - } else Some(updatedMeta) - } - - def getUnitsFromLiveContent(obj: ObjectData)(implicit cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, config: ContentPublishConfig): List[String] = { - logger.info("CollectionPublisher:: getUnitsFromLiveContent:: identifier: " + obj.identifier + " || pkgVersion: " + obj.metadata.getOrElse("pkgVersion", 1).asInstanceOf[Number]) - val objHierarchy = getLiveHierarchy(obj.identifier, readerConfig).get - val children = objHierarchy.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]] - getUnits(children) - } - - private def getUnits(children: List[Map[String, AnyRef]]): List[String] = { - if (children.nonEmpty) { - children.flatMap(child => { - if (child.getOrElse("visibility", "").asInstanceOf[String].equalsIgnoreCase("Parent")) { - val childUnits = if(child.contains("children")) getUnits(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]]) else List.empty - childUnits ++ List(child.getOrElse("identifier", "").asInstanceOf[String]) - } else List.empty[String] - }).filter(rec => rec.nonEmpty) - } else List.empty[String] - } - - def isContentShallowCopy(obj: ObjectData): Boolean = { - val originData: Map[String, AnyRef] = if(obj.metadata.contains("originData")) { - obj.metadata("originData") match { - case strValue: String => ScalaJsonUtil.deserialize[Map[String, AnyRef]](strValue) - case mapValue:util.Map[String, AnyRef] => mapValue.asScala.toMap[String, AnyRef] - case _ => obj.metadata("originData").asInstanceOf[Map[String,AnyRef]] - } - } else Map.empty[String, AnyRef] - originData.nonEmpty && originData.getOrElse("copyType", "").asInstanceOf[String].equalsIgnoreCase("shallow") - } - - def updateOriginPkgVersion(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil): ObjectData = { - val originId = obj.metadata.getOrElse("origin", "").asInstanceOf[String] - val originNodeMetadata = Option(neo4JUtil.getNodeProperties(originId)).getOrElse(neo4JUtil.getNodeProperties(originId)) - if (null != originNodeMetadata) { - val originPkgVer: Double = originNodeMetadata.getOrDefault("pkgVersion", "0").toString.toDouble - if (originPkgVer != 0) { - val originData = obj.metadata("originData") match { - case propVal: String => ScalaJsonUtil.deserialize[Map[String, AnyRef]](propVal) + ("pkgVersion" -> originPkgVer) - case _ => obj.metadata.getOrElse("originData", Map.empty[String, AnyRef]).asInstanceOf[Map[String, AnyRef]] + ("pkgVersion" -> originPkgVer) - } - new ObjectData(obj.identifier, obj.metadata ++ Map("originData" -> originData), obj.extData, obj.hierarchy) - } else obj - } else obj - } - - private def enrichChildren(toEnrichChildren: ListBuffer[Map[String, AnyRef]], collectionResourceChildNodes: mutable.HashSet[String], childNodesToRemove: ListBuffer[String])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, config: PublishConfig): ListBuffer[Map[String, AnyRef]] = { - val newChildren = toEnrichChildren.toList - newChildren.map(child => { - logger.info("CollectionPublisher:: enrichChildren:: child identifier:: " + child.getOrElse("identifier", "") + " || visibility:: " + child.getOrElse("visibility", "") + " || mimeType:: " + child.getOrElse("mimeType", "") + " || objectType:: " + child.getOrElse("objectType", "")) - if (StringUtils.equalsIgnoreCase(child.getOrElse("visibility", "").asInstanceOf[String], "Parent") && StringUtils.equalsIgnoreCase(child.getOrElse("mimeType", "").asInstanceOf[String], COLLECTION_MIME_TYPE)) { - val updatedChildrenData = enrichChildren(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]].to[ListBuffer], collectionResourceChildNodes, childNodesToRemove) - toEnrichChildren(newChildren.indexOf(child)) = child + ("children" -> updatedChildrenData.toList) - } - - if (StringUtils.equalsIgnoreCase(child.getOrElse("visibility", "").asInstanceOf[String], "Default") && EXPANDABLE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String])) { - val childCollectionHierarchy = getLiveHierarchy(child.getOrElse("identifier", "").asInstanceOf[String], readerConfig).get - if (childCollectionHierarchy.nonEmpty) { - val childNodes = childCollectionHierarchy.getOrElse("childNodes", List.empty).asInstanceOf[List[String]] - if (childNodes.nonEmpty && INCLUDE_CHILDNODE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String])) collectionResourceChildNodes ++= childNodes.toSet[String] - toEnrichChildren(newChildren.indexOf(child)) = childCollectionHierarchy ++ Map("index" -> child.getOrElse("index", 0).asInstanceOf[AnyRef], "depth" -> child.getOrElse("depth", 0).asInstanceOf[AnyRef], "parent" -> child.getOrElse("parent", ""), "objectType" -> child.getOrElse("objectType", "Collection").asInstanceOf[String]) - } - } - - if (StringUtils.equalsIgnoreCase(child.getOrElse("visibility", "").asInstanceOf[String], "Default") && !EXPANDABLE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String])) { - val childNode = Option(neo4JUtil.getNodeProperties(child.getOrElse("identifier", "").asInstanceOf[String])).getOrElse(neo4JUtil.getNodeProperties(child.getOrElse("identifier", "").asInstanceOf[String])).asScala.toMap - if (PUBLISHED_STATUS_LIST.contains(childNode.getOrElse("status", "").asInstanceOf[String])) { - logger.info("CollectionPublisher:: enrichChildren:: fetched child node:: " + childNode.getOrElse("IL_UNIQUE_ID", "").asInstanceOf[String] + " || objectType:: " + childNode.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String]) - toEnrichChildren(newChildren.indexOf(child)) = childNode ++ Map("identifier" ->childNode.getOrElse("IL_UNIQUE_ID", "").asInstanceOf[String], "objectType" ->childNode.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String], "index" -> child.getOrElse("index", 0).asInstanceOf[AnyRef], "parent" -> child.getOrElse("parent", "").asInstanceOf[String], "depth" -> child.getOrElse("depth", 0).asInstanceOf[AnyRef]) - ("collections", "children", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE","IL_UNIQUE_ID") - } else childNodesToRemove += child.getOrElse("identifier", "").asInstanceOf[String] - } - }) - - toEnrichChildren - } - - - private def processCollection(obj: ObjectData, children: List[Map[String, AnyRef]])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig): ObjectData = { - val dataMap: mutable.Map[String, AnyRef] = processChildren(children) - logger.info("CollectionPublisher:: processCollection:: dataMap: " + dataMap) - val updatedObj: ObjectData = if (dataMap.nonEmpty) { - val updatedMetadataMap: Map[String, AnyRef] = dataMap.flatMap(record => { - if (!"concepts".equalsIgnoreCase(record._1) && !"keywords".equalsIgnoreCase(record._1)) { - Map(record._1 -> record._2.asInstanceOf[Set[String]].toArray[String]) - } else Map.empty[String, AnyRef] - }).filter(record => record._1.nonEmpty).toMap[String, AnyRef] - val keywords = dataMap.getOrElse("keywords", Set.empty).asInstanceOf[Set[String]].toArray[String] - val finalKeywords: Array[String] = if (null != keywords && keywords.nonEmpty) { - val updatedKeywords: Array[String] = if (obj.metadata.contains("keywords")) { - obj.metadata("keywords") match { - case _: Array[String] => keywords ++ obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[Array[String]] - case kwValue: String => keywords ++ Array[String](kwValue) - case _: util.Collection[String] => keywords ++ obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[util.Collection[String]].asScala.toArray[String] - case _ => keywords - } - } else keywords - updatedKeywords.filter(record => record.trim.nonEmpty).distinct - } else if(obj.metadata.contains("keywords")) { - obj.metadata("keywords") match { - case _: Array[String] => obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[Array[String]] - case kwValue: String => Array[String](kwValue) - case _: util.Collection[String] => obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[util.Collection[String]].asScala.toArray[String] - case _ => Array.empty[String] - } - } else Array.empty[String] - new ObjectData(obj.identifier, obj.metadata ++ updatedMetadataMap + ("keywords" -> finalKeywords), obj.extData, obj.hierarchy) - } else obj - - val enrichedObject = if(children.nonEmpty) enrichCollection(updatedObj) else updatedObj - // addResourceToCollection(enrichedObject, children.to[ListBuffer]) - TODO - enrichedObject - } - - private def processChildren(children: List[Map[String, AnyRef]]): mutable.Map[String, AnyRef] = { - val dataMap: mutable.Map[String, AnyRef] = mutable.Map.empty - processChildren(children, dataMap) - dataMap - } - - private def processChildren(children: List[Map[String, AnyRef]], dataMap: mutable.Map[String, AnyRef]): Unit = { - if (null != children && children.nonEmpty) { - for (child <- children) { - mergeMap(dataMap, processChild(child)) - if (child.contains("children")) processChildren(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]], dataMap) - } - } - } - - private def processChild(childMetadata: Map[String, AnyRef]): Map[String, AnyRef] = { - val taggingProperties = List("language", "domain", "ageGroup", "genre", "theme", "keywords") - val result: Map[String, AnyRef] = childMetadata.flatMap(prop => { - if (taggingProperties.contains(prop._1)) { - childMetadata(prop._1) match { - case propStrValue: String => Map(prop._1 -> Set(propStrValue)) - case propListValue: List[_] => Map(prop._1 -> propListValue.toSet) - case propVal: java.util.List[String] => Map(prop._1 -> propVal.asScala.toSet[String]) - case _ => Map.empty[String, AnyRef] - } - } else Map.empty[String, AnyRef] - }).filter(rec => rec._1.nonEmpty) - result - } - - private def mergeMap(dataMap: mutable.Map[String, AnyRef], childDataMap: Map[String, AnyRef]): mutable.Map[String, AnyRef] = { - if (dataMap.isEmpty) dataMap ++= childDataMap - else { - dataMap.map(record => { - dataMap += (record._1 -> (if (childDataMap.contains(record._1)) childDataMap(record._1).asInstanceOf[Set[String]] ++ record._2.asInstanceOf[Set[String]] else record._2.asInstanceOf[Set[String]])) - }) - if (!dataMap.equals(childDataMap)) { - childDataMap.map(record => { - if (!dataMap.contains(record._1)) dataMap += record - }) - } - } - dataMap - } - - def enrichCollection(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig): ObjectData = { - val nodeMetadata = mutable.Map.empty[String, AnyRef] ++ obj.metadata - val contentId = obj.identifier - logger.info("CollectionPublisher:: enrichCollection:: Processing Collection Content :" + contentId) - val content = obj.hierarchy.get - if (content.isEmpty) return obj - val leafCount = getLeafNodeCount(content) - val totalCompressedSize = getTotalCompressedSize(content, 0.0) - - val df = new DecimalFormat("0", DecimalFormatSymbols.getInstance(Locale.ENGLISH)) - df.setMaximumFractionDigits(0) - - nodeMetadata.put("leafNodesCount", leafCount.asInstanceOf[AnyRef]) - nodeMetadata.put("totalCompressedSize", df.format(totalCompressedSize).toLong.asInstanceOf[AnyRef]) - - nodeMetadata.put("leafNodes", updateLeafNodeIds(content)) - val mimeTypeMap: mutable.Map[String, AnyRef] = mutable.Map.empty[String, AnyRef] - val contentTypeMap: mutable.Map[String, AnyRef] = mutable.Map.empty[String, AnyRef] - getTypeCount(content, "mimeType", mimeTypeMap) - getTypeCount(content, "contentType", contentTypeMap) - - val updatedContent = content ++ Map("leafNodesCount" -> leafCount, "totalCompressedSize" -> df.format(totalCompressedSize).toLong, "mimeTypesCount" -> ScalaJsonUtil.serialize(mimeTypeMap), "contentTypesCount" -> ScalaJsonUtil.serialize(contentTypeMap)).asInstanceOf[Map[String, AnyRef]] - nodeMetadata.put("mimeTypesCount", ScalaJsonUtil.serialize(mimeTypeMap)) - nodeMetadata.put("contentTypesCount", ScalaJsonUtil.serialize(contentTypeMap)) - val uploadedFileUrl: Array[String] = generateTOC(obj, nodeMetadata.toMap) - if(uploadedFileUrl.nonEmpty) { - nodeMetadata.put("toc_url", uploadedFileUrl(1)) - nodeMetadata.put("s3Key", uploadedFileUrl(0)) - } - - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ") - val updatedOn = sdf.format(new Date()) - nodeMetadata.put("sYS_INTERNAL_LAST_UPDATED_ON", updatedOn) - val updatedMetadata: Map[String, AnyRef] = try { - setContentAndCategoryTypes(nodeMetadata.toMap) - } catch { - case e: Exception => logger.error("CollectionPublisher:: enrichCollection:: Error while stringify mimeTypeCount or contentTypesCount:", e) - nodeMetadata.toMap - } - - new ObjectData(obj.identifier, updatedMetadata, obj.extData, Option(updatedContent)) - } - - private def updateLeafNodeIds(content: Map[String, AnyRef]): Array[String] = { - val leafNodeIds: mutable.Set[String] = mutable.Set.empty[String] - getLeafNodesIds(content, leafNodeIds) - leafNodeIds.toArray - } - - private def getTypeCount(data: Map[String, AnyRef], `type`: String, typeMap: mutable.Map[String, AnyRef]): Unit = { - val children = data.getOrElse("children", List.empty).asInstanceOf[List[AnyRef]] - if (null != children && children.nonEmpty) { - for (child <- children) { - val childMap = child.asInstanceOf[Map[String, AnyRef]] - val typeValue = childMap.getOrElse(`type`, "").asInstanceOf[String] - if (null != typeValue) if (typeMap.contains(typeValue)) { - var count = typeMap.getOrElse(typeValue, 0).asInstanceOf[Int] - count += 1 - typeMap.put(typeValue, count.asInstanceOf[AnyRef]) - } - else typeMap.put(typeValue, 1.asInstanceOf[AnyRef]) - if (childMap.contains("children")) getTypeCount(childMap, `type`, typeMap) - } - } - } - - @SuppressWarnings(Array("unchecked")) - private def getLeafNodeCount(data: Map[String, AnyRef]): Int = { - val leafNodeIds: mutable.Set[String] = mutable.Set.empty[String] - getLeafNodesIds(data, leafNodeIds) - leafNodeIds.size - } - - private def getLeafNodesIds(data: Map[String, AnyRef], leafNodeIds: mutable.Set[String]): Unit = { - if (INCLUDE_LEAFNODE_OBJECTS.contains(data.getOrElse("objectType", "")) && StringUtils.equals(data.getOrElse("visibility", "").asInstanceOf[String], "Default")) leafNodeIds += data.getOrElse("identifier", "").asInstanceOf[String] - val children = data.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]] - if (children.nonEmpty) { - for (child <- children) { - getLeafNodesIds(child, leafNodeIds) - } - } - else if (!EXCLUDE_LEAFNODE_OBJECTS.contains(data.getOrElse("objectType", "").asInstanceOf[String])) leafNodeIds.add(data.getOrElse("identifier", "").asInstanceOf[String]) - } - - private def getTotalCompressedSize(data: Map[String, AnyRef], totalCompressed: Double): Double = { - val children = data.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]] - if (children.nonEmpty) { - val childrenSizes = children.map(child => { - val childSize = - if (!EXPANDABLE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String]) && StringUtils.equals(child.getOrElse("visibility", "").asInstanceOf[String], "Default")) { - child.getOrElse("totalCompressedSize", child.getOrElse("size", 0).asInstanceOf[Number].doubleValue).asInstanceOf[Number].doubleValue - } else 0 - - getTotalCompressedSize(child, childSize) - }).sum - totalCompressed + childrenSizes - } - else totalCompressed - } - - def generateTOC(obj: ObjectData, content: Map[String, AnyRef])(implicit cloudStorageUtil: CloudStorageUtil, config: PublishConfig): Array[String] = { - logger.info("CollectionPublisher:: generateTOC:: Write hierarchy to JSON File :" + obj.identifier) - val file = new File(getTOCBasePath(obj.identifier) + "_toc.json") - try { - val data = ScalaJsonUtil.serialize(content) - FileUtils.writeStringToFile(file, data, "UTF-8") - if (file.exists) { - logger.debug("CollectionPublisher:: generateTOC:: Upload File to cloud storage :" + file.getName) - val uploadedFileUrl = cloudStorageUtil.uploadFile(getAWSPath(obj.identifier), file, Option.apply(true)) - if (null != uploadedFileUrl && uploadedFileUrl.length > 1) { - logger.info("CollectionPublisher:: generateTOC:: Update cloud storage url to node" + uploadedFileUrl(1)) - uploadedFileUrl - } else Array.empty - } else Array.empty - } catch { - case e: JsonProcessingException => logger.error("CollectionPublisher:: generateTOC:: Error while parsing map object to string.", e) - throw new InvalidInputException("CollectionPublisher:: generateTOC:: Error while parsing map object to string.", e) - case e: Exception => logger.error("CollectionPublisher:: generateTOC:: Error while uploading file ", e) - throw new InvalidInputException("CollectionPublisher:: generateTOC:: Error while uploading file", e) - } finally try { - logger.info("CollectionPublisher:: generateTOC:: Deleting Uploaded files") - FileUtils.deleteDirectory(file.getParentFile) - } catch { - case e: IOException => - logger.error("CollectionPublisher:: generateTOC::Error while deleting file ", e) - } - } - - private def getTOCBasePath(contentId: String)(implicit cloudStorageUtil: CloudStorageUtil, config: PublishConfig): String = { - if (contentId.nonEmpty) "/tmp" + File.separator + System.currentTimeMillis + "_temp" + File.separator + contentId else "" - } - - private def getAWSPath(identifier: String)(implicit cloudStorageUtil: CloudStorageUtil, config: PublishConfig): String = { - val contentConfig = config.asInstanceOf[ContentPublishConfig] - val folderName = contentConfig.contentFolder - if (folderName.nonEmpty) folderName + File.separator + Slug.makeSlug(identifier, isTransliterate = true) + File.separator + contentConfig.artifactFolder else folderName - } - - def setContentAndCategoryTypes(input: Map[String, AnyRef])(implicit config: PublishConfig): Map[String, AnyRef] = { - val contentConfig = config.asInstanceOf[ContentPublishConfig] - val categoryMap = contentConfig.categoryMap - val contentType = input.getOrElse("contentType", "").asInstanceOf[String] - val primaryCategory = input.getOrElse("primaryCategory","").asInstanceOf[String] - val (updatedContentType, updatedPrimaryCategory): (String, String) = - if(contentType.nonEmpty && (primaryCategory.isEmpty || primaryCategory.isBlank)) { (contentType, categoryMap.getOrDefault(contentType,"").asInstanceOf[String]) } - else if((contentType.isEmpty || contentType.isBlank) && primaryCategory.nonEmpty) { (categoryMap.asScala.filter(entry => StringUtils.equalsIgnoreCase(entry._2.asInstanceOf[String], primaryCategory)).keys.headOption.getOrElse(""), primaryCategory) } - else (contentType, primaryCategory) - - input ++ Map("contentType" -> updatedContentType, "primaryCategory" -> updatedPrimaryCategory) - } - - def updateHierarchyMetadata(children: List[Map[String, AnyRef]], objMetadata: Map[String, AnyRef], collRelationalMetadata: Map[String, AnyRef])(implicit config: PublishConfig): List[Map[String, AnyRef]] = { - if (children.nonEmpty) { - children.map(child => { - if (StringUtils.equalsIgnoreCase("Parent", child.getOrElse("visibility", "").asInstanceOf[String])) { //set child metadata -- compatibilityLevel, appIcon, posterImage, lastPublishedOn, pkgVersion, status - val updatedChild = populatePublishMetadata(child, objMetadata) - updatedChild + ("children" -> updateHierarchyMetadata(updatedChild.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]], objMetadata, collRelationalMetadata)) - } else { - //TODO: Populate relationalMetadata here for child contents - if (collRelationalMetadata.nonEmpty) { - val parent = child.getOrElse("parent", "").asInstanceOf[String] - val unitRelationalMetadata = collRelationalMetadata(parent).asInstanceOf[Map[String, AnyRef]].getOrElse("relationalMetadata", Map.empty).asInstanceOf[Map[String, AnyRef]] - if (unitRelationalMetadata.nonEmpty) { - val childRelationalMetadata = unitRelationalMetadata.getOrElse(child.getOrElse("identifier","").asInstanceOf[String], Map.empty).asInstanceOf[Map[String, AnyRef]] - if(childRelationalMetadata.nonEmpty) { - child + ("relationalMetadata" -> childRelationalMetadata) - } else child - } else child - } else child - } - }) - } else children - } - - private def populatePublishMetadata(content: Map[String, AnyRef], objMetadata: Map[String, AnyRef])(implicit config: PublishConfig): Map[String, AnyRef] = { - //TODO: For appIcon, posterImage and screenshot createThumbNail method has to be implemented. - val leafNodeIds: mutable.Set[String] = mutable.Set.empty[String] - getLeafNodesIds(content, leafNodeIds) - - val updatedContent = content ++ - Map("compatibilityLevel" -> (if (null != content.get("compatibilityLevel")) content.getOrElse("compatibilityLevel", 1).asInstanceOf[Number].intValue else 1), - "lastPublishedOn" -> objMetadata("lastPublishedOn"), "pkgVersion" -> objMetadata.getOrElse("pkgVersion", 1).asInstanceOf[Number].intValue, "leafNodesCount" -> getLeafNodeCount(content), - "leafNodes" -> leafNodeIds.toArray[String], "status" -> objMetadata("status"), "lastUpdatedOn" -> objMetadata("lastUpdatedOn"), - "downloadUrl" -> objMetadata("downloadUrl"), "variants" -> objMetadata("variants")).asInstanceOf[Map[String, AnyRef]] - - // PRIMARY CATEGORY MAPPING IS DONE - setContentAndCategoryTypes(updatedContent) - } - - def publishHierarchy(children: List[Map[String, AnyRef]], obj: ObjectData, readerConfig: ExtDataConfig, config: PublishConfig)(implicit cassandraUtil: CassandraUtil): Boolean = { - val identifier = obj.identifier.replace(".img", "") - val contentConfig = config.asInstanceOf[ContentPublishConfig] - val nestedFields = contentConfig.nestedFields.asScala.toList - val nodeMetadata = obj.metadata.map(property => { - property._2 match { - case propVal: String => if (nestedFields.contains(property._1)) (property._1 -> ScalaJsonUtil.deserialize[AnyRef](propVal)) else property - case _ => property - } - }) - val hierarchy: Map[String, AnyRef] = nodeMetadata ++ Map("children" -> children) - val data = Map("hierarchy" -> hierarchy) ++ obj.extData.getOrElse(Map()) - val query: Insert = QueryBuilder.insertInto(readerConfig.keyspace, readerConfig.table) - query.value(readerConfig.primaryKey.head, identifier) - data.map(d => { - readerConfig.propsMapping.getOrElse(d._1, "") match { - case "blob" => query.value(d._1.toLowerCase, QueryBuilder.fcall("textAsBlob", d._2)) - case "string" => d._2 match { - case value: String => query.value(d._1.toLowerCase, value) - case _ => query.value(d._1.toLowerCase, ScalaJsonUtil.serialize(d._2)) - } - case _ => query.value(d._1, d._2) - } - }) - logger.info(s"CollectionPublisher:: publishHierarchy:: Publishing Hierarchy data for $identifier | Query : ${query.toString}") - val result = cassandraUtil.upsert(query.toString) - if (result) { - logger.info(s"CollectionPublisher:: publishHierarchy:: Hierarchy data saved successfully for $identifier") - } else { - val msg = s"CollectionPublisher:: publishHierarchy:: Hierarchy Data Insertion Failed For $identifier" - logger.error(msg) - throw new InvalidInputException(msg) - } - result - } - - private def updateRootChildrenList(obj: ObjectData, nextLevelNodes: List[Map[String, AnyRef]]): ObjectData = { - val childrenMap: List[Map[String, AnyRef]] = - nextLevelNodes.map(record => { - Map("identifier" -> record.getOrElse("identifier", "").asInstanceOf[String], - "name" -> record.getOrElse("name", "").asInstanceOf[String], - "objectType" -> record.getOrElse("objectType", "").asInstanceOf[String], - "description" -> record.getOrElse("description", "").asInstanceOf[String], - "index" -> record.getOrElse("index", 0).asInstanceOf[AnyRef]) - }) - - new ObjectData(obj.identifier, obj.metadata ++ Map("children" -> childrenMap, "objectType" -> "content"), obj.extData, Option(obj.hierarchy.get + ("children" -> nextLevelNodes))) - } - - def syncNodes(successObj: ObjectData, children: List[Map[String, AnyRef]], unitNodes: List[String])(implicit esUtil: ElasticSearchUtil, neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definition: ObjectDefinition, config: PublishConfig): Map[String, Map[String, AnyRef]] = { - val contentConfig = config.asInstanceOf[ContentPublishConfig] - val nestedFields = contentConfig.nestedFields.asScala.toList - val nodes = ListBuffer.empty[ObjectData] - val nodeIds = ListBuffer.empty[String] - - getNodeForSyncing(children, nodes, nodeIds) - logger.info("CollectionPublisher:: syncNodes:: after getNodeForSyncing:: nodes:: " + nodes.toList + " || nodeIds:: " + nodeIds) - logger.info("CollectionPublisher:: syncNodes:: unitNodes:: " + unitNodes) - - // Filtering for removed nodes from Live version. nodeIds is list of nodes from Draft version. unitNodes is list of nodes from Live version. - val orphanUnitNodes = if (unitNodes.nonEmpty) unitNodes.filter(unitNode => !nodeIds.contains(unitNode)) else unitNodes - logger.info("CollectionPublisher:: syncNodes:: after getNodeForSyncing:: orphanUnitNodes:: " + orphanUnitNodes) - - if (nodes.isEmpty && orphanUnitNodes.isEmpty) return Map.empty - - val errors = mutable.Map.empty[String, String] - val messages: Map[String, Map[String, AnyRef]] = getMessages(nodes.toList, definition, nestedFields, errors)(esUtil) - logger.info("CollectionPublisher:: syncNodes:: after getMessages:: messages:: " + messages) - if (errors.nonEmpty) logger.error("CollectionPublisher:: syncNodes:: Error! while forming ES document data from nodes, below nodes are ignored: " + errors) - if (messages.nonEmpty) - try { - logger.info("CollectionPublisher:: syncNodes:: Number of units to be synced : " + messages.size + " || " + messages.keySet) - esUtil.bulkIndexWithIndexId(contentConfig.compositeSearchIndexName, contentConfig.compositeSearchIndexType, messages) - logger.info("CollectionPublisher:: syncNodes:: UnitIds synced : " + messages.keySet) - } catch { - case e: Exception => e.printStackTrace() - logger.error("CollectionPublisher:: syncNodes:: Elastic Search indexing failed: " + e) - } - - try //Unindexing not utilized units - if (orphanUnitNodes.nonEmpty) orphanUnitNodes.map(unitNodeId => esUtil.deleteDocument(unitNodeId)) - catch { - case e: Exception => - logger.error("CollectionPublisher:: syncNodes:: Elastic Search indexing failed: " + e) - } - - // Syncing collection metadata - val doc: Map[String, AnyRef] = getDocument(new ObjectData(successObj.identifier, successObj.metadata.-("children"), successObj.extData, successObj.hierarchy), true, nestedFields)(esUtil) - val jsonDoc: String = ScalaJsonUtil.serialize(doc) - logger.info("CollectionPublisher:: syncNodes:: collection doc: " + jsonDoc) - esUtil.addDocument(successObj.identifier, jsonDoc) - - messages - } - - private def getNodeForSyncing(children: List[Map[String, AnyRef]], nodes: ListBuffer[ObjectData], nodeIds: ListBuffer[String])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig): Unit = { - if (children.nonEmpty) { - children.foreach((child: Map[String, AnyRef]) => { - try { - if (StringUtils.equalsIgnoreCase("Parent", child.getOrElse("visibility", "").asInstanceOf[String])) { - logger.info("CollectionPublisher:: getNodeForSyncing:: child identifier: " + child.getOrElse("identifier", "").asInstanceOf[String]) - - val nodeMetadata = mutable.Map() ++ child - - // TODO - Relation related CODE is MISSING - Line 735 in Publish Finalizer - - if (nodeMetadata.getOrElse("objectType", "").asInstanceOf[String].isEmpty) { - nodeMetadata += ("objectType" -> "Collection") - } - if (nodeMetadata.getOrElse("graphId", "").asInstanceOf[String].isEmpty) { - nodeMetadata += ("graph_id" -> "domain") - } - - if (nodeMetadata.contains("children")) nodeMetadata.remove("children") - - logger.info("CollectionPublisher:: getNodeForSyncing:: nodeMetadata: " + nodeMetadata) - - if (!nodeIds.contains(child.getOrElse("identifier", "").asInstanceOf[String])) { - nodes += new ObjectData(child.getOrElse("identifier", "").asInstanceOf[String], nodeMetadata.toMap[String, AnyRef], Option(Map.empty[String, AnyRef]), Option(Map.empty[String, AnyRef])) - nodeIds += child.getOrElse("identifier", "").asInstanceOf[String] - } - - getNodeForSyncing(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]], nodes, nodeIds) - } - } catch { - case e: Exception => logger.error("CollectionPublisher:: getNodeForSyncing:: Error while generating node map. ", e) - } - }) - } - } - - def saveImageHierarchy(obj: ObjectData, readerConfig: ExtDataConfig, collRelationalMetadata: Map[String, AnyRef])(implicit cassandraUtil: CassandraUtil): Boolean = { - val identifier = if(obj.identifier.endsWith(".img")) obj.identifier else obj.identifier+".img" - val hierarchy: Map[String, AnyRef] = obj.hierarchy.getOrElse(Map()) - val query: Insert = QueryBuilder.insertInto(readerConfig.keyspace, readerConfig.table) - query.value(readerConfig.primaryKey.head, identifier) - query.value("hierarchy", ScalaJsonUtil.serialize(hierarchy)) - query.value("relational_metadata", ScalaJsonUtil.serialize(collRelationalMetadata)) - logger.info(s"CollectionPublisher:: saveImageHierarchy:: Publishing Hierarchy data for $identifier | Query : ${query.toString}") - val result = cassandraUtil.upsert(query.toString) - if (result) { - logger.info(s"CollectionPublisher:: saveImageHierarchy:: Hierarchy data saved successfully for $identifier") - } else { - val msg = s"CollectionPublisher:: saveImageHierarchy:: Hierarchy Data Insertion Failed For $identifier" - logger.error(msg) - throw new InvalidInputException(msg) - } - result - } - - def fetchDialListForContextUpdate(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, config: PublishConfig): Map[String, AnyRef] = { - val isCollectionShallowCopy = isContentShallowCopy(obj) - - val DialContextMap: Map[String, AnyRef] = if (isCollectionShallowCopy) Map.empty[String, AnyRef] else { - val draftHierarchy = getHierarchy(obj.identifier, obj.pkgVersion, readerConfig).get - val publishedNodeId = obj.identifier.replaceAll(".img","") - val publishedHierarchy = if (obj.pkgVersion > 0) { getHierarchy(publishedNodeId, 0, readerConfig).get } else Map.empty[String, AnyRef] - val draftDIALcodesMap: mutable.Map[List[String], String] = mutable.Map.empty[List[String], String] - getDIALListFromHierarchy(draftHierarchy, draftDIALcodesMap) - val publishedDIALcodesMap: mutable.Map[List[String], String] = mutable.Map.empty[List[String], String] - if(publishedHierarchy.nonEmpty) getDIALListFromHierarchy(publishedHierarchy, publishedDIALcodesMap) - - if(obj.metadata.contains("dialcodes")) { - val strDialcodes = ScalaJsonUtil.serialize(obj.metadata("dialcodes")) - draftDIALcodesMap += (ScalaJsonUtil.deserialize[List[String]](strDialcodes) -> draftHierarchy("identifier").asInstanceOf[String]) - } - if(publishedHierarchy.nonEmpty && publishedHierarchy.contains("dialcodes")) { - val strDialcodes = ScalaJsonUtil.serialize(publishedHierarchy("dialcodes")) - publishedDIALcodesMap += (ScalaJsonUtil.deserialize[List[String]](strDialcodes) -> publishedHierarchy("identifier").asInstanceOf[String]) - } - - Map("addContextDialCodes" -> draftDIALcodesMap.filter(rec=> rec._1 != null), "removeContextDialCodes" -> (publishedDIALcodesMap -- draftDIALcodesMap.keySet).filter(rec=> rec._1 != null)) - } - - DialContextMap - } - - private def getDIALListFromHierarchy(data: Map[String, AnyRef], dialcodeMap: mutable.Map[List[String], String]): Unit = { - val dialCodes = data.getOrElse("dialcodes", List.empty[String]).asInstanceOf[List[String]] - if (StringUtils.equals(data.getOrElse("visibility", "").asInstanceOf[String], "Parent") && dialCodes!=null && dialCodes.nonEmpty) dialcodeMap += (dialCodes -> data("identifier").asInstanceOf[String]) - val children = data.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]] - if (children.nonEmpty) { - for (child <- children) { - getDIALListFromHierarchy(child, dialcodeMap) - } - } - } - -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ContentPublisher.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ContentPublisher.scala deleted file mode 100644 index 98db2041c..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ContentPublisher.scala +++ /dev/null @@ -1,218 +0,0 @@ -package org.sunbird.job.content.publish.helpers - -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils -import org.apache.tika.Tika -import org.slf4j.LoggerFactory -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers._ -import org.sunbird.job.util._ - -import java.io.{File, IOException} -import java.nio.file.Files -import java.util -import java.util.regex.Pattern -import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer -import scala.concurrent.ExecutionContext - -trait ContentPublisher extends ObjectReader with ObjectValidator with ObjectEnrichment with EcarGenerator with ObjectUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[ContentPublisher]) - private val level4MimeTypes = List(MimeType.X_Youtube, MimeType.PDF, MimeType.MSWORD, MimeType.EPUB, MimeType.H5P_Archive, MimeType.X_URL) - private val level4ContentTypes = List("Course", "CourseUnit", "LessonPlan", "LessonPlanUnit") - private val pragmaMimeTypes = List(MimeType.X_Youtube, MimeType.PDF) // Ne to check for other mimetype - private val youtubeMimetypes = List(MimeType.X_Youtube, MimeType.Youtube) - private val validateArtifactUrlMimetypes = List(MimeType.PDF, MimeType.EPUB, MimeType.MSWORD) - private val ignoreValidationMimeType = List(MimeType.Collection, MimeType.Plugin_Archive, MimeType.ASSETS) - private val YOUTUBE_REGEX = "^(http(s)?:\\/\\/)?((w){3}.)?youtu(be|.be)?(\\.com)?\\/.+" - - override def getExtData(identifier: String, pkgVersion: Double, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] = { - mimeType match { - case MimeType.ECML_Archive => - val ecmlBody = getContentBody(identifier, readerConfig) - Some(ObjectExtData(Some(Map[String, AnyRef]("body" -> ecmlBody)))) - case _ => - None - } - } - - override def getHierarchy(identifier: String, pkgVersion: Double, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = None - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, - cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, - definitionConfig: DefinitionConfig): Option[ObjectData] = { - val contentConfig = config.asInstanceOf[ContentPublishConfig] - val extraMeta = Map("pkgVersion" -> (obj.pkgVersion + 1).asInstanceOf[AnyRef], "lastPublishedOn" -> getTimeStamp, - "flagReasons" -> null, "body" -> null, "publishError" -> null, "variants" -> null, "downloadUrl" -> null) - val contentSize = obj.metadata.getOrElse("size", 0).toString.toDouble - val configSize = contentConfig.artifactSizeForOnline - val publishType = obj.getString("publish_type", "Public") - val status = if (StringUtils.equalsIgnoreCase("Unlisted", publishType)) "Unlisted" else "Live" - val updatedMeta: Map[String, AnyRef] = if (contentSize > configSize) obj.metadata ++ extraMeta ++ Map("contentDisposition" -> "online-only", "status" -> status) else obj.metadata ++ extraMeta ++ Map("status" -> status) - - val updatedCompatibilityLevel = setCompatibilityLevel(obj, updatedMeta).getOrElse(updatedMeta) - val updatedPragma = setPragma(obj, updatedCompatibilityLevel).getOrElse(updatedCompatibilityLevel) - - //delete basePath if exists - Files.deleteIfExists(new File(ExtractableMimeTypeHelper.getBasePath(obj.identifier, contentConfig.bundleLocation)).toPath) - - if (contentConfig.isECARExtractionEnabled && contentConfig.extractableMimeTypes.contains(obj.mimeType)) { - ExtractableMimeTypeHelper.copyExtractedContentPackage(obj, contentConfig, "version", cloudStorageUtil) - ExtractableMimeTypeHelper.copyExtractedContentPackage(obj, contentConfig, "latest", cloudStorageUtil) - } - val updatedPreviewUrl = updatePreviewUrl(obj, updatedPragma, cloudStorageUtil, contentConfig).getOrElse(updatedPragma) - Some(new ObjectData(obj.identifier, updatedPreviewUrl, obj.extData, obj.hierarchy)) - } - - override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = { - Some(List(obj.metadata ++ obj.extData.getOrElse(Map()).filter(p => !excludeBundleMeta.contains(p._1)))) - } - - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - def validateMetadata(obj: ObjectData, identifier: String, config: PublishConfig): List[String] = { - logger.info("Validating Content metadata for : " + obj.identifier) - val messages = ListBuffer[String]() - val artifactUrl = obj.getString("artifactUrl", "") - if (ignoreValidationMimeType.contains(obj.mimeType)) { - // Validation not required. Nothing to do. - } else if (obj.mimeType.equalsIgnoreCase(MimeType.ECML_Archive)) { // Either 'body' or 'artifactUrl' is needed - if ((obj.extData.isEmpty || !obj.extData.get.contains("body") || obj.extData.get.getOrElse("body",null) == null || obj.extData.get.getOrElse("body", "").asInstanceOf[String].isEmpty || obj.extData.get.getOrElse("body", "").asInstanceOf[String].isBlank) && (artifactUrl.isEmpty || artifactUrl.isBlank)) { - messages += s"""Either 'body' or 'artifactUrl' are required for processing of ECML content for : $identifier""" - } - } else { - val allowedExtensionsWord: util.List[String] = config.asInstanceOf[ContentPublishConfig].allowedExtensionsWord - - if (StringUtils.isBlank(artifactUrl)) - messages += s"""There is no artifactUrl available for : $identifier""" - else if (youtubeMimetypes.contains(obj.mimeType) && !isValidYouTubeUrl(artifactUrl)) - messages += s"""Invalid youtube Url = $artifactUrl for : $identifier""" - else if (validateArtifactUrlMimetypes.contains(obj.mimeType) && !isValidUrl(artifactUrl, obj.mimeType, allowedExtensionsWord)) { // valid url check by downloading the file and then delete it - // artifactUrl + valid url check by downloading the file - obj.mimeType match { - case MimeType.PDF => - messages += s"""Error! Invalid File Extension. Uploaded file $artifactUrl is not a pdf file for : $identifier""" - case MimeType.EPUB => - messages += s"""Error! Invalid File Extension. Uploaded file $artifactUrl is not a epub file for : $identifier""" - case MimeType.MSWORD => - messages += s"""Error! Invalid File Extension. | Uploaded file $artifactUrl should be among the Allowed_file_extensions for mimeType doc $allowedExtensionsWord for : $identifier""" - } - } - } - messages.toList - } - - private def isValidYouTubeUrl(artifactUrl: String): Boolean = { - logger.info(s"Validating if the given youtube url = $artifactUrl is valid or not.") - Pattern.matches(YOUTUBE_REGEX, artifactUrl) - } - - private def isValidUrl(url: String, mimeType: String, allowedExtensionsWord: util.List[String]): Boolean = { - val destPath = s"""${File.separator}tmp${File.separator}validUrl""" - // val destPath = s"""$bundlePath${File.separator}${StringUtils.replace(id, ".img", "")}""" - var isValid = false - try { - val file: File = FileUtils.downloadFile(url, destPath) - if (exceptionChecks(mimeType, file, allowedExtensionsWord)) isValid = true - } catch { - case e: Exception => - logger.error("isValidUrl: Error while checking mimeType.") - e.printStackTrace() - throw new InvalidInputException("isValidUrl: Error while checking mimeType ", e) - } finally { - FileUtils.deleteQuietly(destPath) - } - isValid - } - - private def exceptionChecks(mimeType: String, file: File, allowedExtensionsWord: util.List[String]): Boolean = { - try { - val extension = FilenameUtils.getExtension(file.getPath) - logger.info("Validating File For MimeType: " + file.getName) - if (extension.nonEmpty) { - val tika = new Tika - val fileType = tika.detect(file) - mimeType match { - case MimeType.PDF => - if (StringUtils.equalsIgnoreCase(extension, "pdf") && fileType == MimeType.PDF) return true - case MimeType.EPUB => - if (StringUtils.equalsIgnoreCase(extension, "epub") && fileType == "application/epub+zip") return true - case MimeType.MSWORD => - if (allowedExtensionsWord.contains(extension)) return true - } - } - } catch { - case e: IOException => - logger.error("exceptionChecks: Error while checking mimeType.") - e.printStackTrace() - throw new InvalidInputException("exceptionChecks: Error while checking mimeType ", e) - } - false - } - - def getObjectWithEcar(data: ObjectData, pkgTypes: List[String])(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig, httpUtil: HttpUtil): ObjectData = { - try { - logger.info("ContentPublisher:getObjectWithEcar: Ecar generation done for Content: " + data.identifier) - val ecarMap: Map[String, String] = generateEcar(data, pkgTypes) - val variants: java.util.Map[String, java.util.Map[String, String]] = ecarMap.map { case (key, value) => key.toLowerCase -> Map[String, String]("ecarUrl" -> value, "size" -> httpUtil.getSize(value).toString).asJava }.asJava - logger.info("ContentPublisher ::: getObjectWithEcar ::: ecar map ::: " + ecarMap) - val meta: Map[String, AnyRef] = Map("downloadUrl" -> ecarMap.getOrElse(EcarPackageType.FULL, ""), "variants" -> variants) - new ObjectData(data.identifier, data.metadata ++ meta, data.extData, data.hierarchy) - } catch { - case _: java.lang.IllegalArgumentException => throw new InvalidInputException(s"Invalid input found For $data.identifier") - } - } - - private def setCompatibilityLevel(obj: ObjectData, updatedMeta: Map[String, AnyRef]): Option[Map[String, AnyRef]] = { - if (level4MimeTypes.contains(obj.mimeType) - || level4ContentTypes.contains(obj.getString("contentType", ""))) { - logger.info("setting compatibility level for content id : " + obj.identifier + " as 4.") - Some(updatedMeta ++ Map("compatibilityLevel" -> 4.asInstanceOf[AnyRef])) - } else None - } - - private def setPragma(obj: ObjectData, updatedMeta: Map[String, AnyRef]): Option[Map[String, AnyRef]] = { - if (pragmaMimeTypes.contains(obj.mimeType)) { - val pgm: java.util.List[String] = obj.metadata.getOrElse("pragma", new java.util.ArrayList[String]()).asInstanceOf[java.util.List[String]] - val pragma: List[String] = pgm.asScala.toList - val value = "external" - if (!pragma.contains(value)) { - Some(updatedMeta ++ Map("pragma" -> (pragma ++ List(value)))) - } else None - } else None - } - - private def updatePreviewUrl(obj: ObjectData, updatedMeta: Map[String, AnyRef], cloudStorageUtil: CloudStorageUtil, config: ContentPublishConfig): Option[Map[String, AnyRef]] = { - if (StringUtils.isNotBlank(obj.mimeType)) { - logger.debug("Checking Required Fields For: " + obj.mimeType) - obj.mimeType match { - case MimeType.Collection | MimeType.Plugin_Archive | MimeType.Android_Package | MimeType.ASSETS => - None - case MimeType.ECML_Archive | MimeType.HTML_Archive | MimeType.H5P_Archive => - val latestFolderS3Url = ExtractableMimeTypeHelper.getCloudStoreURL(obj, cloudStorageUtil, config) - val relativeLatestFolder = if(config.isrRelativePathEnabled) { - val paths = config.config.getStringList("cloudstorage.write_base_path").asScala.toArray - val repArray = CSPMetaUtil.getReplacementData(paths, config.getString("cloudstorage.read_base_path", "")) - StringUtils.replaceEach(latestFolderS3Url, paths, repArray) - } else latestFolderS3Url - val updatedPreviewUrl = updatedMeta ++ Map("previewUrl" -> relativeLatestFolder, "streamingUrl" -> latestFolderS3Url) - Some(updatedPreviewUrl) - case _ => - val artifactUrl = obj.getString("artifactUrl", null) - val updatedPreviewUrl = updatedMeta ++ Map("previewUrl" -> artifactUrl) - if (config.isStreamingEnabled && !config.streamableMimeType.contains(obj.mimeType)) Some(updatedPreviewUrl ++ Map("streamingUrl" -> artifactUrl)) else Some(updatedPreviewUrl) - } - } else None - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ECMLExtractor.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ECMLExtractor.scala deleted file mode 100644 index ef1d02e58..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ECMLExtractor.scala +++ /dev/null @@ -1,8 +0,0 @@ -package org.sunbird.job.content.publish.helpers - -import org.sunbird.job.content.publish.processor.{BaseProcessor, MissingAssetValidatorProcessor} -import org.sunbird.job.util.CloudStorageUtil - -class ECMLExtractor(basePath: String, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil) extends BaseProcessor(basePath, identifier) with MissingAssetValidatorProcessor { - -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ExtractableMimeTypeHelper.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ExtractableMimeTypeHelper.scala deleted file mode 100644 index 9da549256..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/ExtractableMimeTypeHelper.scala +++ /dev/null @@ -1,246 +0,0 @@ -package org.sunbird.job.content.publish.helpers - -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.content.publish.processor.{JsonParser, Media, Plugin, XmlParser} -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.{CloudStorageUtil, FileUtils, Slug} -import org.xml.sax.{InputSource, SAXException} - -import java.io._ -import javax.xml.parsers.{DocumentBuilderFactory, ParserConfigurationException} -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext, Future} - -object ExtractableMimeTypeHelper { - - private[this] val logger = LoggerFactory.getLogger("ExtractableMimeTypeHelper") - private val extractablePackageExtensions = List(".zip", ".h5p", ".epub") - - def getCloudStoreURL(obj: ObjectData, cloudStorageUtil: CloudStorageUtil, config: ContentPublishConfig): String = { - val path = getExtractionPath(obj, config, "latest") - cloudStorageUtil.getURI(path, Option.apply(config.extractableMimeTypes.contains(obj.mimeType))) - } - - private def getExtractionPath(obj: ObjectData, config: ContentPublishConfig, suffix: String): String = { - obj.mimeType match { - case "application/vnd.ekstep.ecml-archive" => config.contentFolder + File.separator + "ecml" + File.separator + obj.identifier + "-" + suffix - case "application/vnd.ekstep.html-archive" => config.contentFolder + File.separator + "html" + File.separator + obj.identifier + "-" + suffix - case "application/vnd.ekstep.h5p-archive" => config.contentFolder + File.separator + "h5p" + File.separator + obj.identifier + "-" + suffix - case _ => "" - } - } - - def getBasePath(objectId: String, tempFileLocation: String): String = { - if (!StringUtils.isBlank(objectId)) tempFileLocation + File.separator + System.currentTimeMillis + "_temp" + File.separator + objectId else "" - } - - def copyExtractedContentPackage(obj: ObjectData, contentConfig: ContentPublishConfig, extractionType: String, cloudStorageUtil: CloudStorageUtil): Unit = { - if (!isExtractedSnapshotExist(obj)) throw new InvalidInputException("Error! Snapshot Type Extraction doesn't Exists.") - val sourcePrefix = getExtractionPath(obj, contentConfig, "snapshot") - val destinationPrefix = getExtractionPath(obj, contentConfig, extractionType) - cloudStorageUtil.copyObjectsByPrefix(sourcePrefix, destinationPrefix, isFolder = true) - } - - private def isExtractedSnapshotExist(obj: ObjectData): Boolean = { - extractablePackageExtensions.exists(key => StringUtils.endsWithIgnoreCase(obj.getString("artifactUrl", null), key)) - } - - def processECMLBody(obj: ObjectData, config: ContentPublishConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Map[String, AnyRef] = { - val basePath = config.bundleLocation + "/" + System.currentTimeMillis + "_tmp" + "/" + obj.identifier - val ecmlBody = obj.extData.get.getOrElse("body", "").toString - val ecmlType: String = getECMLType(ecmlBody) - val ecrfObj: Plugin = getEcrfObject(ecmlType, ecmlBody) - - // localize assets - download assets to local base path (tmp folder) for validation - localizeAssets(obj.identifier, ecrfObj, basePath, config) - - // validate assets - val processedEcrf: Plugin = new ECMLExtractor(basePath, obj.identifier).process(ecrfObj) - - // getECMLString - val processedEcml: String = getEcmlStringFromEcrf(processedEcrf, ecmlType) - - // write ECML String to basePath - writeECMLFile(basePath, processedEcml, ecmlType) - - // create zip package - val zipFileName: String = basePath + File.separator + System.currentTimeMillis + "_" + Slug.makeSlug(obj.identifier) + ".zip" - FileUtils.createZipPackage(basePath, zipFileName) - - // upload zip file to blob and set artifactUrl - val result: Array[String] = uploadArtifactToCloud(new File(zipFileName), obj.identifier, None, config) - - // upload local extracted directory to blob - extractPackageInCloud(new File(zipFileName), obj, "snapshot", slugFile = true, basePath, config) - - val contentSize = (new File(zipFileName)).length - - // delete local folder - FileUtils.deleteQuietly(basePath) - - obj.metadata ++ Map("artifactUrl" -> result(1), "cloudStorageKey" -> result(0), "size" -> contentSize.asInstanceOf[AnyRef]) - } - - private def getEcrfObject(ecmlType: String, ecmlBody: String): Plugin = { - ecmlType match { - case "ecml" => XmlParser.parse(ecmlBody) - case "json" => JsonParser.parse(ecmlBody) - case _ => classOf[Plugin].newInstance() - } - } - - private def getEcmlStringFromEcrf(processedEcrf: Plugin, ecmlType: String): String = { - ecmlType match { - case "ecml" => XmlParser.toString(processedEcrf) - case "json" => JsonParser.toString(processedEcrf) - case _ => "" - } - } - - private def getECMLType(contentBody: String): String = { - if (!StringUtils.isBlank(contentBody)) { - if (isValidJSON(contentBody)) "json" - else if (isValidXML(contentBody)) "ecml" - else throw new InvalidInputException("Invalid Content Body") - } - else throw new InvalidInputException("Invalid Content Body. ECML content should have body.") - } - - private def isValidJSON(contentBody: String): Boolean = { - if (!StringUtils.isBlank(contentBody)) try { - val objectMapper = new ObjectMapper - objectMapper.enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) - objectMapper.readTree(contentBody) - true - } catch { - case ex@(_: IOException) => - logger.error("isValidJSON - Exception when validating the JSON :: ", ex) - false - } - else false - } - - private def isValidXML(contentBody: String): Boolean = { - if (!StringUtils.isBlank(contentBody)) try { - val dbFactory = DocumentBuilderFactory.newInstance - val dBuilder = dbFactory.newDocumentBuilder - dBuilder.parse(new InputSource(new StringReader(contentBody))) - true - } catch { - case ex@(_: ParserConfigurationException | _: SAXException | _: IOException) => - logger.error("isValidXML - Exception when validating the XML :: ", ex) - false - } else false - } - - private def writeECMLFile(basePath: String, ecml: String, ecmlType: String): Unit = { - try { - if (StringUtils.isBlank(ecml)) throw new InvalidInputException("[Unable to write Empty ECML File.]") - if (StringUtils.isBlank(ecmlType)) throw new InvalidInputException("[System is in a fix between (XML & JSON) ECML Type.]") - val file = new File(basePath + "/" + "index." + ecmlType) - FileUtils.writeStringToFile(file, ecml) - } catch { - case e: Exception => - logger.error("writeECMLFile - Exception when write ECML file :: ", e) - throw new Exception("[Unable to Write ECML File.]", e) - } - } - - private def extractPackageInCloud(uploadFile: File, obj: ObjectData, extractionType: String, slugFile: Boolean, basePath: String, config: ContentPublishConfig)(implicit cloudStorageUtil: CloudStorageUtil) = { - val file = Slug.createSlugFile(uploadFile) - val mimeType = obj.mimeType - validationForCloudExtraction(file, extractionType, mimeType, config) - if (config.extractableMimeTypes.contains(mimeType)) { - cloudStorageUtil.uploadDirectory(getExtractionPath(obj, config, extractionType), new File(basePath), Option(slugFile)) - } - } - - private def uploadArtifactToCloud(uploadedFile: File, identifier: String, filePath: Option[String] = None, config: ContentPublishConfig)(implicit cloudStorageUtil: CloudStorageUtil): Array[String] = { - val urlArray = { - try { - val folder = if (filePath.isDefined) filePath.get + File.separator + config.contentFolder + File.separator + Slug.makeSlug(identifier, isTransliterate = true) + File.separator + config.artifactFolder - else config.contentFolder + File.separator + Slug.makeSlug(identifier, isTransliterate = true) + File.separator + config.artifactFolder - cloudStorageUtil.uploadFile(folder, uploadedFile) - } catch { - case e: Exception => - cloudStorageUtil.deleteFile(uploadedFile.getAbsolutePath, Option(false)) - logger.error("Error while uploading the Artifact file.", e) - throw new Exception("Error while uploading the Artifact File.", e) - } - } - urlArray - } - - private def validationForCloudExtraction(file: File, extractionType: String, mimeType: String, config: ContentPublishConfig): Unit = { - if (!file.exists() || (!extractablePackageExtensions.contains("." + FilenameUtils.getExtension(file.getName)) && config.extractableMimeTypes.contains(mimeType))) - throw new InvalidInputException("Error! File doesn't Exist.") - if (extractionType == null) - throw new InvalidInputException("Error! Invalid Content Extraction Type.") - } - - - private def localizeAssets(contentId: String, ecrfObj: Plugin, basePath: String, config: ContentPublishConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Unit = { - val medias: List[Media] = if (null != ecrfObj && null != ecrfObj.manifest) ecrfObj.manifest.medias else List.empty - if (null != medias && medias.nonEmpty) processAssetsDownload(contentId, medias, basePath, config) - } - - private def processAssetsDownload(contentId: String, medias: List[Media], basePath: String, config: ContentPublishConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Map[String, String] = { - val downloadResultMap = Await.result(downloadAssetFiles(contentId, medias, basePath, config), Duration.apply(config.assetDownloadDuration)) - downloadResultMap.filter(record => record.nonEmpty).flatten.toMap - } - - private def downloadAssetFiles(identifier: String, mediaFiles: List[Media], basePath: String, config: ContentPublishConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Future[List[Map[String, String]]] = { - val futures = mediaFiles.map(mediaFile => { - logger.info(s"ExtractableMimeTypeHelper ::: downloadAssetFiles ::: Processing file: ${mediaFile.id} for : " + identifier) - if (!StringUtils.equals("youtube", mediaFile.`type`) && !StringUtils.isBlank(mediaFile.src) && !StringUtils.isBlank(mediaFile.`type`)) { - val downloadPath = if (isWidgetTypeAsset(mediaFile.`type`)) basePath + "/" + "widgets" else basePath + "/" + "assets" - val subFolder = { - if (!mediaFile.src.startsWith("http") && !mediaFile.src.startsWith("https")) { - val f = new File(mediaFile.src) - if (f.exists) f.delete - StringUtils.stripStart(f.getParent, "/") - } else "" - } - val fDownloadPath = if (StringUtils.isNotBlank(subFolder)) downloadPath + File.separator + subFolder + File.separator else downloadPath + File.separator - createDirectoryIfNeeded(fDownloadPath) - logger.info(s"ExtractableMimeTypeHelper ::: downloadAssetFiles ::: fDownloadPath: $fDownloadPath & src : ${mediaFile.src}") - - if (mediaFile.src.startsWith("https://") || mediaFile.src.startsWith("http://")) { - FileUtils.downloadFile(mediaFile.src, fDownloadPath) - } - else { - if (mediaFile.src.contains("assets/public")) { - try { - cloudStorageUtil.downloadFile(fDownloadPath, StringUtils.replace(mediaFile.src, "//", "/").substring(mediaFile.src.indexOf("assets/public") + 14)) - } catch { - case _: Exception => cloudStorageUtil.downloadFile(fDownloadPath, mediaFile.src.substring(mediaFile.src.indexOf("assets/public") + 13)) - } - } - else if (mediaFile.src.startsWith(File.separator)) { - cloudStorageUtil.downloadFile(fDownloadPath, StringUtils.replace(mediaFile.src.substring(1), "//", "/")) - } else { - cloudStorageUtil.downloadFile(fDownloadPath, StringUtils.replace(mediaFile.src, "//", "/")) - } - } - val downloadedFile = new File(fDownloadPath + mediaFile.src.split("/").last) - logger.info("Downloaded file : " + mediaFile.src + " - " + downloadedFile + " | [Content Id '" + identifier + "']") - - Map(mediaFile.id -> downloadedFile.getName) - } else Map.empty[String, String] - }) - Future(futures) - } - - private def isWidgetTypeAsset(assetType: String): Boolean = StringUtils.equalsIgnoreCase(assetType, "js") || StringUtils.equalsIgnoreCase(assetType, "css") || StringUtils.equalsIgnoreCase(assetType, "json") || StringUtils.equalsIgnoreCase(assetType, "plugin") - - private def createDirectoryIfNeeded(directoryName: String): Unit = { - val theDir = new File(directoryName) - if (!theDir.exists) theDir.mkdirs - } - -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/SyncMessagesGenerator.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/SyncMessagesGenerator.scala deleted file mode 100644 index 56ffffc1b..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/helpers/SyncMessagesGenerator.scala +++ /dev/null @@ -1,167 +0,0 @@ -package org.sunbird.job.content.publish.helpers - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.ObjectDefinition -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} - -import scala.collection.mutable -import scala.collection.mutable.ListBuffer - -trait SyncMessagesGenerator { - - private[this] val logger = LoggerFactory.getLogger(classOf[SyncMessagesGenerator]) - - private def getIndexDocument(identifier: String)(esUtil: ElasticSearchUtil): scala.collection.mutable.Map[String, AnyRef] = { - val documentJson: String = esUtil.getDocumentAsString(identifier) - if (documentJson != null && documentJson.nonEmpty) ScalaJsonUtil.deserialize[scala.collection.mutable.Map[String, AnyRef]](documentJson) else scala.collection.mutable.Map[String, AnyRef]() - } - - private def getJsonMessage(message: Map[String, Any], definition: ObjectDefinition, nestedFields: List[String], ignoredFields: List[String]): Map[String, AnyRef] = { - val indexDocument = scala.collection.mutable.Map[String, AnyRef]() - val transactionData = message.getOrElse("transactionData", Map[String, Any]()).asInstanceOf[Map[String, Any]] - logger.debug("SyncMessagesGenerator:: getJsonMessage:: transactionData:: " + transactionData) - if (transactionData.nonEmpty) { - val addedProperties = transactionData.getOrElse("properties", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - logger.debug("SyncMessagesGenerator:: getJsonMessage:: definition.externalProperties:: " + definition.externalProperties) - addedProperties.foreach(property => { - if (!definition.externalProperties.contains(property._1)) { - val propertyNewValue: AnyRef = property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) - if (propertyNewValue == null) indexDocument.remove(property._1) else indexDocument.put(property._1, addMetadataToDocument(property._1, propertyNewValue, nestedFields)) - } - }) - -// val addedRelations = transactionData.getOrElse("addedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] -// if (addedRelations.nonEmpty) { -// addedRelations.foreach(rel => { -// val direction = rel.getOrElse("dir", "").asInstanceOf[String] -// val relationType = rel.getOrElse("rel", "").asInstanceOf[String] -// val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] -// val title = definition.relationLabel(targetObjType, direction, relationType) -// if (title.nonEmpty) { -// val list = indexDocument.getOrElse(title.get, List[String]()).asInstanceOf[List[String]] -// val id = rel.getOrElse("id", "").asInstanceOf[String] -// if (!list.contains(id)) indexDocument.put(title.get, (id :: list).asInstanceOf[AnyRef]) -// } -// }) -// } -// -// val removedRelations = transactionData.getOrElse("removedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] -// removedRelations.foreach(rel => { -// val direction = rel.getOrElse("dir", "").asInstanceOf[String] -// val relationType = rel.getOrElse("rel", "").asInstanceOf[String] -// val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] -// val title = definition.relationLabel(targetObjType, direction, relationType) -// if (title.nonEmpty) { -// val list = indexDocument.getOrElse(title.get, List[String]()).asInstanceOf[List[String]] -// val id = rel.getOrElse("id", "").asInstanceOf[String] -// if (list.contains(id)) { -// val updatedList = list diff List(id) -// indexDocument.put(title.get, updatedList.asInstanceOf[AnyRef]) -// } -// } -// }) - } - - //Ignored fields are removed-> it can be a propertyName or relation Name - indexDocument --= ignoredFields - - indexDocument.put("graph_id", message.getOrElse("graphId", "domain").asInstanceOf[String]) - indexDocument.put("node_id", message.getOrElse("nodeGraphId",0).asInstanceOf[AnyRef]) - indexDocument.put("identifier", message.getOrElse("nodeUniqueId", "").asInstanceOf[String]) - indexDocument.put("objectType", message.getOrElse("objectType", "").asInstanceOf[String]) - indexDocument.put("nodeType", message.getOrElse("nodeType", "").asInstanceOf[String]) - - logger.info("SyncMessagesGenerator:: getJsonMessage:: final indexDocument:: " + indexDocument) - - indexDocument.toMap - } - - private def addMetadataToDocument(propertyName: String, propertyValue: AnyRef, nestedFields: List[String]): AnyRef = { - if (nestedFields.contains(propertyName)) { - propertyValue match { - case propVal: String => ScalaJsonUtil.deserialize[AnyRef](propVal) - case _ => propertyValue - } - } else propertyValue - } - - def getMessages(nodes: List[ObjectData], definition: ObjectDefinition, nestedFields: List[String], errors: mutable.Map[String, String])(esUtil: ElasticSearchUtil): Map[String, Map[String, AnyRef]] = { - val messages = collection.mutable.Map.empty[String, Map[String, AnyRef]] - for (node <- nodes) { - try { - if (definition.getRelationLabels() != null) { - val nodeMap = getNodeMap(node) - logger.debug("SyncMessagesGenerator:: getMessages:: nodeMap:: " + nodeMap) - val message = getJsonMessage(nodeMap, definition, nestedFields, List.empty) - logger.debug("SyncMessagesGenerator:: getMessages:: message:: " + message) - messages.put(node.identifier, message) - } - } catch { - case e: Exception => e.printStackTrace() - errors.put(node.identifier, e.getMessage) - } - } - messages.toMap - } - - - private def getNodeMap(node: ObjectData): Map[String, AnyRef] = { - val transactionData = collection.mutable.Map.empty[String, AnyRef] - if (null != node.metadata && node.metadata.nonEmpty) { - val propertyMap = collection.mutable.Map.empty[String, AnyRef] - for (key <- node.metadata.keySet) { - if (StringUtils.isNotBlank(key)) { - val valueMap = collection.mutable.Map.empty[String, AnyRef] - valueMap.put("ov", null) // old value - valueMap.put("nv", node.metadata(key)) // new value - - // temporary check to not sync body and editorState - if (!StringUtils.equalsIgnoreCase("body", key) && !StringUtils.equalsIgnoreCase("editorState", key)) propertyMap.put(key, valueMap.toMap) - } - } - transactionData.put("properties", propertyMap.toMap) - } - else transactionData.put("properties", Map.empty[String,AnyRef]) - - val relations = ListBuffer.empty[Map[String, AnyRef]] - // add IN relations -// if (null != node.metadata.inRelations && node.metadata.inRelations.nonEmpty) { -// for (rel <- node.metadata.inRelations) { -// val relMap = Map("rel" -> rel.relationType, "id" -> rel.startNodeId, "dir" -> "IN", "type" -> rel.startNodeObjectType, "label" -> getLabel(rel.getStartNodeMetadata)) -// relations += relMap -// } -// } -// // add OUT relations -// if (null != node.getOutRelations && !node.getOutRelations.isEmpty) { -// for (rel <- node.getOutRelations) { -// val relMap = Map("rel" -> rel.getRelationType, "id" -> rel.getEndNodeId, "dir" -> "OUT", "type" -> rel.getEndNodeObjectType, "label" -> getLabel(rel.getEndNodeMetadata)) -// relations += relMap -// } -// } - transactionData.put("addedRelations", relations.toList) - - Map("operationType"-> "UPDATE", "graphId" -> node.metadata.getOrElse("graphId","domain").asInstanceOf[String], "nodeGraphId"-> 0.asInstanceOf[AnyRef], "nodeUniqueId"-> node.identifier, "objectType"-> node.dbObjType, - "nodeType"-> "DATA_NODE", "transactionData" -> transactionData.toMap, "syncMessage" -> true.asInstanceOf[AnyRef]) - } - - - def getDocument(node: ObjectData, updateRequest: Boolean, nestedFields: List[String])(esUtil: ElasticSearchUtil): Map[String, AnyRef] = { - val message = getNodeMap(node) - val identifier: String = message.getOrElse("nodeUniqueId", "").asInstanceOf[String] - val indexDocument = mutable.Map[String, AnyRef]() - val transactionData: Map[String, AnyRef] = message.getOrElse("transactionData", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - if (transactionData.nonEmpty) { - val addedProperties: Map[String, AnyRef] = transactionData.getOrElse("properties", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - addedProperties.foreach(property => { - val propertyNewValue: AnyRef = property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) - if (propertyNewValue == null) indexDocument.remove(property._1) else indexDocument.put(property._1, addMetadataToDocument(property._1, propertyNewValue, nestedFields)) - }) - } - indexDocument.put("identifier", message.getOrElse("nodeUniqueId", "").asInstanceOf[String]) - indexDocument.put("objectType", message.getOrElse("objectType", "").asInstanceOf[String]) - indexDocument.toMap - } - -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/BaseProcessor.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/BaseProcessor.scala deleted file mode 100644 index 4bc1fd197..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/BaseProcessor.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.sunbird.job.content.publish.processor - -import org.sunbird.job.util.CloudStorageUtil - -class BaseProcessor(basePath: String, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil) extends IProcessor(basePath, identifier) { - override def process(ecrf: Plugin): Plugin = { - ecrf - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/EcrfObject.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/EcrfObject.scala deleted file mode 100644 index ad861ee6f..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/EcrfObject.scala +++ /dev/null @@ -1,19 +0,0 @@ -package org.sunbird.job.content.publish.processor - -case class Plugin(id: String, data: Map[String, AnyRef], innerText: String, cData: String, childrenPlugin: List[Plugin], manifest: Manifest, controllers: List[Controller], events: List[Event]) { - def this() = this("", null, "", "", null, null, null, null) -} -case class Manifest(id: String, data: Map[String, AnyRef], innerText: String, cData: String, medias: List[Media]) { - def this() = this("", null, "", "", null) -} -case class Controller(id: String, data: Map[String, AnyRef], innerText: String, cData: String) { - def this() = this("", null, "", "") -} -case class Media(id: String, data: Map[String, AnyRef], innerText: String, cData: String, src: String, `type`: String, childrenPlugin: List[Plugin]) { - def this() = this("", null, "", "", "", "", null) -} -case class Event(id: String, data: Map[String, AnyRef], innerText: String, cData: String, childrenPlugin: List[Plugin]) { - def this() = this("", null, "", "", null) -} - - diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/IProcessor.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/IProcessor.scala deleted file mode 100644 index 2429f8b2e..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/IProcessor.scala +++ /dev/null @@ -1,16 +0,0 @@ -package org.sunbird.job.content.publish.processor - -import org.sunbird.job.util.CloudStorageUtil - -abstract class IProcessor(basePath: String, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil) { - - implicit val ss = cloudStorageUtil.getService - - val widgetTypeAssets: List[String] = List("js", "css", "json", "plugin") - - def process(ecrf: Plugin): Plugin - - def getBasePath(): String = basePath - - def getIdentifier(): String = identifier -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/JsonParser.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/JsonParser.scala deleted file mode 100644 index 0d5e024b7..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/JsonParser.scala +++ /dev/null @@ -1,218 +0,0 @@ -package org.sunbird.job.content.publish.processor - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.util.ScalaJsonUtil - -import scala.collection.mutable.ListBuffer - -object JsonParser { - val nonPluginElements: List[String] = List("manifest", "controller", "media", "events", "event", "__cdata", "__text") - - def parse(jsonString: String): Plugin = { - val jsonMap: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](jsonString) - processDocument(jsonMap) - } - - def processDocument(json: Map[String, AnyRef]): Plugin = { - if (json.keySet.contains("theme")) { - val root = json("theme").asInstanceOf[Map[String, AnyRef]] - Plugin(getId(root), getData(root, "theme"), getInnerText(root), getCdata(root), getChildrenPlugin(root), getManifest(root, validateMedia = true), getControllers(root), getEvents(root)) - } else classOf[Plugin].newInstance() - } - - private def getDataFromMap(map: Map[String, AnyRef], key: String): String = { - if (null != map && map.keySet.contains(key)) { - map(key) match { - case str: String => str - case _ => map(key).toString - } - } else "" - } - - def getId(jsonObject: Map[String, AnyRef]): String = getDataFromMap(jsonObject, "id") - - def getData(jsonObject: Map[String, AnyRef], elementName: String): Map[String, AnyRef] = { - if (null != jsonObject && StringUtils.isNotBlank(elementName)) { - var result = jsonObject.filter(p => !p._1.equalsIgnoreCase("__cdata") && !p._1.equalsIgnoreCase("__text")) - result += ("cwp_element_name" -> elementName) - result - } else Map[String, AnyRef]() - } - - def getInnerText(jsonObject: Map[String, AnyRef]): String = getDataFromMap(jsonObject, "__text") - - def getCdata(jsonObject: Map[String, AnyRef]): String = ScalaJsonUtil.serialize(jsonObject.getOrElse("__cdata", "")) - - def getChildrenPlugin(jsonObject: Map[String, AnyRef]): List[Plugin] = { - val childPluginList: ListBuffer[Plugin] = ListBuffer() - val filteredObject = jsonObject.filter(entry => null != entry._2) - childPluginList ++= filteredObject.filter(entry => entry._2.isInstanceOf[List[Map[String, AnyRef]]] && !nonPluginElements.contains(entry._1)).map(entry => { - val objectList: List[Map[String, AnyRef]] = entry._2.asInstanceOf[List[Map[String, AnyRef]]] - objectList.map(obj => Plugin(getId(obj), getData(obj, entry._1), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj), getManifest(obj, validateMedia = false), getControllers(obj), getEvents(obj))) - }).toList.flatten - childPluginList ++= filteredObject.filter(entry => entry._2.isInstanceOf[Map[String, AnyRef]] && !nonPluginElements.contains(entry._2)).map(entry => { - val obj = entry._2.asInstanceOf[Map[String, AnyRef]] - Plugin(getId(obj), getData(obj, entry._1), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj), getManifest(obj, validateMedia = false), getControllers(obj), getEvents(obj)) - }).toList - childPluginList.toList - } - - def getManifest(jsonObject: Map[String, AnyRef], validateMedia: Boolean): Manifest = { - if (null != jsonObject && jsonObject.keySet.contains("manifest") && jsonObject("manifest").isInstanceOf[List[Map[String, AnyRef]]]) throw new Exception("Error! JSON Object is Expected for the Element. manifest") - else if (null != jsonObject && jsonObject.keySet.contains("manifest") && jsonObject("manifest").isInstanceOf[Map[String, AnyRef]] && jsonObject("manifest").asInstanceOf[Map[String, AnyRef]].keySet.contains("media")) { - val manifestObject = jsonObject("manifest").asInstanceOf[Map[String, AnyRef]] - Manifest(getId(manifestObject), getData(manifestObject, "manifest"), getInnerText(manifestObject), getCdata(manifestObject), getMedias(manifestObject("media"), validateMedia)) - } else classOf[Manifest].newInstance() - } - - def getControllers(jsonObject: Map[String, AnyRef]): List[Controller] = { - if (null != jsonObject && jsonObject.keySet.contains("controller") && jsonObject("controller").isInstanceOf[List[Map[String, Object]]]) { - val controllerList: List[Map[String, AnyRef]] = jsonObject("controller").asInstanceOf[List[Map[String, Object]]] - controllerList.map(obj => { - validateController(obj) - Controller(getId(obj), getData(obj, "controller"), getInnerText(obj), getCdata(obj)) - }) - } else if (null != jsonObject && jsonObject.keySet.contains("controller") && jsonObject("controller").isInstanceOf[Map[String, Object]]) { - val obj = jsonObject("controller").asInstanceOf[Map[String, Object]] - validateController(obj) - List(Controller(getId(obj), getData(obj, "controller"), getInnerText(obj), getCdata(obj))) - } else List() - } - - def validateController(obj: Map[String, AnyRef]): Unit = { - if (StringUtils.isBlank(getDataFromMap(obj, "id"))) - throw new InvalidInputException("Error! Invalid Controller ('id' is required.)") - - val `type` = getDataFromMap(obj, "type") - if (StringUtils.isBlank(getDataFromMap(obj, "type"))) - throw new InvalidInputException("Error! Invalid Controller ('type' is required.)") - if (!"items".equalsIgnoreCase(`type`) && !"data".equalsIgnoreCase(`type`)) - throw new InvalidInputException("Error! Invalid Controller ('type' should be either 'items' or 'data')") - } - - private def getEventsFromObject(jsonObject: AnyRef): List[Event] = { - if (null != jsonObject && jsonObject.isInstanceOf[List[Map[String, AnyRef]]]) { - val jsonList: List[Map[String, AnyRef]] = jsonObject.asInstanceOf[List[Map[String, AnyRef]]] - jsonList.map(obj => Event(getId(obj), getData(obj, "event"), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj))) - } else if (null != jsonObject && jsonObject.isInstanceOf[Map[String, AnyRef]]) { - List(Event(getId(jsonObject.asInstanceOf[Map[String, AnyRef]]), getData(jsonObject.asInstanceOf[Map[String, AnyRef]], "event"), getInnerText(jsonObject.asInstanceOf[Map[String, AnyRef]]), getCdata(jsonObject.asInstanceOf[Map[String, AnyRef]]), getChildrenPlugin(jsonObject.asInstanceOf[Map[String, AnyRef]]))) - } else List() - } - - def getEvents(jsonObject: Map[String, AnyRef]): List[Event] = { - val eventList: ListBuffer[Event] = ListBuffer() - if (jsonObject != null) { - if (jsonObject.keySet.contains("events")) { - val value = jsonObject("events") - value match { - case jsonList: List[Map[String, AnyRef]] => - eventList ++= jsonList.map(obj => Event(getId(obj), getData(obj, "event"), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj))) - case _: Map[String, AnyRef] => - eventList ++= getEventsFromObject(value) - case _ => - } - } else if (jsonObject.keySet.contains("event")) { - eventList ++= getEventsFromObject(jsonObject("event")) - } - } - eventList.toList - } - - def getMedias(manifestObject: AnyRef, validateMedia: Boolean): List[Media] = { - if (null != manifestObject && manifestObject.isInstanceOf[List[Map[String, AnyRef]]]) { - val jsonList: List[Map[String, AnyRef]] = manifestObject.asInstanceOf[List[Map[String, AnyRef]]] - jsonList.map(json => getMedia(json, validateMedia)) - } else if (null != manifestObject && manifestObject.isInstanceOf[Map[String, AnyRef]]) { - List(getMedia(manifestObject.asInstanceOf[Map[String, AnyRef]], validateMedia)) - } else List() - } - - def getMedia(mediaJson: Map[String, AnyRef], validateMedia: Boolean): Media = { - if (null != mediaJson) { - val id = getDataFromMap(mediaJson, "id") - val src = getDataFromMap(mediaJson, "src") - val `type` = getDataFromMap(mediaJson, "type") - if (validateMedia) { - if (StringUtils.isBlank(id) && isMediaIdRequiredForMediaType(`type`)) - throw new InvalidInputException("Error! Invalid Media ('id' is required.)") - if (StringUtils.isBlank(`type`)) - throw new InvalidInputException("Error! Invalid Media ('type' is required.)") - if (StringUtils.isBlank(src)) - throw new InvalidInputException("Error! Invalid Media ('src' is required.)") - } - Media(id, getData(mediaJson, "media"), getInnerText(mediaJson), getCdata(mediaJson), src, `type`, getChildrenPlugin(mediaJson)) - } else classOf[Media].newInstance() - } - - private def isMediaIdRequiredForMediaType(`type`: String): Boolean = { - !(StringUtils.isNotBlank(`type`) && (`type`.equalsIgnoreCase("js") || `type`.equalsIgnoreCase("css"))) - } - - /** - * serialize - * - * @param plugin - * @return - */ - def toString(plugin: Plugin): String = { - val map = plugin.data ++ Map[String, AnyRef]("__text" -> plugin.innerText, "__cdata" -> plugin.cData) ++ getManifestMap(plugin.manifest) ++ getControllersMap(plugin.controllers) ++ getEventsMap(plugin.events) - ScalaJsonUtil.serialize(Map[String, AnyRef]("theme" -> map)) - } - - def getManifestMap(manifest: Manifest): Map[String, AnyRef] = { - if (null != manifest && null != manifest.medias && manifest.medias.nonEmpty) { - val map: Map[String, AnyRef] = manifest.data ++ Map[String, AnyRef]("__text" -> manifest.innerText, "__cdata" -> manifest.cData) ++ getMediaMap(manifest.medias) - Map[String, AnyRef]("manifest" -> map) - } else Map[String, AnyRef]() - } - - def getControllersMap(controllers: List[Controller]): Map[String, AnyRef] = { - if (null != controllers && controllers.nonEmpty) { - val controllerMap: List[Map[String, AnyRef]] = controllers.map(controller => { - controller.data ++ Map[String, AnyRef]("__text" -> controller.innerText, "__cdata" -> controller.cData) - }) - Map[String, AnyRef]("controller" -> controllerMap) - } else Map[String, AnyRef]() - } - - def getEventsMap(events: List[Event]): Map[String, AnyRef] = { - if (null != events && events.nonEmpty) { - val eventsList: List[Map[String, AnyRef]] = events.map(event => event.data ++ Map[String, AnyRef]("__text" -> event.innerText, "__cdata" -> event.cData) ++ getChildPluginMap(event.childrenPlugin)) - - if (eventsList.length > 1) { - Map[String, AnyRef]("events" -> Map[String, AnyRef]("event" -> eventsList)) - } else { - Map[String, AnyRef]("event" -> eventsList.head) - } - } else Map[String, AnyRef]() - } - - def getMediaMap(medias: List[Media]): Map[String, AnyRef] = { - val mediasMap = medias.map(media => { - - val updatedMediaData = media.data.map(mediaInfo => { - if(mediaInfo._1.equalsIgnoreCase("src")) - ("src" -> (StringUtils.stripStart(mediaInfo._2.toString, "/"))) - else mediaInfo - }) - - updatedMediaData ++ Map[String, AnyRef]("__text" -> media.innerText, "__cdata" -> media.cData) ++ getChildPluginMap(media.childrenPlugin) - }) - Map[String, AnyRef]("media" -> mediasMap) - } - - def getChildPluginMap(plugins: List[Plugin]): Map[String, AnyRef] = { - if (null != plugins && plugins.nonEmpty) { - plugins.filter(plugin => StringUtils.isNotBlank(getDataFromMap(plugin.data, "cwp_element_name"))) - .groupBy(plugin => getDataFromMap(plugin.data, "cwp_element_name")) - .map(entry => { - if (entry._2.size == 1) - entry._1 -> entry._2.map(plugin => plugin.data ++ Map[String, AnyRef]("__text" -> plugin.innerText, "__cdata" -> plugin.cData) ++ getManifestMap(plugin.manifest) ++ getControllersMap(plugin.controllers) ++ getEventsMap(plugin.events)).head - else - entry._1 -> entry._2.map(plugin => plugin.data ++ Map[String, AnyRef]("__text" -> plugin.innerText, "__cdata" -> plugin.cData) ++ getManifestMap(plugin.manifest) ++ getControllersMap(plugin.controllers) ++ getEventsMap(plugin.events)) - }) - } else Map[String, AnyRef]() - } - -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/MissingAssetValidatorProcessor.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/MissingAssetValidatorProcessor.scala deleted file mode 100644 index 4f6e964cf..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/MissingAssetValidatorProcessor.scala +++ /dev/null @@ -1,45 +0,0 @@ -package org.sunbird.job.content.publish.processor - -import org.sunbird.job.exception.InvalidInputException - -import java.io.File - -trait MissingAssetValidatorProcessor extends IProcessor { - - abstract override def process(ecrf: Plugin): Plugin = { - validateMissingAssets(ecrf) - super.process(ecrf) - } - - def getMediaId(media: Media): String = { - if(null != media.data && media.data.nonEmpty){ - val plugin = media.data.get("plugin") - val ver = media.data.get("version") - if((null != plugin && plugin.toString.nonEmpty) && (null != ver && ver.toString.nonEmpty)) - media.id + "_" + plugin+ "_" + ver - else media.id - }else media.id - } - - def validateMissingAssets(ecrf: Plugin): Any = { - if(null != ecrf.manifest){ - val medias:List[Media] = ecrf.manifest.medias - if(null != medias){ - val mediaIds = medias.map(media => getMediaId(media)).toList - if(mediaIds.size != mediaIds.distinct.size) - throw new InvalidInputException("Error! Duplicate Asset Id used in the manifest. Asset Ids are: " - + mediaIds.groupBy(identity).mapValues(_.size).filter(p => p._2 > 1).keySet) - - val nonYoutubeMedias = medias.filter(media => !"youtube".equalsIgnoreCase(media.`type`)) - nonYoutubeMedias.map(media => { - if(widgetTypeAssets.contains(media.`type`) && !new File(getBasePath() + File.separator + "widgets" + File.separator + media.src).exists()) - throw new InvalidInputException("Error! Missing Asset. | [Asset Id '" + media.id) - else if(!widgetTypeAssets.contains(media.`type`) && !media.src.startsWith("http") && !media.src.startsWith("https") && !new File(getBasePath() + File.separator + "assets" + File.separator + media.src).exists()) - throw new InvalidInputException("Error! Missing Asset. | [Asset Id '" + media.id) - else if (!widgetTypeAssets.contains(media.`type`) && (media.src.startsWith("http") || media.src.startsWith("https")) && !new File(getBasePath() + File.separator + "assets" + File.separator + media.src.split("/").last).exists()) - throw new InvalidInputException("Error! Missing Asset. | [Asset Id '" + media.id) - }) - } - } - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/XMLLoaderWithCData.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/XMLLoaderWithCData.scala deleted file mode 100644 index 9b825187e..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/XMLLoaderWithCData.scala +++ /dev/null @@ -1,36 +0,0 @@ -package org.sunbird.job.content.publish.processor - -import org.xml.sax.InputSource -import org.xml.sax.ext.{DefaultHandler2, LexicalHandler} - -import scala.xml.factory.XMLLoader -import scala.xml.parsing.FactoryAdapter -import scala.xml.{Elem, PCData, SAXParser, TopScope} - -object XMLLoaderWithCData extends XMLLoader[Elem] { - def lexicalHandler(adapter: FactoryAdapter): LexicalHandler = - new DefaultHandler2 { - def captureCData(): Unit = { - adapter.hStack push PCData(adapter.buffer.toString) - adapter.buffer.clear() - } - - override def startCDATA(): Unit = adapter.captureText() - override def endCDATA(): Unit = captureCData() - } - - override def loadXML(source: InputSource, parser: SAXParser): Elem = { - val newAdapter = adapter - - val xmlReader = parser.getXMLReader - xmlReader.setProperty( - "http://xml.org/sax/properties/lexical-handler", - lexicalHandler(newAdapter)) - - newAdapter.scopeStack push TopScope - parser.parse(source, newAdapter) - newAdapter.scopeStack.pop() - - newAdapter.rootElem.asInstanceOf[Elem] - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/XmlParser.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/XmlParser.scala deleted file mode 100644 index 5b5bf9aa6..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/publish/processor/XmlParser.scala +++ /dev/null @@ -1,232 +0,0 @@ -package org.sunbird.job.content.publish.processor - -import org.apache.commons.lang3.{StringEscapeUtils, StringUtils} -import org.sunbird.job.exception.InvalidInputException - -import scala.collection.mutable.ListBuffer -import scala.xml._ - -object XmlParser { - - val nonPluginElements: List[String] = List("manifest", "controller", "media", "events", "event", "__cdata", "__text") - val START_TAG_OPENING: String = "<" - val END_TAG_OPENING: String = "" - val ATTRIBUTE_KEY_VALUE_SEPARATOR: String = "=" - val BLANK_SPACE: String = " " - val DOUBLE_QUOTE: String = "\"" - - def parse(xml: String): Plugin = { - val xmlObj: Node = XMLLoaderWithCData.loadString(xml) - processDocument(xmlObj) - } - - def processDocument(root: Node): Plugin = { - if (null != root) { - Plugin(getId(root), getData(root), "", getCdata(root), getChildrenPlugin(root), getManifest(root, validateNode = true), getControllers(root \ "controllers"), getEvents(root)) - } else classOf[Plugin].newInstance() - } - - def getAttributesMap(node: Node): Map[String, AnyRef] = { - node.attributes.asAttrMap - } - - def getId(node: Node): String = { - getAttributesMap(node).getOrElse("id", "").asInstanceOf[String] - } - - def getData(node: Node): Map[String, AnyRef] = { - if (null != node) Map("cwp_element_name" -> node.label) ++ getAttributesMap(node) else Map() - } - - //TODO: Review the below code, this is as per the existing logic - def getCdata(node: Node): String = { - if (null != node && node.child.nonEmpty) { - val childNodes = node.child - var cdata = "" - childNodes.toList.filter(childNode => childNode.isInstanceOf[PCData]).map(childNode => { - cdata = childNode.text - }) - cdata - } else "" - } - - def getChildrenPlugin(node: Node): List[Plugin] = { - if (null != node && node.child.nonEmpty) { - val nodeList = node.child - nodeList.toList.filter(childNode => childNode.isInstanceOf[Elem] && !nonPluginElements.contains(childNode.label) && !"event".equalsIgnoreCase(childNode.label)) - .map(chilNode => Plugin(getId(chilNode), getData(chilNode), getInnerText(chilNode), getCdata(chilNode), getChildrenPlugin(chilNode), getManifest(chilNode, validateNode = false), getControllers(chilNode \ "controllers"), getEvents(chilNode))) - } else { - List() - } - } - - def getInnerText(node: Node): String = { - if (null != node && node.isInstanceOf[Elem] && node.child.nonEmpty) { - val childNodes = node.child - val innerTextlist = childNodes.toList.filter(childNode => childNode.isInstanceOf[Text]).map(item => item.text) - if (innerTextlist.nonEmpty) innerTextlist.head else "" - } else "" - } - - def getMedia(node: Node, validateNode: Boolean): Media = { - if (null != node) { - val attributeMap = getAttributesMap(node) - val id: String = attributeMap.getOrElse("id", "").asInstanceOf[String] - val `type`: String = attributeMap.getOrElse("type", "").asInstanceOf[String] - val src: String = attributeMap.getOrElse("src", "").asInstanceOf[String] - if (validateNode) { - if (StringUtils.isBlank(id) && !(StringUtils.isNotBlank(`type`) && (StringUtils.equalsIgnoreCase(`type`, "js") || StringUtils.equalsIgnoreCase(`type`, "css")))) - throw new InvalidInputException("Error! Invalid Media ('id' is required.) in '" + node.buildString(true) + "' ...") - if (StringUtils.isBlank(`type`)) - throw new InvalidInputException("Error! Invalid Media ('type' is required.) in '" + node.buildString(true) + "' ...") - if (StringUtils.isBlank(src)) - throw new InvalidInputException("Error! Invalid Media ('src' is required.) in '" + node.buildString(true) + "' ...") - } - Media(id, getData(node), getInnerText(node), getCdata(node), src, `type`, getChildrenPlugin(node)) - } else classOf[Media].newInstance() - } - - def getManifest(node: Node, validateNode: Boolean): Manifest = { - val childNodes = node.child - var manifestNode: Node = null - childNodes.toList.filter(childNode => StringUtils.equalsIgnoreCase(childNode.label, "manifest")).map(childNode => manifestNode = childNode) - val mediaList = { - if (null != manifestNode && manifestNode.child.nonEmpty) { - manifestNode.child.toList.filter(childNode => childNode.isInstanceOf[Elem] && "media".equalsIgnoreCase(childNode.label)).map(childNode => getMedia(childNode, validateNode)) - } else List() - } - if (null != manifestNode) { - Manifest(getId(manifestNode), getData(manifestNode), getInnerText(manifestNode), getCdata(manifestNode), mediaList) - } else classOf[Manifest].newInstance() - } - - def getControllers(nodeList: Seq[Node]): List[Controller] = { - if (null != nodeList && nodeList.length > 0) { - nodeList.toList.filter(node => node.isInstanceOf[Elem]).map(node => Controller(node.\@("id"), getData(node), getInnerText(node), getCdata(node))) - } else { - List() - } - } - - def getEvents(node: Node): List[Event] = { - var eventsList: ListBuffer[Event] = ListBuffer() - if (null != node && node.child.nonEmpty) { - val childNodes = node.child - childNodes.toList.map(childNode => { - if (childNode.isInstanceOf[Elem] && "events".equalsIgnoreCase(childNode.label)) { - eventsList ++= getEvents(childNode) - } - if (childNode.isInstanceOf[Elem] && "event".equalsIgnoreCase(childNode.label)) { - eventsList += Event(getId(childNode), getData(childNode), getInnerText(childNode), getCdata(childNode), getChildrenPlugin(childNode)) - } - }) - } - eventsList.toList - } - - /** - * serialize - * - */ - def toString(plugin: Plugin): String = { - val strBuilder = StringBuilder.newBuilder - if (null != plugin) { - strBuilder.append(getElementXml(plugin.data)) - .append(getInnerTextXml(plugin.innerText)) - .append(getCdataXml(plugin.cData)) - .append(getContentManifestXml(plugin.manifest)) - .append(getContentControllersXml(plugin.controllers)) - .append(getPluginsXml(plugin.childrenPlugin)) - .append(getEventsXml(plugin.events)) - .append(getEndTag(plugin.data.getOrElse("cwp_element_name", "").asInstanceOf[String])) - } - strBuilder.toString() - } - - def getElementXml(data: Map[String, AnyRef]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != data) { - strBuilder.append(START_TAG_OPENING + data("cwp_element_name")) - data.filterKeys(key => !StringUtils.equalsIgnoreCase("cwp_element_name", key)).map(entry => strBuilder.append(BLANK_SPACE + entry._1 + ATTRIBUTE_KEY_VALUE_SEPARATOR + DOUBLE_QUOTE + entry._2 + DOUBLE_QUOTE)) - strBuilder.append(TAG_CLOSING) - } - strBuilder - } - - def getInnerTextXml(innerText: String): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (StringUtils.isNotBlank(innerText)) strBuilder.append(StringEscapeUtils.escapeXml(innerText)) - strBuilder - } - - def getCdataXml(cData: String): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (StringUtils.isNotBlank(cData)) strBuilder.append("") - strBuilder - } - - def getContentManifestXml(manifest: Manifest): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != manifest && null != manifest.medias && manifest.medias.nonEmpty) { - strBuilder.append(getElementXml(manifest.data)).append(getInnerTextXml(manifest.innerText)) - .append(getCdataXml(manifest.cData)) - .append(getMediaXml(manifest.medias)) - .append(getEndTag(manifest.data.getOrElse("cwp_element_name", "").asInstanceOf[String])) - } - strBuilder - } - - def getPluginsXml(childrenPlugin: List[Plugin]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != childrenPlugin && childrenPlugin.nonEmpty) { - childrenPlugin.map(plugin => strBuilder.append(toString(plugin))) - } - strBuilder - } - - def getContentControllersXml(controllers: List[Controller]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != controllers && controllers.nonEmpty) { - controllers.map(controller => { - strBuilder.append(getElementXml(controller.data)) - .append(getInnerTextXml(controller.innerText)) - .append(getCdataXml(controller.cData)) - .append(getEndTag(controller.data.getOrElse("cwp_element_name", "").asInstanceOf[String])) - }) - } - strBuilder - } - - def getEventsXml(events: List[Event]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != events && events.nonEmpty) { - if (events.size > 1) strBuilder.append(START_TAG_OPENING + "events" + TAG_CLOSING) - events.map(event => strBuilder.append(getElementXml(event.data)).append(getInnerTextXml(event.innerText)).append(getCdataXml(event.cData)).append(getPluginsXml(event.childrenPlugin)).append(getEndTag("event"))) - if (events.size > 1) strBuilder.append(getEndTag("events")) - } - strBuilder - } - - def getEndTag(str: String): String = { - if (StringUtils.isNotBlank(str)) END_TAG_OPENING + str + TAG_CLOSING - else "" - } - - def getMediaXml(medias: List[Media]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != medias && medias.nonEmpty) { - medias.map(media => { - - val updatedMediaData = media.data.map(mediaInfo => { - if(mediaInfo._1.equalsIgnoreCase("src")) - ("src" -> (StringUtils.stripStart(mediaInfo._2.toString, "/"))) - else mediaInfo - }) - - strBuilder.append(getElementXml(updatedMediaData)).append(getInnerTextXml(media.innerText)).append(media.cData).append(getPluginsXml(media.childrenPlugin)).append(getEndTag("media")) - }) - } - strBuilder - } -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/task/ContentPublishConfig.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/task/ContentPublishConfig.scala deleted file mode 100644 index 29eefc0e2..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/task/ContentPublishConfig.scala +++ /dev/null @@ -1,103 +0,0 @@ -package org.sunbird.job.content.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.content.publish.domain.Event - -import java.util -import scala.collection.JavaConverters._ - -class ContentPublishConfig(override val config: Config) extends PublishConfig(config, "content-publish") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val publishMetaTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Job Configuration - val jobEnv: String = config.getString("job.env") - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val postPublishTopic: String = config.getString("kafka.post_publish.topic") - val mvcTopic: String = config.getString("kafka.mvc.topic") - val kafkaErrorTopic: String = config.getString("kafka.error.topic") - val inputConsumerName = "content-publish-consumer" - - // Parallelism - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val eventRouterParallelism: Int = config.getInt("task.router.parallelism") - - // Metric List - val totalEventsCount = "total-events-count" - val skippedEventCount = "skipped-event-count" - val contentPublishEventCount = "content-publish-count" - val contentPublishSuccessEventCount = "content-publish-success-count" - val contentPublishFailedEventCount = "content-publish-failed-count" - val videoStreamingGeneratorEventCount = "video-streaming-event-count" - val collectionPublishEventCount = "collection-publish-count" - val collectionPublishSuccessEventCount = "collection-publish-success-count" - val collectionPublishFailedEventCount = "collection-publish-failed-count" - val collectionPostPublishProcessEventCount = "collection-post-publish-process-count" - val mvProcessorEventCount = "mvc-processor-event-count" - val dialcodeContextUpdaterEventCount = "dialcode-context-updater-event-count" - - // Cassandra Configurations - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - val contentKeyspaceName: String = config.getString("content.keyspace") - val contentTableName: String = config.getString("content.table") - val hierarchyKeyspaceName: String = config.getString("hierarchy.keyspace") - val hierarchyTableName: String = config.getString("hierarchy.table") - - // Neo4J Configurations - val graphRoutePath: String = config.getString("neo4j.routePath") - val graphName: String = config.getString("neo4j.graph") - - // Redis Configurations - val nodeStore: Int = config.getInt("redis.database.contentCache.id") - - // Out Tags - val contentPublishOutTag: OutputTag[Event] = OutputTag[Event]("content-publish") - val collectionPublishOutTag: OutputTag[Event] = OutputTag[Event]("collection-publish") - val generateVideoStreamingOutTag: OutputTag[String] = OutputTag[String]("video-streaming-generator-request") - val failedEventOutTag: OutputTag[String] = OutputTag[String]("failed-event") - val generatePostPublishProcessTag: OutputTag[String] = OutputTag[String]("post-publish-process-request") - val mvcProcessorTag: OutputTag[String] = OutputTag[String]("mvc-processor-request") - val dialcodeContextUpdaterOutTag: OutputTag[String] = OutputTag[String]("dialcode-context-updater-request") - - // Service Urls - val printServiceBaseUrl: String = config.getString("service.print.basePath") - - val definitionBasePath: String = if (config.hasPath("schema.basePath")) config.getString("schema.basePath") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - val schemaSupportVersionMap: Map[String, AnyRef] = if (config.hasPath("schema.supportedVersion")) config.getObject("schema.supportedVersion").unwrapped().asScala.toMap else Map[String, AnyRef]() - - val supportedObjectType: util.List[String] = if (config.hasPath("content.objectType")) config.getStringList("content.objectType") else util.Arrays.asList[String]("Content", "ContentImage") - val supportedMimeType: util.List[String] = if (config.hasPath("content.mimeType")) config.getStringList("content.mimeType") else util.Arrays.asList[String]("application/pdf") - val streamableMimeType: util.List[String] = if (config.hasPath("content.stream.mimeType")) config.getStringList("content.stream.mimeType") else util.Arrays.asList[String]("video/mp4") - val isStreamingEnabled: Boolean = if (config.hasPath("content.stream.enabled")) config.getBoolean("content.stream.enabled") else false - val assetDownloadDuration: String = if (config.hasPath("content.asset_download_duration")) config.getString("content.asset_download_duration") else "60 seconds" - - val isECARExtractionEnabled: Boolean = if (config.hasPath("content.isECARExtractionEnabled")) config.getBoolean("content.isECARExtractionEnabled") else true - val contentFolder: String = if (config.hasPath("cloud_storage.folder.content")) config.getString("cloud_storage.folder.content") else "content" - val artifactFolder: String = if (config.hasPath("cloud_storage.folder.artifact")) config.getString("cloud_storage.folder.artifact") else "artifact" - val retryAssetDownloadsCount: Integer = if (config.hasPath("content.retry_asset_download_count")) config.getInt("content.retry_asset_download_count") else 1 - val artifactSizeForOnline: Double = if (config.hasPath("content.artifact.size.for_online")) config.getDouble("content.artifact.size.for_online") else 209715200 - val bundleLocation: String = if (config.hasPath("content.bundleLocation")) config.getString("content.bundleLocation") else "/data/contentBundle/" - - val extractableMimeTypes = List("application/vnd.ekstep.ecml-archive", "application/vnd.ekstep.html-archive", "application/vnd.ekstep.plugin-archive", "application/vnd.ekstep.h5p-archive") - - val categoryMap: java.util.Map[String, AnyRef] = if (config.hasPath("contentTypeToPrimaryCategory")) config.getAnyRef("contentTypeToPrimaryCategory").asInstanceOf[java.util.Map[String, AnyRef]] else new util.HashMap[String, AnyRef]() - - val esConnectionInfo: String = config.getString("es.basePath") - val compositeSearchIndexName: String = if (config.hasPath("compositesearch.index.name")) config.getString("compositesearch.index.name") else "compositesearch" - val compositeSearchIndexType: String = if (config.hasPath("search.document.type")) config.getString("search.document.type") else "cs" - val nestedFields: util.List[String] = if (config.hasPath("content.nested.fields")) config.getStringList("content.nested.fields") else util.Arrays.asList[String]("badgeAssertions","targets","badgeAssociations") - - val allowedExtensionsWord: util.List[String] = if (config.hasPath("mimetype.allowed_extensions.word")) config.getStringList("mimetype.allowed_extensions.word") else util.Arrays.asList[String]("doc", "docx", "ppt", "pptx", "key", "odp", "pps", "odt", "wpd", "wps", "wks") - val enableDIALContextUpdate: String = if (config.hasPath("enableDIALContextUpdate")) config.getString("enableDIALContextUpdate") else "No" - - val isrRelativePathEnabled: Boolean = if (config.hasPath("cloudstorage.metadata.replace_absolute_path")) config.getBoolean("cloudstorage.metadata.replace_absolute_path") else false -} diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/task/ContentPublishStreamTask.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/task/ContentPublishStreamTask.scala deleted file mode 100644 index 64777a369..000000000 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/task/ContentPublishStreamTask.scala +++ /dev/null @@ -1,62 +0,0 @@ -package org.sunbird.job.content.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.content.function.{CollectionPublishFunction, ContentPublishFunction, PublishEventRouter} -import org.sunbird.job.content.publish.domain.Event -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - -import java.io.File -import java.util - -class ContentPublishStreamTask(config: ContentPublishConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - val processStreamTask = env.addSource(source).name(config.inputConsumerName) - .uid(config.inputConsumerName).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new PublishEventRouter(config)) - .name("publish-event-router").uid("publish-event-router") - .setParallelism(config.eventRouterParallelism) - - val contentPublish = processStreamTask.getSideOutput(config.contentPublishOutTag).process(new ContentPublishFunction(config, httpUtil)) - .name("content-publish-process").uid("content-publish-process").setParallelism(1) - - contentPublish.getSideOutput(config.generateVideoStreamingOutTag).addSink(kafkaConnector.kafkaStringSink(config.postPublishTopic)) - contentPublish.getSideOutput(config.mvcProcessorTag).addSink(kafkaConnector.kafkaStringSink(config.mvcTopic)) - contentPublish.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaErrorTopic)) - - val collectionPublish = processStreamTask.getSideOutput(config.collectionPublishOutTag).process(new CollectionPublishFunction(config, httpUtil)) - .name("collection-publish-process").uid("collection-publish-process").setParallelism(1) - collectionPublish.getSideOutput(config.generatePostPublishProcessTag).addSink(kafkaConnector.kafkaStringSink(config.postPublishTopic)) - collectionPublish.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaErrorTopic)) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object ContentPublishStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("content-publish.conf").withFallback(ConfigFactory.systemEnvironment())) - val publishConfig = new ContentPublishConfig(config) - val kafkaUtil = new FlinkKafkaConnector(publishConfig) - val httpUtil = new HttpUtil - val task = new ContentPublishStreamTask(publishConfig, kafkaUtil, httpUtil) - task.process() - } -} \ No newline at end of file diff --git a/publish-pipeline/content-publish/src/test/resources/logback-test.xml b/publish-pipeline/content-publish/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/publish-pipeline/content-publish/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/publish-pipeline/content-publish/src/test/resources/test.conf b/publish-pipeline/content-publish/src/test/resources/test.conf deleted file mode 100644 index c762241e4..000000000 --- a/publish-pipeline/content-publish/src/test/resources/test.conf +++ /dev/null @@ -1,137 +0,0 @@ -include "base-test.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.publish.job.request" - post_publish.topic = "sunbirddev.content.postpublish.request" - mvc.topic = "sunbirddev.mvc.processor.job.request" - error.topic = "sunbirddev.learning.events.failed" - groupId = "local-content-publish-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - router.parallelism = 1 -} - -redis { - database { - contentCache.id = 0 - } -} - -content { - bundleLocation = "/tmp/contentBundle" - isECARExtractionEnabled = true - retry_asset_download_count = 1 - keyspace = "dev_content_store" - table = "content_data" - tmp_file_location = "/tmp" - objectType = ["Content", "ContentImage","Collection","CollectionImage"] - mimeType = ["application/pdf", "video/avi", "video/mpeg", "video/quicktime", "video/3gpp", "video/mpeg", "video/mp4", "video/ogg", "video/webm", "application/vnd.ekstep.html-archive","application/vnd.ekstep.ecml-archive","application/vnd.ekstep.content-collection" - "application/vnd.ekstep.ecml-archive", - "application/vnd.ekstep.html-archive", - "application/vnd.android.package-archive", - "application/vnd.ekstep.content-archive", - "application/octet-stream", - "application/json", - "application/javascript", - "application/xml", - "text/plain", - "text/html", - "text/javascript", - "text/xml", - "text/css", - "image/jpeg", "image/jpg", "image/png", "image/tiff", "image/bmp", "image/gif", "image/svg+xml", - "image/x-quicktime", - "video/avi", "video/mpeg", "video/quicktime", "video/3gpp", "video/mpeg", "video/mp4", "video/ogg", "video/webm", - "video/msvideo", - "video/x-msvideo", - "video/x-qtc", - "video/x-mpeg", - "audio/mp3", "audio/mp4", "audio/mpeg", "audio/ogg", "audio/webm", "audio/x-wav", "audio/wav", - "audio/mpeg3", - "audio/x-mpeg-3", - "audio/vorbis", - "application/x-font-ttf", - "application/pdf", "application/epub", "application/msword", - "application/vnd.ekstep.h5p-archive", - "application/vnd.ekstep.plugin-archive", - "video/x-youtube", "video/youtube", - "text/x-url"] - asset_download_duration = "60 seconds" - - stream { - enabled = true - mimeType = ["video/mp4", "video/webm"] - } - artifact.size.for_online=209715200 - - downloadFiles { - spine = ["appIcon"] - full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"] - } - - nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions"] -} - -hierarchy { - keyspace = "dev_hierarchy_store" - table = "content_hierarchy" -} - -cloud_storage { - folder { - content = "content" - artifact = "artifact" - } -} - -service { - print.basePath = "http://11.2.6.6/print" -} - - -contentTypeToPrimaryCategory { - ClassroomTeachingVideo: "Explanation Content" - ConceptMap: "Learning Resource" - Course: "Course" - CuriosityQuestionSet: "Practice Question Set" - eTextBook: "eTextbook" - Event: "Event" - EventSet: "Event Set" - ExperientialResource: "Learning Resource" - ExplanationResource: "Explanation Content" - ExplanationVideo: "Explanation Content" - FocusSpot: "Teacher Resource" - LearningOutcomeDefinition: "Teacher Resource" - MarkingSchemeRubric: "Teacher Resource" - PedagogyFlow: "Teacher Resource" - PracticeQuestionSet: "Practice Question Set" - PracticeResource: "Practice Question Set" - SelfAssess: "Course Assessment" - TeachingMethod: "Teacher Resource" - TextBook: "Digital Textbook" - Collection: "Content Playlist" - ExplanationReadingMaterial: "Learning Resource" - LearningActivity: "Learning Resource" - LessonPlan: "Content Playlist" - LessonPlanResource: "Teacher Resource" - PreviousBoardExamPapers: "Learning Resource" - TVLesson: "Explanation Content" - OnboardingResource: "Learning Resource" - ReadingMaterial: "Learning Resource" - Template: "Template" - Asset: "Asset" - Plugin: "Plugin" - LessonPlanUnit: "Lesson Plan Unit" - CourseUnit: "Course Unit" - TextBookUnit: "Textbook Unit" - Asset: "Certificate Template" -} - -enableDIALContextUpdate = "Yes" \ No newline at end of file diff --git a/publish-pipeline/content-publish/src/test/resources/test.cql b/publish-pipeline/content-publish/src/test/resources/test.cql deleted file mode 100644 index 3416fa82a..000000000 --- a/publish-pipeline/content-publish/src/test/resources/test.cql +++ /dev/null @@ -1,58 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS dev_content_store WITH replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS dev_content_store.content_data ( - content_id text, - body blob, - PRIMARY KEY (content_id) -); - -CREATE KEYSPACE IF NOT EXISTS dev_hierarchy_store WITH replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS dev_hierarchy_store.content_hierarchy ( - identifier text, - hierarchy text, - relational_metadata text, - PRIMARY KEY (identifier) -); - -INSERT INTO dev_content_store.content_data(content_id, body) VALUES ( -'do_11321328578759884811663', -textAsBlob('

A person having less ‘haemoglobin’ is suffering from:

Jaundice

Anaemia

Malaria

Chikungunya

\",\"media\":[],\"responseDeclaration\":{\"responseValue\":{\"cardinality\":\"single\",\"type\":\"integer\",\"correct_response\":{\"value\":\"2\"}}},\"options\":[{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Jaundice

\",\"resvalue\":0,\"resindex\":0}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Anaemia

\",\"resvalue\":1,\"resindex\":1}},{\"answer\":true,\"value\":{\"type\":\"text\",\"body\":\"

Malaria

\",\"resvalue\":2,\"resindex\":2}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Chikungunya

\",\"resvalue\":3,\"resindex\":3}}],\"questionCount\":0}]]>') -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_2133950809948078081503', -'{"ownershipType":["createdBy"],"copyright":"firstorg","se_gradeLevelIds":["tn_k-12_5_gradelevel_class1"],"keywords":["pdf"],"subject":["English"],"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","organisation":["First Org","ST MARYS SCHOOL BHATHUBASTI"],"language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142","do_2132926691727114241157"],"objectType":"Content","se_mediums":["English"],"gradeLevel":["Class 1"],"appIcon":"","primaryCategory":"Digital Textbook","children":[{"ownershipType":["createdBy"],"parent":"do_2133950809948078081503","code":"76abafa2a0c2cfef90b52db1ef41fb82","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"This chapter describes about human body","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142","do_2132926691727114241157"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.254+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816031047681516","code":"20cc1f31e62f924c6e47bf04c994376b","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":[],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.252+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816030883841508","code":"7991af5c2e51e4d3d7b83167aaac8829","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"xyz","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":[],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.253+0000","objectType":"Content","primaryCategory":"Textbook Unit","contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030965761510","lastStatusChangedOn":"2021-10-25T06:03:17.253+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797253","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":3,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":0,"name":"5.1.1 Key parts in the head","status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950816030883841508","code":"3bf70f06d3e8dba010d8806fd94259b1","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":[],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.228+0000","objectType":"Content","primaryCategory":"Textbook Unit","contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816028917761504","lastStatusChangedOn":"2021-10-25T06:03:17.228+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797228","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":3,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":0,"name":"5.1.2 Other parts","status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030883841508","lastStatusChangedOn":"2021-10-25T06:03:17.252+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797252","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":2,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":0,"name":"5.1 Parts of Body","status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950816031047681516","code":"40a1ed37e0fad94eca76b2a96fe086ab","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.254+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816031047681514","code":"b186b1bbcc9c58db865f75e34345179e","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.251+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816030801921506","code":"9cf84ff2fb08f9af4c23eb09df9b2520","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.253+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816030965761512","copyright":"Tamil Nadu, P P MAHAPUR","previewUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133916718042972161142/sample-mp4-file.mp4","subject":["Home Science"],"channel":"01269878797503692810","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133916718042972161142/oct20-mp4_1634728008255_do_2133916718042972161142_2.ecar","organisation":["Tamil Nadu","P P MAHAPUR"],"showNotification":true,"language":["English"],"mimeType":"video/mp4","variants":{"full":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133916718042972161142/oct20-mp4_1634728008255_do_2133916718042972161142_2.ecar","size":"10504301"},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133916718042972161142/oct20-mp4_1634728009035_do_2133916718042972161142_2_SPINE.ecar","size":"1465"}},"objectType":"Content","se_mediums":["English","Tamil"],"gradeLevel":["Class 4","Class 2"],"primaryCategory":"Explanation Content","appId":"preprod.diksha.app","contentEncoding":"identity","artifactUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133916718042972161142/sample-mp4-file.mp4","me_totalPlaySessionCount":{"portal":4},"lockKey":"68bcaed4-8eaf-4f01-b5b4-a34a08887de7","sYS_INTERNAL_LAST_UPDATED_ON":"2021-10-25T05:46:53.202+0000","contentType":"Resource","se_gradeLevels":["Class 4","Class 2"],"identifier":"do_2133916718042972161142","lastUpdatedBy":"4cd4c690-eab6-4938-855a-447c7b1b8ea9","audience":["Teacher"],"me_totalTimeSpentInSec":{"portal":325},"visibility":"Default","author":"Adarsh","consumerId":"962257b5-dab6-44a6-b8db-6933b3963dc7","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"08631a74-4b94-4cf7-a818-831135248a4a","version":2,"se_subjects":["Home Science"],"license":"CC BY 4.0","prevState":"Review","size":1.054662E7,"lastPublishedOn":"2021-10-20T11:06:48.254+0000","name":"Oct20 Mp4","attributions":["Tester"],"status":"Live","code":"9194086b-4c4c-45d6-92c9-98828a4aae74","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Review","streamingUrl":"https://ntppreprodmedia-inct.streaming.media.azure.net/29886fa7-95d5-4cc3-bf5f-be9c66fb78cb/sample-mp4-file.ism/manifest(format=m3u8-aapl-v3)","medium":["English","Tamil"],"idealScreenSize":"normal","createdOn":"2021-10-20T10:26:02.051+0000","se_boards":["State (Tamil Nadu)"],"copyrightYear":2021,"contentDisposition":"inline","additionalCategories":["Textbook"],"licenseterms":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.","lastUpdatedOn":"2021-10-20T11:36:17.429+0000","dialcodeRequired":"No","lastStatusChangedOn":"2021-10-20T11:06:49.079+0000","createdFor":["01269878797503692810","0133480174794670081832"],"creator":"ContentcreatorTN","os":["All"],"se_FWIds":["tn_k-12_5"],"pkgVersion":2,"versionKey":"1634729777429","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":5,"lastSubmittedOn":"2021-10-20T10:30:29.896+0000","createdBy":"4cd4c690-eab6-4938-855a-447c7b1b8ea9","compatibilityLevel":1,"board":"State (Tamil Nadu)","resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030965761512","lastStatusChangedOn":"2021-10-25T06:03:17.253+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797253","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":4,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":1,"name":"dsffgdg","status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030801921506","lastStatusChangedOn":"2021-10-25T06:03:17.251+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797251","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":3,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":1,"name":"5.2.1 Respiratory System","status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816031047681514","lastStatusChangedOn":"2021-10-25T06:03:17.254+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797254","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":2,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":1,"name":"5.2 Organ Systems","status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950816031047681516","unitIdentifiers":["do_21329099910853427215263"],"copyright":"Kendriya_Vidyalaya,2020","organisationId":"32625405-bc59-4622-89bb-d06934d690ef","previewUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","keywords":["pdf"],"subject":["Hindi"],"channel":"013085024460783616158023","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/pdf_1.pdf_1634813465403_do_2132926691727114241157_2.ecar","language":["English"],"source":"https://dock.preprod.ntp.net.in/api/content/v1/read/do_21329119747088384015821","mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/pdf_1.pdf_1634813465403_do_2132926691727114241157_2.ecar","size":"259984"},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/pdf_1.pdf_1634813465462_do_2132926691727114241157_2_SPINE.ecar","size":"5422"}},"objectType":"Content","se_mediums":["English"],"gradeLevel":["Class 10"],"appIcon":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/content.thumb.jpg","primaryCategory":"Teacher Resource","appId":"preprod.diksha.portal","contentEncoding":"identity","artifactUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","me_totalPlaySessionCount":{"portal":3},"sYS_INTERNAL_LAST_UPDATED_ON":"2021-10-25T05:46:52.419+0000","contentType":"MarkingSchemeRubric","se_gradeLevels":["Class 10"],"identifier":"do_2132926691727114241157","audience":["Administrator"],"me_totalTimeSpentInSec":{"portal":221},"visibility":"Default","consumerId":"962257b5-dab6-44a6-b8db-6933b3963dc7","discussionForum":{"enabled":"Yes"},"index":3,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"fcc5bf38-810b-4543-aa00-e96a1d59573a","version":2,"pragma":["external"],"se_subjects":["Hindi"],"license":"CC BY 4.0","prevState":"Review","size":256184.0,"lastPublishedOn":"2021-10-21T10:51:03.459+0000","name":"PDF_1.pdf","attributions":["kanmani"],"status":"Live","code":"c644316c-0a39-7c95-9ff4-c4f027d4c024","credentials":{"enabled":"No"},"prevStatus":"Processing","origin":"do_21329119747088384015821","description":"pdf","streamingUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","medium":["English"],"posterImage":"https://drive.google.com/uc?export=download&id=1z2kHz_wfjcOcDKfenkxWwqIlwtro6uv0","idealScreenSize":"normal","createdOn":"2021-06-02T13:24:39.874+0000","se_boards":["CBSE"],"processId":"2bb3d2ed-a0b5-4502-831b-e7e0e3423e4c","contentDisposition":"inline","lastUpdatedOn":"2021-10-21T10:51:05.506+0000","originData":{"identifier":"do_21329119747088384015821","repository":"https://dock.preprod.ntp.net.in/api/content/v1/read/do_21329119747088384015821"},"collectionId":"do_21329099910788710415244","dialcodeRequired":"No","lastStatusChangedOn":"2021-10-21T10:51:05.506+0000","creator":"కాయల్ నధి","os":["All"],"cloudStorageKey":"content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":2,"versionKey":"1622641657267","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"s3Key":"ecar_files/do_2132926691727114241157/pdf_1.pdf_1622641665371_do_2132926691727114241157_1.0.ecar","lastSubmittedOn":"2021-06-02T13:24:41.290+0000","createdBy":"4096d5cf-af62-48d6-88a3-e007219b8e59","compatibilityLevel":4,"board":"CBSE","programId":"013c04f0-c1c9-11eb-be1d-e5048cd700e9"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T10:29:28.137+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816031047681516","lastStatusChangedOn":"2021-10-25T06:03:17.254+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797254","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":1,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":2,"name":"5. Human Body","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950809948078081503","code":"c1bed6fb-7e28-c033-d87a-365511c10614","credentials":{"enabled":"No"},"channel":"0126825293972439041","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-10-25T10:29:28.133+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133952124365455361558","code":"2ebb6e7c-e01b-5be1-7c33-d00334fcdd46","credentials":{"enabled":"No"},"channel":"0126825293972439041","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-10-25T10:29:28.137+0000","objectType":"Content","primaryCategory":"Textbook Unit","contentDisposition":"inline","lastUpdatedOn":"2021-10-25T10:29:28.137+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133952124365783041560","lastStatusChangedOn":"2021-10-25T10:29:28.137+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1635157768137","license":"CC BY 4.0","idealScreenDensity":"hdpi","depth":2,"compatibilityLevel":1,"name":"6.1 Medula Oblongata","attributions":[],"status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T10:29:28.133+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133952124365455361558","lastStatusChangedOn":"2021-10-25T10:29:28.133+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1635157768133","license":"CC BY 4.0","idealScreenDensity":"hdpi","depth":1,"compatibilityLevel":1,"name":"6. Inhuman Mind","attributions":[],"status":"Draft"}],"contentEncoding":"gzip","lockKey":"02bcb42d-d22d-4b08-a872-b8a22372d2a0","generateDIALCodes":"Yes","totalCompressedSize":1.0802804E7,"mimeTypesCount":"{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":7,\"video/mp4\":1}","sYS_INTERNAL_LAST_UPDATED_ON":"2021-10-25T06:05:50.280+0000","contentType":"TextBook","se_gradeLevels":["Class 1"],"trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_2133950809948078081503","audience":["Student"],"se_boardIds":["tn_k-12_5_board_statetamilnadu"],"subjectIds":["tn_k-12_5_subject_english"],"toc_url":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133950809948078081503/artifact/do_2133950809948078081503_toc.json","visibility":"Default","contentTypesCount":"{\"TextBookUnit\":7,\"MarkingSchemeRubric\":1,\"Resource\":1}","author":"Content 1 Creator1","consumerId":"962257b5-dab6-44a6-b8db-6933b3963dc7","childNodes":["do_2133950816030965761510","do_2133950816030883841508","do_2133950816031047681516","do_2133950816028917761504","do_2133916718042972161142","do_2133950816030965761512","do_2133950816030801921506","do_2133950816031047681514","do_2132926691727114241157","do_2133952124365783041560","do_2133952124365455361558"],"discussionForum":{"enabled":"No"},"mediaType":"content","osId":"org.ekstep.quiz.app","lastPublishedBy":"30b2571f-08f9-49ce-b97a-c643df0c82f7","languageCode":["en"],"version":2,"se_subjects":["English"],"license":"CC BY 4.0","prevState":"Review","size":12426.0,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","name":"Test collection publishing","mediumIds":["tn_k-12_5_medium_english"],"status":"Draft","code":"org.sunbird.nV84DB","credentials":{"enabled":"No"},"prevStatus":"Processing","description":"Enter description for TextBook","medium":["English"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:02:03.002+0000","se_boards":["State (Tamil Nadu)"],"se_mediumIds":["tn_k-12_5_medium_english"],"copyrightYear":2021,"contentDisposition":"inline","additionalCategories":["Textbook"],"lastUpdatedOn":"2021-10-25T06:05:49.619+0000","dialcodeRequired":"No","lastStatusChangedOn":"2021-10-25T10:29:28.562+0000","createdFor":["0126825293972439041","01323953134532198492165"],"creator":"Content 1 Creator1","os":["All"],"se_subjectIds":["tn_k-12_5_subject_english"],"se_FWIds":["tn_k-12_5"],"pkgVersion":1.0,"versionKey":"1635157768888","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":0,"s3Key":"ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","boardIds":["tn_k-12_5_board_statetamilnadu"],"lastSubmittedOn":"2021-10-25T06:04:24.346+0000","createdBy":"0b96635f-fe2b-4ab0-a511-05cfce8faa3f","compatibilityLevel":1,"leafNodesCount":2,"userConsent":"Yes","gradeLevelIds":["tn_k-12_5_gradelevel_class1"],"board":"State (Tamil Nadu)","resourceType":"Book"}', -'{"do_2133950809948078081503":{"name":"Collection Publish T21","children":["do_2133916718042972161142"],"root":true},"do_2133916718042972161142":{"name":"Collection Parent","children":["do_11340096165525094411"],"root":false,"relationalMetadata":{"do_11340096165525094411":{"name":"Test Name RM L1 - R1","keywords":["Overwriting content KW1"]}}},"do_11340096165525094411":{"name":"PDF Content","children":[],"root":false}' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_123', -'{"identifier":"do_123","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}', -'{"do_123":{"name":"Collection Publish T21","children":["do_11340511137112064018","do_11340511137080934412"],"root":true},"do_11340511137112064018":{"name":"Collection Parent","children":["do_11340096165525094411"],"root":false,"relationalMetadata":{"do_11340096165525094411":{"name":"Test Name RM L1 - R1","keywords":["Overwriting content KW1"]}}},"do_11340511137080934412":{"name":"Collection Parent","children":["do_11340096165525094411"],"root":false,"relationalMetadata":{"do_11340096165525094411":{"name":"Test Name RM L1 - R1","keywords":["Overwriting content KW1"]}}},"do_11340096165525094411":{"name":"PDF Content","children":[],"root":false},"do_113405111371145216110":{"name":"test","children":[], "root":false}}' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy) VALUES ( -'do_21354027142511820812318.img', -'{ "ownershipType": [ "createdBy" ], "copyright": "tn", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "keywords": [ "வாக்கியங்கள்", "വളരെ", "बतख़!", "Drop", "മലയാളം" ], "subject": [ "English" ], "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "organisation": [ "Tamil Nadu" ], "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "size": "16926" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar", "size": "8313" } }, "leafNodes": [ "do_21353901951355289618149" ], "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 1" ], "appIcon": "", "primaryCategory": "Digital Textbook", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "80986053-eb44-0c35-1c3b-c84806d4775a", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "size": "16926" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar", "size": "8313" } }, "leafNodes": [ "do_21353901951355289618149" ], "idealScreenSize": "normal", "createdOn": "2022-05-18T10:50:55.849+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354031968951500812320", "copyright": "tn", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "keywords": [ "Drop" ], "subject": [ "Accountancy" ], "targetMediumIds": [ "tn_k-12_5_medium_english" ], "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar", "organisation": [ "Tamil Nadu", "MPPS GYARAGONDANAHALLI" ], "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar", "size": "13797" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862667_do_21354022673739776012312_1_ONLINE.ecar", "size": "5181" } }, "leafNodes": [ "do_21353901951355289618149" ], "targetGradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "objectType": "Content", "se_mediums": [ "English" ], "primaryCategory": "Course", "appId": "staging.sunbird.portal", "contentEncoding": "gzip", "lockKey": "3efeaad1-490b-4044-8185-52a421057844", "generateDIALCodes": "No", "totalCompressedSize": 209850, "mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":1}", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T07:45:16.264+0000", "contentType": "Course", "se_gradeLevels": [ "Class 1" ], "trackable": { "enabled": "Yes", "autoBatch": "No" }, "identifier": "do_21354022673739776012312", "audience": [ "Student" ], "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "subjectIds": [ "tn_k-12_5_subject_accountancy" ], "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "visibility": "Default", "contentTypesCount": "{\"eTextBook\":1,\"CourseUnit\":1}", "author": "Guest name changed", "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "childNodes": [ "do_21353901951355289618149", "do_21354022740915814412313" ], "discussionForum": { "enabled": "Yes" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "version": 2, "se_subjects": [ "Accountancy" ], "license": "CC BY 4.0", "prevState": "Review", "size": 13797, "lastPublishedOn": "2022-05-18T07:44:22.530+0000", "name": "newcert", "targetBoardIds": [ "tn_k-12_5_board_statetamilnadu" ], "status": "Live", "code": "org.sunbird.11SAbB", "credentials": { "enabled": "Yes" }, "prevStatus": "Processing", "description": "Enter description for Course", "idealScreenSize": "normal", "createdOn": "2022-05-18T07:41:49.160+0000", "reservedDialcodes": { "U5X4C5": 0 }, "batches": [ { "createdFor": [ "01269878797503692810", "01275630040485068839017" ], "endDate": "2022-05-20", "name": "nc", "batchId": "013540229187338240118", "enrollmentType": "open", "enrollmentEndDate": "2022-05-19", "startDate": "2022-05-18", "status": 1 } ], "se_boards": [ "State (Tamil Nadu)" ], "targetSubjectIds": [ "tn_k-12_5_subject_english" ], "se_mediumIds": [ "tn_k-12_5_medium_english" ], "copyrightYear": 2022, "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "lastUpdatedOn": "2022-05-18T07:44:22.749+0000", "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-18T07:44:22.749+0000", "createdFor": [ "01269878797503692810" ], "creator": "Guest name changed", "os": [ "All" ], "se_subjectIds": [ "tn_k-12_5_subject_accountancy", "tn_k-12_5_subject_english" ], "se_FWIds": [ "tn_k-12_5" ], "targetFWIds": [ "tn_k-12_5" ], "pkgVersion": 1, "versionKey": "1652859827944", "idealScreenDensity": "hdpi", "framework": "tn_k-12_5", "dialcodes": [ "U5X4C5" ], "depth": 0, "s3Key": "content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "lastSubmittedOn": "2022-05-18T07:43:47.614+0000", "createdBy": "fca2925f-1eee-4654-9177-fece3fd6afc9", "compatibilityLevel": 4, "leafNodesCount": 1, "userConsent": "Yes", "resourceType": "Course" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354031968951500812320", "lastStatusChangedOn": "2022-05-18T10:50:55.849+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "pkgVersion": 1, "versionKey": "1652871055849", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "R8C7G2" ], "depth": 1, "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "compatibilityLevel": 1, "leafNodesCount": 1, "name": "Ch1 ", "attributions": [], "status": "Draft" }, { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "22c05b75-ae2a-6280-c192-2e2a4157d141", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "size": "16926" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar", "size": "8313" } }, "leafNodes": [], "idealScreenSize": "normal", "createdOn": "2022-05-18T10:50:55.854+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354031968955596812322", "copyright": "tn", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "keywords": [ "बतख़!", "വളരെ", "வாக்கியங்கள்", "ಕನ್ನಡ" ], "subject": [ "English" ], "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "organisation": [ "Tamil Nadu", "MPPS HANUMANNAHALLI" ], "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018" } }, "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 1" ], "appIcon": "", "primaryCategory": "Digital Textbook", "contentEncoding": "gzip", "lockKey": "35daaa23-b864-40b6-8709-fedc31de95c0", "generateDIALCodes": "Yes", "totalCompressedSize": 0, "mimeTypesCount": "{\"application/vnd.ekstep.content-collection\":3}", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-17T06:12:28.810+0000", "contentType": "TextBook", "se_gradeLevels": [ "Class 1" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21353947184001843212121", "audience": [ "Student" ], "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "subjectIds": [ "tn_k-12_5_subject_english" ], "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/artifact/do_21353947184001843212121_toc.json", "visibility": "Default", "contentTypesCount": "{\"TextBookUnit\":3}", "author": "Guest name changed", "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "childNodes": [ "do_21353947322769408012133", "do_21353947287477452812131", "do_21353947357250355212135", "do_21353947535707340812137" ], "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "version": 2, "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "size": 4019, "lastPublishedOn": "2022-05-17T06:12:27.957+0000", "name": "ಕನ್ನಡ", "mediumIds": [ "tn_k-12_5_medium_english" ], "status": "Draft", "code": "org.sunbird.zRvSTj", "credentials": { "enabled": "No" }, "prevStatus": "Processing", "description": "ಕನ್ನಡ", "medium": [ "English" ], "idealScreenSize": "normal", "createdOn": "2022-05-17T06:05:58.602+0000", "se_boards": [ "State (Tamil Nadu)" ], "se_mediumIds": [ "tn_k-12_5_medium_english" ], "copyrightYear": 2020, "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "lastUpdatedOn": "2022-05-17T06:13:08.755+0000", "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-17T06:13:08.396+0000", "createdFor": [ "01269878797503692810" ], "creator": "Guest name changed", "os": [ "All" ], "se_subjectIds": [ "tn_k-12_5_subject_english" ], "se_FWIds": [ "tn_k-12_5" ], "pkgVersion": 1, "versionKey": "1652767988755", "idealScreenDensity": "hdpi", "framework": "tn_k-12_5", "depth": 0, "s3Key": "content/do_21353947184001843212121/artifact/do_21353947184001843212121_toc.json", "boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "lastSubmittedOn": "2022-05-17T06:12:01.588+0000", "createdBy": "fca2925f-1eee-4654-9177-fece3fd6afc9", "compatibilityLevel": 1, "leafNodesCount": 0, "userConsent": "Yes", "gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "board": "State (Tamil Nadu)", "resourceType": "Book" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354031968955596812322", "lastStatusChangedOn": "2022-05-18T10:50:55.854+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 2, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "pkgVersion": 1, "versionKey": "1652871055854", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "L5L9A2" ], "depth": 1, "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "compatibilityLevel": 1, "leafNodesCount": 0, "name": "Ch2", "attributions": [], "status": "Draft" }, { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "6b8ccba1-9aa8-19d7-fe3b-35ca4be5d6cf", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "idealScreenSize": "normal", "createdOn": "2022-05-18T11:01:38.790+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354032495648768012324", "copyright": "2013", "se_gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "previewUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/cbse-copy-88.pdf", "keywords": [ "Drop" ], "subject": [ "English" ], "channel": "01345815127107174426", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/aparna-86_1652722733905_do_21353901750772531217809_1.ecar", "language": [ "English" ], "source": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901750772531217809", "mimeType": "application/pdf", "variants": { "full": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/aparna-86_1652722733905_do_21353901750772531217809_1.ecar", "size": "193281" }, "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/aparna-86_1652722733964_do_21353901750772531217809_1_SPINE.ecar", "size": "10073" } }, "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 3" ], "appIcon": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/rose.thumb.jpg", "primaryCategory": "eTextbook", "contentEncoding": "identity", "artifactUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/cbse-copy-88.pdf", "contentType": "eTextBook", "se_gradeLevels": [ "Class 3" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21353901750772531217809", "audience": [ "Student" ], "se_boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "visibility": "Default", "author": "Aparna", "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "56c84dee-7149-47af-902d-0138e080cec0", "version": 2, "pragma": [ "external" ], "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "size": 209850.0, "lastPublishedOn": "2022-05-16T17:38:53.877+0000", "name": "Aparna 86", "topic": [ "Water" ], "mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "attributions": [ "Nadiya Anusha" ], "status": "Live", "topicsIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "code": "3d09fc2e-5705-8f9f-2e1d-4c9c8fd19de2", "interceptionPoints": {}, "credentials": { "enabled": "No" }, "prevStatus": "Processing", "origin": "do_21353901750772531217809", "description": "about water", "streamingUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/cbse-copy-88.pdf", "medium": [ "English" ], "posterImage": "https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_21353901750772531217809/artifact/rose.jpg", "idealScreenSize": "normal", "createdOn": "2022-05-16T17:38:50.225+0000", "se_boards": [ "CBSE" ], "se_mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "processId": "497335f0-51d2-4cec-a3ba-6eb81309aa87", "contentDisposition": "inline", "lastUpdatedOn": "2022-05-16T17:38:54.005+0000", "originData": { "identifier": "do_21353901750772531217809", "repository": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901750772531217809" }, "se_topicIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-16T17:38:54.005+0000", "createdFor": [ "01345815127107174426" ], "creator": "Aparna", "os": [ "All" ], "se_subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "se_FWIds": [ "ekstep_ncert_k-12" ], "pkgVersion": 1, "versionKey": "1652722732649", "idealScreenDensity": "hdpi", "framework": "ekstep_ncert_k-12", "depth": 2, "boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "lastSubmittedOn": "2022-05-16T17:38:52.212+0000", "createdBy": "c16ccf69-bd42-4dd3-a1e6-435322e78c97", "se_topics": [ "Water" ], "compatibilityLevel": 4, "gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "board": "CBSE", "programId": "9ae297d0-d51a-11ec-907e-e7dd0cd587df" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T11:01:38.790+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354032495648768012324", "lastStatusChangedOn": "2022-05-18T11:01:38.790+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 3, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "versionKey": "1652871698790", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "D1D5H9" ], "depth": 1, "compatibilityLevel": 1, "name": "Ch3", "attributions": [], "status": "Draft" }, { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "bf120af0-d50b-b700-baca-1f30214bdb27", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "idealScreenSize": "normal", "createdOn": "2022-05-18T11:01:38.794+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354032495652044812326", "copyright": "2013", "se_gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "previewUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "keywords": [ "Drop" ], "subject": [ "English" ], "channel": "01345815127107174426", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar", "language": [ "English" ], "source": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149", "mimeType": "application/pdf", "variants": { "full": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar", "size": "193301" }, "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745316_do_21353901951355289618149_1_SPINE.ecar", "size": "10093" } }, "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 3" ], "appIcon": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/rose.thumb.jpg", "primaryCategory": "eTextbook", "contentEncoding": "identity", "artifactUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "contentType": "eTextBook", "se_gradeLevels": [ "Class 3" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21353901951355289618149", "audience": [ "Student" ], "se_boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "visibility": "Default", "author": "Aparna", "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "56c84dee-7149-47af-902d-0138e080cec0", "version": 2, "pragma": [ "external" ], "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "size": 209850.0, "lastPublishedOn": "2022-05-16T17:39:05.200+0000", "name": "Aparna 87", "topic": [ "Water" ], "mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "attributions": [ "Nadiya Anusha" ], "status": "Live", "topicsIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "code": "2bf23df3-3fa3-c5ad-e1d0-6784a96cb2b8", "interceptionPoints": {}, "credentials": { "enabled": "No" }, "prevStatus": "Processing", "origin": "do_21353901951355289618149", "description": "about water", "streamingUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "medium": [ "English" ], "posterImage": "https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_21353901951355289618149/artifact/rose.jpg", "idealScreenSize": "normal", "createdOn": "2022-05-16T17:39:01.560+0000", "se_boards": [ "CBSE" ], "se_mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "processId": "497335f0-51d2-4cec-a3ba-6eb81309aa87", "contentDisposition": "inline", "lastUpdatedOn": "2022-05-16T17:39:05.366+0000", "originData": { "identifier": "do_21353901951355289618149", "repository": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149" }, "se_topicIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-16T17:39:05.366+0000", "createdFor": [ "01345815127107174426" ], "creator": "27aprilindividual@yopmail.com", "os": [ "All" ], "se_subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "se_FWIds": [ "ekstep_ncert_k-12" ], "pkgVersion": 1, "versionKey": "1652722743949", "idealScreenDensity": "hdpi", "framework": "ekstep_ncert_k-12", "depth": 2, "boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "lastSubmittedOn": "2022-05-16T17:39:03.621+0000", "createdBy": "c6698266-f5d8-4e72-a831-5da9b3205d16", "se_topics": [ "Water" ], "compatibilityLevel": 4, "gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "board": "CBSE", "programId": "9ae297d0-d51a-11ec-907e-e7dd0cd587df" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T11:01:38.794+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354032495652044812326", "lastStatusChangedOn": "2022-05-18T11:01:38.794+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 4, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "versionKey": "1652871698794", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "F6I6S6" ], "depth": 1, "compatibilityLevel": 1, "name": "Ch4", "attributions": [], "status": "Draft" } ], "appId": "staging.sunbird.portal", "contentEncoding": "gzip", "lockKey": "3e222f39-d917-4582-96ee-458f1ab80f30", "generateDIALCodes": "Yes", "totalCompressedSize": 209850, "mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":8,\"\":1}", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T11:02:51.732+0000", "contentType": "TextBook", "se_gradeLevels": [ "Class 1" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21354027142511820812318", "audience": [ "Student" ], "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "subjectIds": [ "tn_k-12_5_subject_english" ], "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json", "visibility": "Default", "contentTypesCount": "{\"TextBookUnit\":6,\"eTextBook\":1,\"Course\":1,\"CourseUnit\":1,\"\":1}", "author": "ContentcreatorTN", "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "childNodes": [ "do_21354022673739776012312", "do_21354031968951500812320", "do_21353947184001843212121", "do_21354031968955596812322", "do_21353901750772531217809", "do_21354032495648768012324", "do_21353901951355289618149", "do_21354032495652044812326" ], "discussionForum": { "enabled": "No" }, "mediaType": "content", "osId": "org.ekstep.quiz.app", "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "languageCode": [ "en" ], "version": 2, "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "qrCodeProcessId": "40c3e296-ccdc-487d-8cda-9f0a88c071d2", "size": 16926, "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "name": "DialCodeHierarchy", "mediumIds": [ "tn_k-12_5_medium_english" ], "status": "Draft", "code": "org.sunbird.D3eL5z", "credentials": { "enabled": "No" }, "prevStatus": "Processing", "description": "Enter description for TextBook", "medium": [ "English" ], "idealScreenSize": "normal", "createdOn": "2022-05-18T09:12:44.203+0000", "reservedDialcodes": { "A3S8L6": 2, "R8C7G2": 1, "D1D5H9": 4, "V2A8N1": 0, "L5L9A2": 5, "F6I6S6": 3 }, "se_boards": [ "State (Tamil Nadu)" ], "se_mediumIds": [ "tn_k-12_5_medium_english" ], "copyrightYear": 2022, "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "lastUpdatedOn": "2022-05-18T10:53:41.611+0000", "dialcodeRequired": "Yes", "lastStatusChangedOn": "2022-05-18T11:01:39.892+0000", "createdFor": [ "01269878797503692810" ], "creator": "ContentcreatorTN", "os": [ "All" ], "se_subjectIds": [ "tn_k-12_5_subject_english" ], "se_FWIds": [ "tn_k-12_5" ], "pkgVersion": 1, "versionKey": "1652871771396", "idealScreenDensity": "hdpi", "framework": "tn_k-12_5", "dialcodes": [ "V2A8N1" ], "depth": 0, "s3Key": "content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json", "boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "lastSubmittedOn": "2022-05-18T10:53:03.017+0000", "createdBy": "4cd4c690-eab6-4938-855a-447c7b1b8ea9", "compatibilityLevel": 1, "leafNodesCount": 1, "userConsent": "Yes", "gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "board": "State (Tamil Nadu)", "resourceType": "Book" }' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy) VALUES ( -'do_21354027142511820812318', -'{"ownershipType": [ "createdBy"],"publish_type": "public","copyright": "tn","se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1"],"keywords": [ "வாக்கியங்கள்", "വളരെ", "बतख़!", "Drop", "മലയാളം"],"subject": [ "English"],"channel": "01269878797503692810","downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar","organisation": [ "Tamil Nadu"],"language": [ "English"],"mimeType": "application/vnd.ekstep.content-collection","variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar\",\"size\":\"16926\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar\",\"size\":\"8313\"}}","body": null,"leafNodes": [ "do_21353901951355289618149"],"objectType": "Content","se_mediums": [ "English"],"gradeLevel": [ "Class 1"],"appIcon": "","children": [ { "lastStatusChangedOn": "2022-05-18T10:50:55.849+0000", "parent": "do_21354027142511820812318", "children": [{ "copyright": "tn", "lastStatusChangedOn": "2022-05-18T07:41:49.160+0000", "publish_type": "public", "parent": "do_21354031968951500812320", "author": "Guest name changed", "se_mediumIds": [ "tn_k-12_5_medium_english" ], "organisation": [ "Tamil Nadu", "MPPS GYARAGONDANAHALLI" ], "children": [ {"lastStatusChangedOn": "2022-05-18T07:43:11.157+0000","parent": "do_21354022673739776012312","children": [ { "copyright": "2013", "lastStatusChangedOn": "2022-05-16T17:39:05.366+0000", "originData": "{\"identifier\":\"do_21353901951355289618149\",\"repository\":\"https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149\"}", "parent": "do_21354022740915814412313", "author": "Aparna", "se_mediumIds": ["ekstep_ncert_k-12_medium_english" ], "mediaType": "content", "name": "Aparna 87", "se_topicIds": ["ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "discussionForum": "{\"enabled\":\"No\"}", "createdOn": "2022-05-16T17:39:01.560+0000", "createdFor": ["01345815127107174426" ], "channel": "01345815127107174426", "source": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149", "lastUpdatedOn": "2022-05-16T17:39:05.366+0000", "subject": ["English" ], "size": 209850.0, "se_topics": ["Water" ], "streamingUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "identifier": "do_21353901951355289618149", "se_gradeLevelIds": ["ekstep_ncert_k-12_gradelevel_class3" ], "description": "about water", "gradeLevel": ["Class 3" ], "ownershipType": ["createdBy" ], "mediumIds": ["ekstep_ncert_k-12_medium_english" ], "compatibilityLevel": 4, "audience": ["Student" ], "trackable": "{\"enabled\":\"No\",\"autoBatch\":\"No\"}", "se_boards": ["CBSE" ], "os": ["All" ], "primaryCategory": "eTextbook", "appIcon": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/rose.thumb.jpg", "languageCode": ["en" ], "se_mediums": ["English" ], "se_subjectIds": ["ekstep_ncert_k-12_subject_english" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar", "se_subjects": ["English" ], "medium": ["English" ], "attributions": ["Nadiya Anusha" ], "framework": "ekstep_ncert_k-12", "posterImage": "https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_21353901951355289618149/artifact/rose.jpg", "creator": "27aprilindividual@yopmail.com", "versionKey": "1652722743949", "mimeType": "application/pdf", "code": "2bf23df3-3fa3-c5ad-e1d0-6784a96cb2b8", "se_boardIds": ["ekstep_ncert_k-12_board_cbse" ], "license": "CC BY 4.0", "version": 2, "prevStatus": "Processing", "contentType": "eTextBook", "prevState": "Review", "language": ["English" ], "board": "CBSE", "lastPublishedOn": "2022-05-16T17:39:05.200+0000", "objectType": "Content", "origin": "do_21353901951355289618149", "subjectIds": ["ekstep_ncert_k-12_subject_english" ], "status": "Live", "pragma": ["external" ], "programId": "9ae297d0-d51a-11ec-907e-e7dd0cd587df", "createdBy": "c6698266-f5d8-4e72-a831-5da9b3205d16", "dialcodeRequired": "No", "lastSubmittedOn": "2022-05-16T17:39:03.621+0000", "interceptionPoints": "{}", "keywords": ["Drop" ], "idealScreenSize": "normal", "contentEncoding": "identity", "depth": 2, "lastPublishedBy": "56c84dee-7149-47af-902d-0138e080cec0", "topic": ["Water" ], "topicsIds": ["ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "se_gradeLevels": ["Class 3" ], "osId": "org.ekstep.quiz.app", "se_FWIds": ["ekstep_ncert_k-12" ], "contentDisposition": "inline", "previewUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "boardIds": ["ekstep_ncert_k-12_board_cbse" ], "artifactUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "visibility": "Default", "credentials": "{\"enabled\":\"No\"}", "processId": "497335f0-51d2-4cec-a3ba-6eb81309aa87", "variants": "{\"full\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar\",\"size\":\"193301\"},\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745316_do_21353901951355289618149_1_SPINE.ecar\",\"size\":\"10093\"}}", "gradeLevelIds": ["ekstep_ncert_k-12_gradelevel_class3" ], "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi" }],"mediaType": "content","name": "Course Unit1","discussionForum": { "enabled": "No"},"createdOn": "2022-05-18T07:43:11.157+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-18T07:43:47.944+0000","identifier": "do_21354022740915814412313","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"trackable": { "enabled": "No", "autoBatch": "No"},"os": [ "All"],"primaryCategory": "Course Unit","languageCode": [ "en"],"downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar","attributions": [],"versionKey": "1652859791157","mimeType": "application/vnd.ekstep.content-collection","code": "8335cfaa-74bc-171d-5c8d-4e895f8fb701","license": "CC BY 4.0","leafNodes": [ "do_21353901951355289618149"],"version": 2,"contentType": "CourseUnit","language": [ "English"],"lastPublishedOn": "2022-05-18T07:44:22.530+0000","objectType": "Content","status": "Live","dialcodeRequired": "No","idealScreenSize": "normal","contentEncoding": "gzip","leafNodesCount": 1,"depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar\",\"size\":\"13797\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862667_do_21354022673739776012312_1_ONLINE.ecar\",\"size\":\"5181\"}}","index": 1,"pkgVersion": 1,"idealScreenDensity": "hdpi" } ], "body": null, "mediaType": "content", "name": "newcert", "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "batches": [ {"name": "nc","createdFor": [ "01269878797503692810", "01275630040485068839017"],"enrollmentType": "open","endDate": "2022-05-20","enrollmentEndDate": "2022-05-19","status": 1,"batchId": "013540229187338240118","startDate": "2022-05-18" } ], "discussionForum": { "enabled": "Yes" }, "createdOn": "2022-05-18T07:41:49.160+0000", "createdFor": [ "01269878797503692810" ], "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-18T07:43:47.944+0000", "subject": [ "Accountancy" ], "size": 13797, "publishError": null, "targetMediumIds": [ "tn_k-12_5_medium_english" ], "identifier": "do_21354022673739776012312", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "description": "Enter description for Course", "resourceType": "Course", "ownershipType": [ "createdBy" ], "compatibilityLevel": 4, "targetBoardIds": [ "tn_k-12_5_board_statetamilnadu" ], "audience": [ "Student" ], "trackable": { "enabled": "Yes", "autoBatch": "No" }, "se_boards": [ "State (Tamil Nadu)" ], "os": [ "All" ], "primaryCategory": "Course", "se_mediums": [ "English" ], "se_subjectIds": [ "tn_k-12_5_subject_accountancy", "tn_k-12_5_subject_english" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar", "se_subjects": [ "Accountancy" ], "lockKey": "3efeaad1-490b-4044-8185-52a421057844", "attributions": [], "framework": "tn_k-12_5", "creator": "Guest name changed", "totalCompressedSize": 209850, "versionKey": "1652859827944", "mimeType": "application/vnd.ekstep.content-collection", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T07:44:22.582+0000", "code": "org.sunbird.11SAbB", "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "license": "CC BY 4.0", "leafNodes": [ "do_21353901951355289618149" ], "version": 2, "contentType": "Course", "language": [ "English" ], "lastPublishedOn": "2022-05-18T07:44:22.530+0000", "contentTypesCount": "{\"eTextBook\":1,\"CourseUnit\":1}", "objectType": "Content", "subjectIds": [ "tn_k-12_5_subject_accountancy" ], "status": "Live", "reservedDialcodes": { "U5X4C5": 0 }, "targetFWIds": [ "tn_k-12_5" ], "createdBy": "fca2925f-1eee-4654-9177-fece3fd6afc9", "dialcodeRequired": "No", "lastSubmittedOn": "2022-05-18T07:43:47.614+0000", "keywords": [ "Drop" ], "dialcodes": null, "userConsent": "Yes", "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 1, "depth": 2, "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "flagReasons": null, "targetSubjectIds": [ "tn_k-12_5_subject_english" ], "mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":1}", "se_gradeLevels": [ "Class 1" ], "osId": "org.ekstep.quiz.app", "copyrightYear": 2022, "se_FWIds": [ "tn_k-12_5" ], "s3Key": "content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "childNodes": [ "do_21353901951355289618149", "do_21354022740915814412313" ], "visibility": "Default", "credentials": { "enabled": "Yes" }, "targetGradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar\",\"size\":\"13797\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862667_do_21354022673739776012312_1_ONLINE.ecar\",\"size\":\"5181\"}}", "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi"} ], "mediaType": "content", "name": "Ch1 ", "discussionForum": {"enabled": "No" }, "createdOn": "2022-05-18T10:50:55.849+0000", "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "identifier": "do_21354031968951500812320", "ownershipType": ["createdBy" ], "compatibilityLevel": 1, "audience": ["Student" ], "os": ["All" ], "primaryCategory": "Textbook Unit", "languageCode": ["en" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "attributions": [], "versionKey": "1652871055849", "mimeType": "application/vnd.ekstep.content-collection", "code": "80986053-eb44-0c35-1c3b-c84806d4775a", "license": "CC BY 4.0", "leafNodes": ["do_21353901951355289618149" ], "version": 2, "contentType": "TextBookUnit", "language": ["English" ], "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "objectType": "Content", "status": "Live", "dialcodeRequired": "Yes", "dialcodes": ["R8C7G2" ], "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 1, "depth": 1, "osId": "org.ekstep.launcher", "contentDisposition": "inline", "visibility": "Parent", "credentials": {"enabled": "No" }, "variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar\",\"size\":\"16926\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar\",\"size\":\"8313\"}}", "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi" }, { "lastStatusChangedOn": "2022-05-18T10:50:55.854+0000", "parent": "do_21354027142511820812318", "children": [{ "parent": "do_21354031968955596812322", "children": [ {"lastStatusChangedOn": "2022-05-17T06:08:04.909+0000","parent": "do_21353947184001843212121","children": [ { "lastStatusChangedOn": "2022-05-17T06:08:47.990+0000", "parent": "do_21353947287477452812131", "mediaType": "content", "name": "വളരെ", "discussionForum": {"enabled": "No" }, "createdOn": "2022-05-17T06:08:47.990+0000", "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-17T06:12:02.265+0000", "identifier": "do_21353947322769408012133", "description": "വളരെ", "ownershipType": ["createdBy" ], "compatibilityLevel": 1, "audience": ["Student" ], "os": ["All" ], "primaryCategory": "Textbook Unit", "languageCode": ["en" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "attributions": [], "versionKey": "1652767727990", "mimeType": "application/vnd.ekstep.content-collection", "code": "f00aabd6-ca25-00de-97c7-44aa0d08b803", "license": "CC BY 4.0", "leafNodes": [], "version": 2, "contentType": "TextBookUnit", "language": ["English" ], "lastPublishedOn": "2022-05-17T06:12:27.957+0000", "objectType": "Content", "status": "Draft", "dialcodeRequired": "No", "keywords": ["വളരെ" ], "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 0, "depth": 2, "osId": "org.ekstep.launcher", "contentDisposition": "inline", "visibility": "Parent", "credentials": {"enabled": "No" }, "variants": {"spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019"},"online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018"} }, "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi" }],"mediaType": "content","name": "வாக்கியங்கள்","discussionForum": { "enabled": "No"},"createdOn": "2022-05-17T06:08:04.909+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-17T06:12:02.265+0000","identifier": "do_21353947287477452812131","description": "வாக்கியங்கள்","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"os": [ "All"],"primaryCategory": "Textbook Unit","languageCode": [ "en"],"downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar","attributions": [],"versionKey": "1652767684909","mimeType": "application/vnd.ekstep.content-collection","code": "a1e342e1-a00b-d708-598a-1c52c4fc2cc7","license": "CC BY 4.0","leafNodes": [],"version": 2,"contentType": "TextBookUnit","language": [ "English"],"lastPublishedOn": "2022-05-17T06:12:27.957+0000","objectType": "Content","status": "Draft","dialcodeRequired": "No","keywords": [ "வாக்கியங்கள்"],"idealScreenSize": "normal","contentEncoding": "gzip","leafNodesCount": 0,"depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018" }},"index": 1,"pkgVersion": 1,"idealScreenDensity": "hdpi" }, {"lastStatusChangedOn": "2022-05-17T06:09:30.081+0000","parent": "do_21353947184001843212121","mediaType": "content","name": "बतख़","discussionForum": { "enabled": "No"},"createdOn": "2022-05-17T06:09:30.081+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-17T06:12:02.265+0000","identifier": "do_21353947357250355212135","description": "बतख़","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"os": [ "All"],"primaryCategory": "Textbook Unit","languageCode": [ "en"],"downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar","attributions": [],"versionKey": "1652767770081","mimeType": "application/vnd.ekstep.content-collection","code": "710a43da-9a04-1d67-7d51-17bdce7fc1aa","license": "CC BY 4.0","leafNodes": [],"version": 2,"contentType": "TextBookUnit","language": [ "English"],"lastPublishedOn": "2022-05-17T06:12:27.957+0000","objectType": "Content","status": "Draft","dialcodeRequired": "No","keywords": [ "बतख़!"],"idealScreenSize": "normal","contentEncoding": "gzip","leafNodesCount": 0,"depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018" }},"index": 2,"pkgVersion": 1,"idealScreenDensity": "hdpi" }, {"lastStatusChangedOn": "2022-05-17T06:13:07.924+0000","parent": "do_21353947184001843212121","mediaType": "content","name": "മലയാളം","discussionForum": { "enabled": "No"},"createdOn": "2022-05-17T06:13:07.924+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-17T06:13:07.924+0000","identifier": "do_21353947535707340812137","description": "മലയാളം","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"os": [ "All"],"primaryCategory": "Textbook Unit","languageCode": [ "en"],"attributions": [],"versionKey": "1652767987924","mimeType": "application/vnd.ekstep.content-collection","code": "3e976c16-cefa-f237-f444-d25f2fa8f915","license": "CC BY 4.0","version": 2,"contentType": "TextBookUnit","language": [ "English"],"objectType": "Content","status": "Draft","dialcodeRequired": "No","keywords": [ "മലയാളം"],"idealScreenSize": "normal","contentEncoding": "gzip","depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"index": 3,"idealScreenDensity": "hdpi" } ], "identifier": "do_21353947184001843212121", "objectType": "Content", "depth": 2, "index": 1, "contentType": null, "primaryCategory": null} ], "mediaType": "content", "name": "Ch2", "discussionForum": {"enabled": "No" }, "createdOn": "2022-05-18T10:50:55.854+0000", "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "identifier": "do_21354031968955596812322", "ownershipType": ["createdBy" ], "compatibilityLevel": 1, "audience": ["Student" ], "os": ["All" ], "primaryCategory": "Textbook Unit", "languageCode": ["en" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "attributions": [], "versionKey": "1652871055854", "mimeType": "application/vnd.ekstep.content-collection", "code": "22c05b75-ae2a-6280-c192-2e2a4157d141", "license": "CC BY 4.0", "leafNodes": [], "version": 2, "contentType": "TextBookUnit", "language": ["English" ], "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "objectType": "Content", "status": "Live", "dialcodeRequired": "Yes", "dialcodes": ["A3S8L6" ], "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 0, "depth": 1, "osId": "org.ekstep.launcher", "contentDisposition": "inline", "visibility": "Parent", "credentials": {"enabled": "No" }, "variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar\",\"size\":\"16926\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar\",\"size\":\"8313\"}}", "index": 2, "pkgVersion": 1, "idealScreenDensity": "hdpi" }],"primaryCategory": "Digital Textbook","appId": "staging.sunbird.portal","contentEncoding": "gzip","lockKey": "3e222f39-d917-4582-96ee-458f1ab80f30","generateDIALCodes": "Yes","totalCompressedSize": 209850,"mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":8,\"\":1}","sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T10:53:41.389+0000","contentType": "TextBook","se_gradeLevels": [ "Class 1"],"trackable": { "enabled": "No", "autoBatch": "No"},"identifier": "do_21354027142511820812318","audience": [ "Student"],"se_boardIds": [ "tn_k-12_5_board_statetamilnadu"],"subjectIds": [ "tn_k-12_5_subject_english"],"toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json","visibility": "Default","contentTypesCount": "{\"TextBookUnit\":6,\"eTextBook\":1,\"Course\":1,\"CourseUnit\":1,\"\":1}","author": "ContentcreatorTN","consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada","childNodes": [ "do_21354022673739776012312", "do_21354031968951500812320", "do_21353947184001843212121", "do_21354031968955596812322", "do_21353901951355289618149", "do_21354022740915814412313"],"discussionForum": { "enabled": "No"},"mediaType": "content","osId": "org.ekstep.quiz.app","lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a","version": 2,"se_subjects": [ "English"],"license": "CC BY 4.0","qrCodeProcessId": "40c3e296-ccdc-487d-8cda-9f0a88c071d2","size": 16926,"lastPublishedOn": "2022-05-18T10:53:41.260+0000","name": "DialCodeHierarchy","mediumIds": [ "tn_k-12_5_medium_english"],"attributions": [],"status": "Live","code": "org.sunbird.D3eL5z","publishError": null,"credentials": { "enabled": "No"},"description": "Enter description for TextBook","medium": [ "English"],"idealScreenSize": "normal","createdOn": "2022-05-18T09:12:44.203+0000","reservedDialcodes": "{\"A3S8L6\":2,\"R8C7G2\":1,\"D1D5H9\":4,\"V2A8N1\":0,\"L5L9A2\":5,\"F6I6S6\":3}","se_boards": [ "State (Tamil Nadu)"],"se_mediumIds": [ "tn_k-12_5_medium_english"],"copyrightYear": 2022,"contentDisposition": "inline","additionalCategories": [ "Textbook"],"lastUpdatedOn": "2022-05-18T10:50:56.492+0000","SYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T10:52:07.401+0000","dialcodeRequired": "Yes","lastStatusChangedOn": "2022-05-18T09:12:44.203+0000","createdFor": [ "01269878797503692810"],"creator": "ContentcreatorTN","os": [ "All"],"flagReasons": null,"se_subjectIds": [ "tn_k-12_5_subject_english"],"se_FWIds": [ "tn_k-12_5"],"pkgVersion": 1,"versionKey": "1652871183414","idealScreenDensity": "hdpi","framework": "tn_k-12_5","dialcodes": [ "V2A8N1"],"depth": 0,"s3Key": "content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json","boardIds": [ "tn_k-12_5_board_statetamilnadu"],"lastSubmittedOn": "2022-05-18T10:53:03.017+0000","createdBy": "4cd4c690-eab6-4938-855a-447c7b1b8ea9","compatibilityLevel": 1,"leafNodesCount": 1,"userConsent": "Yes","gradeLevelIds": [ "tn_k-12_5_gradelevel_class1"],"board": "State (Tamil Nadu)","resourceType": "Book" }' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy) VALUES ( -'do_1234', -'{"identifier":"do_1234","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_12345', -'{"identifier":"do_12345","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}', -'' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_123456', -'{"identifier":"do_123456","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}', -'{}' -); diff --git a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index bba138e64..000000000 --- a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val PDF_EVENT1: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1619527882745,"mid":"LP.1619527882745.32dc378a-430f-49f6-83b5-bd73b767ad36","actor":{"id":"content-publish","type":"System"},"context":{"channel":"","pdata":{"id":"org.sunbird.platform","ver":"1.0"}},"object":{"id":"do_11329603741667328018","ver":"1619153418829"},"edata":{"publish_type":"public","metadata":{"identifier":"do_11329603741667328018","mimeType":"application/pdf","objectType":"Content","lastPublishedBy":"sample-last-published-by","pkgVersion":1},"action":"publish","iteration":1}} - |""".stripMargin -} diff --git a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/CollectionPublisherSpec.scala b/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/CollectionPublisherSpec.scala deleted file mode 100644 index 0d946431b..000000000 --- a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/CollectionPublisherSpec.scala +++ /dev/null @@ -1,242 +0,0 @@ -package org.sunbird.job.publish.helpers.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.commons.lang3.StringUtils -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.{doNothing, when} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.content.publish.helpers.CollectionPublisher -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, ElasticSearchUtil, HttpUtil, Neo4JUtil, ScalaJsonUtil} - -import java.text.SimpleDateFormat -import java.util -import java.util.Date -import scala.collection.mutable.ListBuffer -import scala.concurrent.ExecutionContextExecutor - -class CollectionPublisherSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: ContentPublishConfig = new ContentPublishConfig(config) - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val ec: ExecutionContextExecutor = ExecutionContexts.global - implicit val defCache: DefinitionCache = new DefinitionCache() - implicit val defConfig: DefinitionConfig = DefinitionConfig(jobConfig.schemaSupportVersionMap, jobConfig.definitionBasePath) - implicit val publishConfig: PublishConfig = jobConfig.asInstanceOf[PublishConfig] - implicit val httpUtil: HttpUtil = new HttpUtil - val mockElasticUtil: ElasticSearchUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - var definitionCache = new DefinitionCache() - implicit val definition: ObjectDefinition = definitionCache.getDefinition("Collection", jobConfig.schemaSupportVersionMap.getOrElse("collection", "1.0").asInstanceOf[String], jobConfig.definitionBasePath) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.hierarchyKeyspaceName, jobConfig.hierarchyTableName, definition.getExternalPrimaryKey, definition.getExternalProps) - - def getTimeStamp: String = { - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - sdf.format(new Date()) - } - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => - } - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } - - "enrichObjectMetadata" should "enrich the Content pkgVersion metadata" in { - val data = new ObjectData("do_2133950809948078081503", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_2133950809948078081503", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.content-collection", "keywords" -> Array[String]("test keyword"))) - val result: ObjectData = new TestCollectionPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - // "validateMetadata with invalid external data" should "return exception messages" in { - // val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef]), Some(Map[String, AnyRef]("artifactUrl" -> "artifactUrl"))) - // val result: List[String] = new TestCollectionPublisher().validateMetadata(data, data.identifier) - // result.size should be(1) - // } - - "saveExternalData " should "save external data to cassandra table" in { - val data = new ObjectData("do_1234", Map[String, AnyRef](), Some(Map[String, AnyRef]("body" -> "body", "answer" -> "answer"))) - new TestCollectionPublisher().saveExternalData(data, readerConfig) - } - - "getHierarchy " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestCollectionPublisher().getHierarchy(identifier, 1.0, readerConfig) - } - - "getExtDatas " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestCollectionPublisher().getExtDatas(List(identifier), readerConfig) - } - - "getHierarchies " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestCollectionPublisher().getHierarchies(List(identifier), readerConfig) - } - - "getDataForEcar" should "return one element in list" in { - val data = new ObjectData("do_123", Map("objectType" -> "Content"), Some(Map("responseDeclaration" -> "test")), Some(Map())) - val result: Option[List[Map[String, AnyRef]]] = new TestCollectionPublisher().getDataForEcar(data) - result.size should be(1) - } - - "getObjectWithEcar" should "return object with ecar url" in { - val unpublishedChildrenObj: List[Map[String, AnyRef]] = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](unpublishedChildrenData) - val data = new ObjectData("do_123", Map("objectType" -> "Collection", "identifier" -> "do_123", "name" -> "Test Collection", "lastPublishedOn" -> getTimeStamp, "lastUpdatedOn" -> getTimeStamp, "status" -> "Draft", "downloadUrl" -> "downloadUrl", "variants" -> Map.empty[String,AnyRef]), Some(Map()), Some(Map("children" -> unpublishedChildrenObj))) - val result = new TestCollectionPublisher().getObjectWithEcar(data, List(EcarPackageType.SPINE, EcarPackageType.ONLINE))(ec, mockNeo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, jobConfig, defCache, defConfig, httpUtil) - StringUtils.isNotBlank(result.metadata.getOrElse("downloadUrl", "").asInstanceOf[String]) - } - - "syncNodes" should "sync Child Nodes into ElasticSearch" in { - Mockito.reset(mockElasticUtil) - doNothing().when(mockElasticUtil).addDocument(anyString(), anyString()) - doNothing().when(mockElasticUtil).bulkIndexWithIndexId(anyString(), anyString(), any()) - - val do_113405023736512512114Json = """{"identifier":"do_113405023736512512114","graph_id":"domain","node_id":0,"collections":["do_11340502373639782416", "do_113405023736479744112"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_113405023736512512114")).thenReturn(do_113405023736512512114Json) - - val do_11340502373639782416Json = """{"identifier":"do_11340502373639782416","graph_id":"domain","node_id":0,"collections":["do_11340502373642240018", "do_11340502373608652812"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373639782416")).thenReturn(do_11340502373639782416Json) - - val do_11340502373642240018Json = """{"identifier":"do_11340502373642240018","graph_id":"domain","node_id":0,"collections":["do_11336831941257625611"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373642240018")).thenReturn(do_11340502373642240018Json) - - val do_11340502373608652812Json = """{"identifier":"do_11340502373608652812","graph_id":"domain","node_id":0,"collections":["do_11340096165525094411"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373608652812")).thenReturn(do_11340502373608652812Json) - - val do_113405023736479744112Json = """{"identifier":"do_113405023736479744112","graph_id":"domain","node_id":0,"collections":["do_11340502373638144014"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_113405023736479744112")).thenReturn(do_113405023736479744112Json) - - val do_11340502373638144014Json = """{"identifier":"do_11340502373638144014","graph_id":"domain","node_id":0,"collections":["do_113405023736455168110"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373638144014")).thenReturn(do_11340502373638144014Json) - - val do_113405023736455168110Json = """{"identifier":"do_113405023736455168110","graph_id":"domain","node_id":0,"collections":["do_11340096293585715212"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_113405023736455168110")).thenReturn(do_113405023736455168110Json) - - val publishedCollectionNodeMetadataObj: Map[String,AnyRef] = ScalaJsonUtil.deserialize[Map[String,AnyRef]](publishedCollectionNodeMetadata) - val data = new ObjectData("do_123", publishedCollectionNodeMetadataObj, Some(Map.empty[String, AnyRef])) - val syncChildrenData = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](publishedChildrenData) - val messages: Map[String, Map[String, AnyRef]] = new TestCollectionPublisher().syncNodes(data, syncChildrenData, List.empty)(mockElasticUtil, mockNeo4JUtil, cassandraUtil, readerConfig, definition, jobConfig) - - assert(messages != null && messages.size > 0) - } - - "updateHierarchyMetadata" should "update child nodes with published object metadata" in { - val unpublishedChildrenObj: List[Map[String, AnyRef]] = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](unpublishedChildrenData) - val publishedCollectionNodeMetadataObj: Map[String,AnyRef] = ScalaJsonUtil.deserialize[Map[String,AnyRef]](publishedCollectionNodeMetadata) - - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_123", 1, readerConfig)(cassandraUtil, jobConfig).get - // Collection - update and publish children - line 418 in PublishFinalizer - val updatedChildren: List[Map[String, AnyRef]] = new TestCollectionPublisher().updateHierarchyMetadata(unpublishedChildrenObj, publishedCollectionNodeMetadataObj, collRelationalMetadata)(jobConfig) - - assert(updatedChildren.nonEmpty) - - updatedChildren.map(child => { - assert(child.contains("pkgVersion")) - assert(child("pkgVersion").toString.equalsIgnoreCase("1")) - }) - } - - "getRelationalMetadata" should "return empty Map when there is no entry in relational_metadata column" in { - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_1234", 1, readerConfig)(cassandraUtil, jobConfig).get - assert(collRelationalMetadata != null && collRelationalMetadata.isEmpty) - } - - "getRelationalMetadata" should "return empty Map when there is empty entry in relational_metadata column" in { - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_12345", 1, readerConfig)(cassandraUtil, jobConfig).get - assert(collRelationalMetadata != null && collRelationalMetadata.isEmpty) - } - - "getRelationalMetadata" should "return empty Map when there is empty object entry in relational_metadata column" in { - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_123456", 1, readerConfig)(cassandraUtil, jobConfig).get - assert(collRelationalMetadata != null && collRelationalMetadata.isEmpty) - } - - "publishHierarchy " should "save hierarchy data to cassandra table" in { - val publishChildrenData = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](publishedChildrenData) - val publishedCollectionNodeMetadataObj: Map[String,AnyRef] = ScalaJsonUtil.deserialize[Map[String,AnyRef]](publishedCollectionNodeMetadata) - val data = new ObjectData("do_123", publishedCollectionNodeMetadataObj, Some(Map.empty[String, AnyRef])) - val result = new TestCollectionPublisher().publishHierarchy(publishChildrenData, data, readerConfig, jobConfig) - assert(result) - } - - "getUnitsFromLiveContent" should "return object hierarchy" in { - val data = new ObjectData("do_2133950809948078081503", Map("identifier" -> "do_2133950809948078081503"), Some(Map.empty[String, AnyRef])) - val fetchedChildren = new TestCollectionPublisher().getUnitsFromLiveContent(data)(cassandraUtil, readerConfig, jobConfig) - assert(fetchedChildren.nonEmpty) - } - - "updateOriginPkgVersion" should "return origin node Data" in { - val metaData = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_11300581751853056018") - put("identifier", "do_11300581751853056018") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"Yes\",\"autoBatch\":\"Yes\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("pkgVersion", 3.asInstanceOf[AnyRef]) - } - } - val collectionObj = new ObjectData("do_123", Map("objectType" -> "Collection", "identifier" -> "do_123", "name" -> "Test Collection","origin" -> "do_456", "originData" -> Map("name" -> "Contemporary India I", "copyType" -> "deep", "license" -> "CC BY 4.0", "organisation" -> Array("NCERT"))), Some(Map()), Some(Map())) - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(metaData) - val originObj = new TestCollectionPublisher().updateOriginPkgVersion(collectionObj) - assert(originObj.metadata("originData").asInstanceOf[Map[String,AnyRef]]("pkgVersion") == 3) - } - - "saveImageHierarchy" should "insert a record in cassandra DB" in { - val unpublishedChildrenObj: List[Map[String, AnyRef]] = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](unpublishedChildrenData) - val collRelationalMetadata: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](collRelationalMetadataStr) - val data = new ObjectData("do_123.img", Map("objectType" -> "Collection", "identifier" -> "do_123.img", "name" -> "Test Collection", "lastPublishedOn" -> getTimeStamp, "lastUpdatedOn" -> getTimeStamp, "status" -> "Draft", "downloadUrl" -> "downloadUrl", "variants" -> Map.empty[String,AnyRef]), Some(Map()), Some(Map("children" -> unpublishedChildrenObj))) - new TestCollectionPublisher().saveImageHierarchy(data, readerConfig, collRelationalMetadata) - } - - "fetchDialListForContextUpdate" should "fetch the list of added and removed QR codes" in { - val nodeObj = new ObjectData("do_21354027142511820812318.img", Map("objectType" -> "Collection", "identifier" -> "do_21354027142511820812318", "name" -> "DialCodeHierarchy", "lastPublishedOn" -> getTimeStamp, "lastUpdatedOn" -> getTimeStamp, "status" -> "Draft", "pkgVersion" -> 1.asInstanceOf[Number], "versionKey" -> "1652871771396"), Some(Map()), Some(Map())) - val DIALListMap = new TestCollectionPublisher().fetchDialListForContextUpdate(nodeObj)(mockNeo4JUtil, cassandraUtil, readerConfig, jobConfig) - println("DIALListMap:: " + DIALListMap) - assert(DIALListMap.nonEmpty) - } - - - val collRelationalMetadataStr = "{\"do_123\":{\"name\":\"Collection Publish T21\",\"children\":[\"do_11340511137112064018\",\"do_11340511137080934412\"],\"root\":true},\"do_11340511137112064018\":{\"name\":\"Collection Parent\",\"children\":[\"do_11340096165525094411\"],\"root\":false,\"relationalMetadata\":{\"do_11340096165525094411\":{\"name\":\"Test Name RM L1 - R1\",\"keywords\":[\"Overwriting content KW1\"]}}},\"do_11340511137080934412\":{\"name\":\"Collection Parent\",\"children\":[\"do_11340096165525094411\"],\"root\":false,\"relationalMetadata\":{\"do_11340096165525094411\":{\"name\":\"Test Name RM L1 - R1\",\"keywords\":[\"Overwriting content KW1\"]}}},\"do_11340096165525094411\":{\"name\":\"PDF Content\",\"children\":[],\"root\":false},\"do_113405111371145216110\":{\"name\":\"test\",\"children\":[], \"root\":false}}" - val publishedChildrenData = "[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.586+0530\",\"parent\":\"do_11340502356035174411\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.572+0530\",\"parent\":\"do_113405023736512512114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.575+0530\",\"parent\":\"do_11340502373639782416\",\"children\":[{\"copyright\":\"J H S BHARKHOKHA, Tamil Nadu\",\"lastStatusChangedOn\":\"2021-09-17T16:22:50.404+0530\",\"parent\":\"do_11340502373642240018\",\"licenseterms\":\"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\"organisation\":[\"J H S BHARKHOKHA\",\"Tamil Nadu\"],\"mediaType\":\"content\",\"name\":\"jaga Aug 25th more than 200mp mp4 update 1\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-09-17T16:05:29.019+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-09-17T16:22:50.404+0530\",\"size\":363062652,\"identifier\":\"do_11336831941257625611\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"eTextbook\",\"appIcon\":\"https://sunbirddevbbpublic.blob.core.windows.net/sunbird-content-staging/content/assets/do_2137327580080128001217/gateway-of-india.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"\",\"framework\":\"tn_k-12_5\",\"creator\":\"सामग्री निर्माता TN\",\"versionKey\":\"1631875539805\",\"mimeType\":\"video/mp4\",\"code\":\"3bd7411e-c03c-4997-a247-4d43a5cc820b\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Live\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-09-17T16:22:15.047+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"fca2925f-1eee-4654-9177-fece3fd6afc9\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"consumerId\":\"2eaff3db-cdd1-42e5-a611-bebbf906e6cf\",\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"tn_k-12_5\"],\"contentDisposition\":\"online-only\",\"previewUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"artifactUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11336831941257625611/jaga-aug-25th-more-than-200mp-mp4-update-1_1631875935857_do_11336831941257625611_3_SPINE.ecar\",\"size\":\"2172\"}},\"index\":1,\"pkgVersion\":3,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.1 Key parts in the head\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.575+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373642240018\",\"description\":\"xyz\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436575\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"7991af5c2e51e4d3d7b83167aaac8829\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11336831941257625611\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.534+0530\",\"parent\":\"do_11340502373639782416\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:13:39.729+0530\",\"parent\":\"do_11340502373608652812\",\"mediaType\":\"content\",\"name\":\"Collection Publishing PDF Content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:56:17.917+0530\",\"createdFor\":[\"01309282781705830427\"],\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:13:39.729+0530\",\"streamingUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"identifier\":\"do_11340096165525094411\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":4,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"creator\":\"N131\",\"versionKey\":\"1635859577917\",\"mimeType\":\"application/pdf\",\"code\":\"c9ce1ce0-b9b4-402e-a9c3-556701070838\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Processing\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:13:35.589+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"pragma\":[\"external\"],\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPPDFContent1\",\"CPPDFContent2\",\"CollectionKW1\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"size\":\"256918\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar\",\"size\":\"6378\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.2 Other parts\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.534+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373608652812\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436534\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"3bf70f06d3e8dba010d8806fd94259b1\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":2,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1 Parts of Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.572+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373639782416\",\"description\":\"This section describes about various part of the body such as head, hands, legs etc.\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436572\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"20cc1f31e62f924c6e47bf04c994376b\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11336831941257625611\",\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":2,\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.582+0530\",\"parent\":\"do_113405023736512512114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.570+0530\",\"parent\":\"do_113405023736479744112\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.579+0530\",\"parent\":\"do_11340502373638144014\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:16:10.667+0530\",\"parent\":\"do_113405023736455168110\",\"mediaType\":\"content\",\"name\":\"Collection Publish MP4 content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:58:53.445+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:16:10.667+0530\",\"identifier\":\"do_11340096293585715212\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1134009488766730241130/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"versionKey\":\"1635859733445\",\"mimeType\":\"video/mp4\",\"code\":\"e0b58864-3dc5-484a-b194-38c3eddcbce1\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Draft\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:16:08.789+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPMP4ContentKW1\",\"CPMP4ContentKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":5,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"size\":\"2692101\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860770277_do_11340096293585715212_1_SPINE.ecar\",\"size\":\"6275\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"dsffgdg\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.579+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_113405023736455168110\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436579\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"9cf84ff2fb08f9af4c23eb09df9b2520\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":4,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2.1 Respiratory System\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.570+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373638144014\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"attributions\":[],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436570\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"b186b1bbcc9c58db865f75e34345179e\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"keywords\":[\"UnitKW1\",\"UnitKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2 Organ Systems\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.582+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_113405023736479744112\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436582\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"40a1ed37e0fad94eca76b2a96fe086ab\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":2,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5. Human Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.586+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_113405023736512512114\",\"description\":\"This chapter describes about human body\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436586\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"76abafa2a0c2cfef90b52db1ef41fb82\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\",\"do_11336831941257625611\",\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":3,\"depth\":1,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}]" - val unpublishedChildrenData = "[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.180+0530\",\"parent\":\"do_11340511118032076811\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.166+0530\",\"parent\":\"do_113405111371202560114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.170+0530\",\"parent\":\"do_11340511137108787216\",\"children\":[{\"copyright\":\"J H S BHARKHOKHA, Tamil Nadu\",\"lastStatusChangedOn\":\"2021-09-17T16:22:50.404+0530\",\"parent\":\"do_11340511137112064018\",\"licenseterms\":\"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\"organisation\":[\"J H S BHARKHOKHA\",\"Tamil Nadu\"],\"mediaType\":\"content\",\"name\":\"jaga Aug 25th more than 200mp mp4 update 1\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-09-17T16:05:29.019+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-09-17T16:22:50.404+0530\",\"size\":363062652,\"identifier\":\"do_11336831941257625611\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"eTextbook\",\"appIcon\":\"https://sunbirddevbbpublic.blob.core.windows.net/sunbird-content-staging/content/assets/do_2137327580080128001217/gateway-of-india.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"\",\"framework\":\"tn_k-12_5\",\"creator\":\"सामग्री निर्माता TN\",\"versionKey\":\"1631875539805\",\"mimeType\":\"video/mp4\",\"code\":\"3bd7411e-c03c-4997-a247-4d43a5cc820b\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Live\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-09-17T16:22:15.047+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"fca2925f-1eee-4654-9177-fece3fd6afc9\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"consumerId\":\"2eaff3db-cdd1-42e5-a611-bebbf906e6cf\",\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"tn_k-12_5\"],\"contentDisposition\":\"online-only\",\"previewUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"artifactUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11336831941257625611/jaga-aug-25th-more-than-200mp-mp4-update-1_1631875935857_do_11336831941257625611_3_SPINE.ecar\",\"size\":\"2172\"}},\"index\":1,\"pkgVersion\":3,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.1 Key parts in the head\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.170+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.170+0530\",\"identifier\":\"do_11340511137112064018\",\"description\":\"xyz\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134170\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"7991af5c2e51e4d3d7b83167aaac8829\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.133+0530\",\"parent\":\"do_11340511137108787216\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:13:39.729+0530\",\"parent\":\"do_11340511137080934412\",\"mediaType\":\"content\",\"name\":\"Collection Publishing PDF Content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:56:17.917+0530\",\"createdFor\":[\"01309282781705830427\"],\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:13:39.729+0530\",\"streamingUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"identifier\":\"do_11340096165525094411\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":4,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"creator\":\"N131\",\"versionKey\":\"1635859577917\",\"mimeType\":\"application/pdf\",\"code\":\"c9ce1ce0-b9b4-402e-a9c3-556701070838\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Processing\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:13:35.589+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"pragma\":[\"external\"],\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPPDFContent1\",\"CPPDFContent2\",\"CollectionKW1\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"size\":\"256918\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar\",\"size\":\"6378\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.2 Other parts\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.133+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.132+0530\",\"identifier\":\"do_11340511137080934412\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134133\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"3bf70f06d3e8dba010d8806fd94259b1\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":2,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1 Parts of Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.166+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.165+0530\",\"identifier\":\"do_11340511137108787216\",\"description\":\"This section describes about various part of the body such as head, hands, legs etc.\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134166\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"20cc1f31e62f924c6e47bf04c994376b\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.177+0530\",\"parent\":\"do_113405111371202560114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.162+0530\",\"parent\":\"do_113405111371169792112\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.173+0530\",\"parent\":\"do_11340511137105510414\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:16:10.667+0530\",\"parent\":\"do_113405111371145216110\",\"mediaType\":\"content\",\"name\":\"Collection Publish MP4 content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:58:53.445+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:16:10.667+0530\",\"identifier\":\"do_11340096293585715212\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1134009488766730241130/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"versionKey\":\"1635859733445\",\"mimeType\":\"video/mp4\",\"code\":\"e0b58864-3dc5-484a-b194-38c3eddcbce1\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Draft\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:16:08.789+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPMP4ContentKW1\",\"CPMP4ContentKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":5,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"size\":\"2692101\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860770277_do_11340096293585715212_1_SPINE.ecar\",\"size\":\"6275\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"dsffgdg\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.173+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.173+0530\",\"identifier\":\"do_113405111371145216110\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134173\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"9cf84ff2fb08f9af4c23eb09df9b2520\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":4,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2.1 Respiratory System\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.162+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:53:32.894+0530\",\"identifier\":\"do_11340511137105510414\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"attributions\":[],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134162\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"b186b1bbcc9c58db865f75e34345179e\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"keywords\":[\"UnitKW1\",\"UnitKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2 Organ Systems\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.176+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.176+0530\",\"identifier\":\"do_113405111371169792112\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134176\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"40a1ed37e0fad94eca76b2a96fe086ab\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":2,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5. Human Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.180+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.179+0530\",\"identifier\":\"do_113405111371202560114\",\"description\":\"This chapter describes about human body\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134180\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"76abafa2a0c2cfef90b52db1ef41fb82\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":1,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"}]" - val publishedCollectionNodeMetadata = "{\"copyright\":\"tn\",\"lastStatusChangedOn\":\"2021-11-08T15:38:31.391+0530\",\"publish_type\":\"public\",\"author\":\"ContentcreatorTN\",\"children\":[{\"name\":\"5. Human Body\",\"identifier\":\"do_113405111371202560114\",\"description\":\"This chapter describes about human body\",\"objectType\":\"Collection\",\"index\":1}],\"body\":null,\"mediaType\":\"content\",\"name\":\"Collection Publish T20\",\"toc_url\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340511118032076811/artifact/do_11340511118032076811_toc.json\",\"discussionForum\":\"{\\\"enabled\\\":\\\"No\\\"}\",\"createdOn\":\"2021-11-08T15:38:31.391+0530\",\"createdFor\":[\"0125196274181898243\"],\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:53:33.587+0530\",\"size\":12048,\"publishError\":null,\"identifier\":\"do_11340511118032076811\",\"description\":\"Collection Publish\",\"resourceType\":\"Book\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"trackable\":\"{\\\"enabled\\\":\\\"No\\\",\\\"autoBatch\\\":\\\"No\\\"}\",\"os\":[\"All\"],\"primaryCategory\":\"Digital Textbook\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/artifact/16_bvnvrokht6hn97eqcklwk2fs6ppx2z.thumb.png\",\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/collection-publish-t20_1636367178646_do_11340511118032076811_1_SPINE.ecar\",\"attributions\":[],\"framework\":\"ncert_k-12\",\"posterImage\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/assets/do_31321903538537267212904/16_bvnvrokht6hn97eqcklwk2fs6ppx2z.png\",\"creator\":\"NCERT\",\"totalCompressedSize\":363062652,\"versionKey\":\"1636367013587\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"0125196274181898243\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\",\"do_11336831941257625611\",\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBook\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T15:56:15.604+0530\",\"contentTypesCount\":\"{\\\"TextBookUnit\\\":7,\\\"Resource\\\":3}\",\"objectType\":\"Collection\",\"status\":\"Live\",\"createdBy\":\"220d4745-6764-498d-ad37-5e49b8cce716\",\"dialcodeRequired\":\"No\",\"keywords\":[\"CPPDFContent1\",\"UnitKW2\",\"CollectionKW1\",\"CPPDFContent2\",\"UnitKW1\",\"CPMP4ContentKW1\",\"CPMP4ContentKW2\"],\"userConsent\":\"Yes\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":3,\"depth\":0,\"flagReasons\":null,\"mimeTypesCount\":\"{\\\"application/pdf\\\":1,\\\"video/mp4\\\":2,\\\"application/vnd.ekstep.content-collection\\\":7}\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"ncert_k-12\"],\"contentDisposition\":\"inline\",\"additionalCategories\":[],\"childNodes\":[\"do_11336831941257625611\",\"do_11340511137112064018\",\"do_11340511137108787216\",\"do_113405111371202560114\",\"do_11340096165525094411\",\"do_11340511137080934412\",\"do_11340096293585715212\",\"do_113405111371145216110\",\"do_11340511137105510414\",\"do_113405111371169792112\"],\"visibility\":\"Default\",\"credentials\":\"{\\\"enabled\\\":\\\"No\\\"}\",\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/collection-publish-t20_1636367178646_do_11340511118032076811_1_SPINE.ecar\\\",\\\"size\\\":\\\"12048\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/collection-publish-t20_1636367178792_do_11340511118032076811_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5081\\\"}}\",\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}" -} - - -class TestCollectionPublisher extends CollectionPublisher {} \ No newline at end of file diff --git a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/ContentPublisherSpec.scala b/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/ContentPublisherSpec.scala deleted file mode 100644 index 1dd5e99ca..000000000 --- a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/ContentPublisherSpec.scala +++ /dev/null @@ -1,243 +0,0 @@ -package org.sunbird.job.publish.helpers.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.commons.lang3.StringUtils -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.content.publish.helpers.ContentPublisher -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} - -import scala.concurrent.ExecutionContextExecutor - -class ContentPublisherSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: ContentPublishConfig = new ContentPublishConfig(config) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.contentKeyspaceName, jobConfig.contentTableName) - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val ec: ExecutionContextExecutor = ExecutionContexts.global - implicit val defCache: DefinitionCache = new DefinitionCache() - implicit val defConfig: DefinitionConfig = DefinitionConfig(jobConfig.schemaSupportVersionMap, jobConfig.definitionBasePath) - implicit val publishConfig: PublishConfig = jobConfig.asInstanceOf[PublishConfig] - implicit val httpUtil: HttpUtil = new HttpUtil - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => - } - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } - - "enrichObjectMetadata" should "enrich the Content pkgVersion metadata" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf")) - val result: ObjectData = new TestContentPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - ignore should "enrich the Content metadata for application/vnd.ekstep.html-archive should through exception in artifactUrl is not available" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.html-archive")) - val result: ObjectData = new TestContentPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - "enrichObjectMetadata" should "enrich the Content metadata for application/vnd.ekstep.html-archive" in { - val data = new ObjectData("do_1132167819505500161297", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_1132167819505500161297", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.html-archive", "artifactUrl" -> "artifactUrl.zip")) - val result: ObjectData = new TestContentPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - "validateMetadata with invalid external data" should "return exception messages" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef]), Some(Map[String, AnyRef]("artifactUrl" -> "artifactUrl"))) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " return exception messages if extData is set as None" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Either 'body' or 'artifactUrl' are required for processing of ECML content for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " return exception messages if is having body=\"\"" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), Some(Map[String, AnyRef]("body" -> ""))) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Either 'body' or 'artifactUrl' are required for processing of ECML content for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " return exception messages if is having body=null" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), Some(Map[String, AnyRef]("body" -> null))) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Either 'body' or 'artifactUrl' are required for processing of ECML content for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " not return exception messages if is having body=null but artifactUrl is available" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive", "artifactUrl" -> "sampleUrl"), Some(Map[String, AnyRef]("body" -> null))) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " not return exception messages if is having valid body" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"id\":\"theme\",\"version\":\"1.0\",\"startStage\":\"8ab605cc-b26d-4d0d-a827-2112b0330c3a\",\"stage\":[{\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"id\":\"8ab605cc-b26d-4d0d-a827-2112b0330c3a\",\"rotate\":null,\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true,\\\"color\\\":\\\"#FFFFFF\\\",\\\"genieControls\\\":false,\\\"instructions\\\":\\\"\\\"}\"},\"manifest\":{\"media\":[{\"assetId\":\"do_113303238321799168110\"},{\"assetId\":\"do_113303315360907264114\"}]},\"image\":[{\"asset\":\"do_113303238321799168110\",\"x\":20,\"y\":20,\"w\":49.51,\"h\":14.29,\"rotate\":0,\"z-index\":0,\"id\":\"a62c82d4-e497-424a-b9a5-45b585572e20\",\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true}\"}},{\"asset\":\"do_113303315360907264114\",\"x\":20,\"y\":20,\"w\":49.51,\"h\":14.29,\"rotate\":0,\"z-index\":1,\"id\":\"dcc3d9e7-e4e0-409c-b099-434523556c10\",\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true}\"}}]}],\"manifest\":{\"media\":[{\"id\":\"1b34ae41-4c56-4c4d-a0e4-be56169cb7e6\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/controller/navigation_ctrl.js\",\"type\":\"js\"},{\"id\":\"e65308d4-8420-4533-b74d-87c1909e9b12\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/templates/navigation.html\",\"type\":\"js\"},{\"id\":\"org.ekstep.navigation\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.navigation_manifest\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/manifest.json\",\"type\":\"json\"},{\"name\":\"Screenshot 2021-06-17 at 5 17 12 PM\",\"id\":\"do_113303238321799168110\",\"src\":\"/assets/public/content/do_113303238321799168110/artifact/do_113303238321799168110_1623930459486_screenshot-2021-06-17-at-5.17.12-pm.png\",\"type\":\"image\"},{\"name\":\"Screenshot 2021-06-17 at 5 17 12 PM\",\"id\":\"do_113303315360907264114\",\"src\":\"/assets/public/content/do_113303315360907264114/artifact/do_113303315360907264114_1623939863955_screenshot-2021-06-17-at-5.17.12-pm.png\",\"type\":\"image\"}]},\"plugin-manifest\":{\"plugin\":[{\"id\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"type\":\"plugin\",\"depends\":\"\"}]},\"compatibilityVersion\":2}}"))) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " return exception messages if content is having invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://www.youtube.com/"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Invalid youtube Url = https://www.youtube.com/ for : do_123") shouldBe true - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " not return exception messages if content is having valid artifactUrl = https://www.youtube.com/embed/watch?" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://www.youtube.com/embed/watch?"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " not return exception messages if content is having valid artifactUrl = https://www.youtube.com/watch?v=6Js8tBCfbWk" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://www.youtube.com/watch?v=6Js8tBCfbWk"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " not return exception messages if content is having valid artifactUrl = https://youtu.be/6Js8tBCfbWk" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://youtu.be/6Js8tBCfbWk"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/pdf " should " throw InvalidInputException invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf", "artifactUrl" -> "https://www.youtube.com/"), None) - assertThrows[InvalidInputException] { - new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - } - } - - "validateMetadata with mimeType application/pdf " should " exception messages if content is having other fileType in artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130958930694553601102/artifact/index.epub"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Error! Invalid File Extension. Uploaded file https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130958930694553601102/artifact/index.epub is not a pdf file for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/pdf " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/epub " should " return exception messages if content is having invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/epub", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Error! Invalid File Extension. Uploaded file https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf is not a epub file for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/epub " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/epub", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130958930694553601102/artifact/index.epub"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/msword " should " return exception messages if content is having invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/msword", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Error! Invalid File Extension. | Uploaded file https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf should be among the Allowed_file_extensions for mimeType doc [doc, docx, ppt, pptx, key, odp, pps, odt, wpd, wps, wks] for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/msword and .pptx " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/msword", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112216616320983040129/artifact/performance_out_1491286194831.pptx"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/msword and .docx " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/msword", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112216615190192128128/artifact/prdassetstagging-2_1491286084107.docx"), None) - val result: List[String] = new TestContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "saveExternalData " should "save external data to cassandra table" in { - val data = new ObjectData("do_123", Map[String, AnyRef](), Some(Map[String, AnyRef]("body" -> "body", "answer" -> "answer"))) - new TestContentPublisher().saveExternalData(data, readerConfig) - } - - "getExtData " should " get content body for application/vnd.ekstep.ecml-archive mimeType " in { - val identifier = "do_11321328578759884811663" - val result: Option[ObjectExtData] = new TestContentPublisher().getExtData(identifier, 0.0, "application/vnd.ekstep.ecml-archive", readerConfig) - result.getOrElse(new ObjectExtData).data.getOrElse(Map()).contains("body") shouldBe true - } - - "getExtData " should " not get content body for other than application/pdf mimeType " in { - val identifier = "do_11321328578759884811663" - val result: Option[ObjectExtData] = new TestContentPublisher().getExtData(identifier, 0.0, "application/pdf", readerConfig) - result.getOrElse(new ObjectExtData).data.getOrElse(Map()).contains("body") shouldBe false - } - - "getHierarchy " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestContentPublisher().getHierarchy(identifier, 1.0, readerConfig) - } - - "getExtDatas " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestContentPublisher().getExtDatas(List(identifier), readerConfig) - } - - "getHierarchies " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestContentPublisher().getHierarchies(List(identifier), readerConfig) - } - - "getDataForEcar" should "return one element in list" in { - val data = new ObjectData("do_123", Map("objectType" -> "Content"), Some(Map("responseDeclaration" -> "test")), Some(Map())) - val result: Option[List[Map[String, AnyRef]]] = new TestContentPublisher().getDataForEcar(data) - result.size should be(1) - } - - "getObjectWithEcar" should "return object with ecar url" in { - val data = new ObjectData("do_123", Map("objectType" -> "Content", "identifier" -> "do_123", "name" -> "Test PDF Content"), Some(Map("responseDeclaration" -> "test", "media" -> "[{\"id\":\"do_1127129497561497601326\",\"type\":\"image\",\"src\":\"/content/do_1127129497561497601326.img/artifact/sunbird_1551961194254.jpeg\",\"baseUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev\"}]")), Some(Map())) - val result = new TestContentPublisher().getObjectWithEcar(data, List(EcarPackageType.FULL.toString, EcarPackageType.ONLINE))(ec, mockNeo4JUtil, cloudStorageUtil, jobConfig, defCache, defConfig, httpUtil) - StringUtils.isNotBlank(result.metadata.getOrElse("downloadUrl", "").asInstanceOf[String]) - } -} - -class TestContentPublisher extends ContentPublisher {} diff --git a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/ExtractableMimeTypeHelperSpec.scala b/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/ExtractableMimeTypeHelperSpec.scala deleted file mode 100644 index 2137bf6ab..000000000 --- a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/ExtractableMimeTypeHelperSpec.scala +++ /dev/null @@ -1,165 +0,0 @@ -package org.sunbird.job.publish.helpers.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.content.publish.helpers.ExtractableMimeTypeHelper -import org.sunbird.job.content.task.ContentPublishConfig -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.CloudStorageUtil - -class ExtractableMimeTypeHelperSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val ec = ExecutionContexts.global - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: ContentPublishConfig = new ContentPublishConfig(config) - implicit val cloudStorageUtil = new CloudStorageUtil(jobConfig) - - "processECMLBody with xml " should " throw exception Error! Invalid Media ('id' is required.) in ... if media id is blank and type is other than js and css" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with xml " should " throw exception Error! Invalid Media ('type' is required.) in ... if media type is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with xml " should " throw exception Error! Invalid Media ('src' is required.) in ... if media src is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with xml " should " process the ecml body" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "

A person having less ‘haemoglobin’ is suffering from:

Jaundice

Anaemia

Malaria

Chikungunya

\",\"media\":[],\"responseDeclaration\":{\"responseValue\":{\"cardinality\":\"single\",\"type\":\"integer\",\"correct_response\":{\"value\":\"2\"}}},\"options\":[{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Jaundice

\",\"resvalue\":0,\"resindex\":0}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Anaemia

\",\"resvalue\":1,\"resindex\":1}},{\"answer\":true,\"value\":{\"type\":\"text\",\"body\":\"

Malaria

\",\"resvalue\":2,\"resindex\":2}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Chikungunya

\",\"resvalue\":3,\"resindex\":3}}],\"questionCount\":0}]]>")) - ) - val result: Map[String, AnyRef] = ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - result.contains("artifactUrl") shouldBe true - result.get("artifactUrl") shouldNot be(null) - result.contains("cloudStorageKey") shouldBe true - result.get("cloudStorageKey") shouldNot be(null) - } - - "processECMLBody with json " should " throw exception if no body is available" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]()) - ) - - assertThrows[InvalidInputException] { - val result: Map[String, AnyRef] = ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Invalid Content Body if not a valid body" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "Invalid body")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Controller ('id' is required.)" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"\",\"__cdata\":{}}]}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Controller ('type' is required.)" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"\",\"id\":\"dictionary\",\"__cdata\":{}}]}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Controller ('type' should be either 'items' or 'data')" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"some type\",\"id\":\"dictionary\",\"__cdata\":{}}]}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Media ('id' is required.) if media id is blank and type is other than js and css" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"manifest\":{\"media\":[{\"id\":\"\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png\",\"assetId\":\"QuizImage\",\"type\":\"image\",\"preload\":true}]}}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Media ('type' is required.) if media type is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"manifest\":{\"media\":[{\"id\":\"QuizImage\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png\",\"assetId\":\"QuizImage\",\"type\":\"\",\"preload\":true}]}}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Media ('src' is required.) if media src is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"manifest\":{\"media\":[{\"id\":\"QuizImage\",\"src\":\"\",\"assetId\":\"QuizImage\",\"type\":\"image\",\"preload\":true}]}}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " process the ecml body" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"manifest\":{\"media\":[{\"id\":253452185,\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/controller/navigation_ctrl.js\",\"type\":\"js\"},{\"id\":\"c264cc74-39dc-40e6-8e6a-04b1b379bd52\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/templates/navigation.html\",\"type\":\"js\"},{\"id\":\"org.ekstep.navigation\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.navigation_manifest\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"org.ekstep.questionunit.renderer.audioicon\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/audio-icon.png\",\"type\":\"image\"},{\"id\":\"org.ekstep.questionunit.renderer.downarrow\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/down_arrow.png\",\"type\":\"image\"},{\"id\":\"489ca96e-de6d-41c2-a346-3e13a5e81cd1\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/js/components.js\",\"type\":\"js\"},{\"id\":\"d3eab6b4-eb5a-4739-b985-ca85caaae11e\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/css/components.css\",\"type\":\"css\"},{\"id\":\"4152517e-7698-4813-840c-ab0d4bb1a48a\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.js\",\"type\":\"js\"},{\"id\":\"d9a519d7-e450-4b31-be54-5cec9d21d5ce\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.css\",\"type\":\"css\"},{\"id\":\"org.ekstep.questionunit\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionunit_manifest\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"7139ec8e-dc5d-437b-a7c4-62d0be5a28c5\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/style.css\",\"type\":\"css\"},{\"id\":\"bdd3831b-95ca-4398-a6e6-6a61984452a5\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/horizontal_and_vertical.css\",\"type\":\"css\"},{\"id\":\"09bd898a-6132-4ebd-936c-5437a2134229\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/mcq-layouts.js\",\"type\":\"js\"},{\"id\":\"a46352ba-dccb-4c5e-bc1a-84beeb85037c\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/template_controller.js\",\"type\":\"js\"},{\"id\":\"38886295-0bac-4654-bded-6d032cb37431\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/tick_icon.png\",\"type\":\"image\"},{\"id\":\"37aabee2-1538-4391-8d1d-5d406722b084\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio-icon2.png\",\"type\":\"image\"},{\"id\":\"73d9743a-9202-4e68-8797-ad873b67cdb6\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/music-blue.png\",\"type\":\"image\"},{\"id\":\"org.ekstep.questionunit.mcq\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionunit.mcq_manifest\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/manifest.json\",\"type\":\"json\"},{\"id\":\"org.ekstep.questionset.quiz\",\"plugin\":\"org.ekstep.questionset.quiz\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset.quiz-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionset.quiz_manifest\",\"plugin\":\"org.ekstep.questionset.quiz\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset.quiz-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"org.ekstep.iterator\",\"plugin\":\"org.ekstep.iterator\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.iterator-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.iterator_manifest\",\"plugin\":\"org.ekstep.iterator\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.iterator-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"cb4d4669-9291-4583-906c-685ab9a76994\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/telemetry_logger.js\",\"type\":\"js\"},{\"id\":\"dd12429d-d861-4d54-b80d-12047823b058\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/html_audio_plugin.js\",\"type\":\"js\"},{\"id\":\"56254ade-8e4e-4090-b97b-8b9b7bc12620\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/qs_feedback_popup.js\",\"type\":\"js\"},{\"id\":\"org.ekstep.questionset\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionset_manifest\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"QuizImage\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png\",\"assetId\":\"QuizImage\",\"type\":\"image\",\"preload\":true}]},\"id\":\"theme\",\"startStage\":\"scene55138843-9f04-487d-bf33-1920769f2c96\",\"ver\":0.2,\"controller\":[{\"name\":\"assessment_185d692a-1841-4a35-96c1-67cebcde42a2\",\"type\":\"items\",\"id\":\"assessment_185d692a-1841-4a35-96c1-67cebcde42a2\",\"__cdata\":{\"total_items\":1,\"subject\":\"domain\",\"code\":\"ItemSet_921ee047-c676-4e0b-9a71-42f371f7b843\",\"type\":\"materialised\",\"lastUpdatedOn\":\"2016-06-15T09:42:19.585+0000\",\"showImmediateFeedback\":true,\"SET_TYPE\":\"MATERIALISED_SET\",\"createdOn\":\"2016-06-15T09:42:19.575+0000\",\"title\":\"Assessment Title\",\"items\":{\"domain_11824\":[{\"model\":{\"keys\":\"र,रा,रि,री,रु,रू,रे,रै,रो,रौ,रं,रः,ट,टा,टि,टी,टु,टू,टे,टै,टो,टौ,टं,टः\"},\"question_audio\":\"domain_10561\",\"subject\":\"domain\",\"template_id\":\"domain_4171\",\"type\":\"ftb\",\"feedback\":\"\",\"qlevel\":\"EASY\",\"title\":\"Story5Q1\",\"question_image\":\"domain_10624\",\"name\":\"Story5Q1\",\"domain\":\"literacy\",\"max_score\":1,\"question\":\"\",\"template\":\"org.ekstep.ftb.barakhadi\",\"answer\":{\"ans1\":{\"value\":\"रोटी\",\"score\":1}},\"code\":\"org.ekstep.assessmentitem.domain_4533\",\"lastUpdatedOn\":\"2016-06-15T09:40:54.289+0000\",\"concepts\":[{\"identifier\":\"LO1\",\"name\":\"Receptive Vocabulary\",\"objectType\":\"Concept\",\"relation\":\"associatedTo\",\"description\":null,\"index\":null}],\"createdOn\":\"2016-05-23T10:09:03.976+0000\",\"lastUpdatedBy\":\"323\",\"used_for\":\"worksheet\",\"owner\":\"323\",\"gradeLevel\":[\"Other\"],\"language\":[\"Hindi\"],\"identifier\":\"domain_4533\",\"media\":[{\"id\":\"domain_10624\",\"type\":\"image\",\"src\":\"https://ekstep-public.s3-ap-southeast-1.amazonaws.com/content/roti_323_1465971255_1465971255545.jpg\",\"asset_id\":\"domain_10624\",\"preload\":true},{\"id\":\"domain_10561\",\"type\":\"audio\",\"src\":\"https://ekstep-public.s3-ap-southeast-1.amazonaws.com/content/story11q4a1_141_1465970351_1465970351272.mp3\",\"asset_id\":\"domain_10561\",\"preload\":true}]}]},\"SET_OBJECT_TYPE_KEY\":\"AssessmentItem\",\"shuffle\":false,\"lastUpdatedBy\":\"323\",\"owner\":\"Ilimi\",\"used_for\":\"assessment\",\"max_score\":1,\"gradeLevel\":[\"Grade 1\"],\"item_sets\":[{\"id\":\"domain_11824\",\"count\":1}],\"language\":[\"English\"],\"identifier\":\"domain_11824\"}},{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"template\":[{\"text\":[{\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":70,\"model\":\"item.title\",\"w\":80,\"x\":10,\"y\":6,\"z-index\":101},{\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":100,\"h\":15,\"id\":\"newText\",\"model\":\"item.ans1\",\"valign\":\"middle\",\"w\":35,\"x\":58,\"y\":68,\"z-index\":100}],\"shape\":[{\"event\":{\"type\":\"click\"},\"h\":15,\"hitArea\":true,\"opacity\":1,\"w\":80,\"x\":10,\"y\":6,\"z-index\":99},{\"event\":{\"action\":{\"asset\":\"bKeyboard\",\"command\":\"custom\",\"id\":\"newText\",\"invoke\":\"switchTarget\",\"type\":\"command\"},\"type\":\"click\"},\"h\":15,\"hitArea\":true,\"stroke\":\"black\",\"stroke-width\":5,\"w\":35,\"x\":58,\"y\":67,\"z-index\":99},{\"event\":{\"action\":{\"asset_model\":\"item.question_audio\",\"command\":\"play\",\"type\":\"command\"},\"type\":\"click\"},\"h\":40,\"hitArea\":true,\"stroke-width\":5,\"w\":20,\"x\":10,\"y\":58,\"z-index\":99}],\"keyboard\":{\"id\":\"bKeyboard\",\"keys\":\"item.keys\",\"limit\":10,\"target\":\"newText\",\"type\":\"custom\",\"x\":5,\"y\":15},\"g\":{\"image\":{\"h\":75,\"model\":\"item.question_image\",\"w\":100,\"x\":0,\"y\":0,\"z-index\":102},\"text\":{\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":\"2em\",\"h\":25,\"model\":\"item.question\",\"valign\":\"middle\",\"w\":100,\"x\":0,\"y\":75,\"z-index\":103},\"h\":40,\"w\":20,\"x\":10,\"y\":58},\"id\":\"org.ekstep.ftb.barakhadi\"},{\"text\":{\"event\":{\"action\":[{\"command\":\"stop\",\"sound\":true,\"type\":\"command\"},{\"asset_model\":\"item.question_audio\",\"command\":\"play\",\"type\":\"command\"}],\"type\":\"click\"},\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":\"2em\",\"lineHeight\":2,\"model\":\"item.question\",\"valign\":\"middle\",\"w\":100,\"x\":0,\"y\":15},\"shape\":{\"event\":{\"action\":[{\"command\":\"stop\",\"sound\":true,\"type\":\"command\"},{\"asset_model\":\"item.question_audio\",\"command\":\"play\",\"type\":\"command\"}],\"type\":\"click\"},\"h\":10,\"hitArea\":true,\"type\":\"rect\",\"w\":100,\"x\":0,\"y\":15},\"mcq\":{\"options\":{\"shape\":{\"h\":100,\"stroke\":\"black\",\"type\":\"roundrect\",\"w\":100,\"x\":0,\"y\":0},\"text\":{\"align\":\"center\",\"font\":\"Verdana\",\"fontsize\":\"1.6em\",\"h\":100,\"model\":\"option.value.text\",\"valign\":\"middle\",\"w\":100,\"x\":0,\"y\":0},\"event\":{\"action\":[{\"command\":\"stop\",\"sound\":true,\"type\":\"command\"},{\"asset_model\":\"option.value.audio\",\"command\":\"play\",\"type\":\"command\"}],\"type\":\"click\"},\"cols\":4,\"h\":30,\"highlight\":\"yellow\",\"layout\":\"table\",\"marginX\":5,\"marginY\":5,\"options\":\"options\",\"w\":97,\"x\":1.5,\"y\":50},\"model\":\"item\",\"multi_select\":false,\"shadow\":\"#56B1F7\"},\"id\":\"org.ekstep.mcq.ta\"},{\"text\":{\"model\":\"item.title\",\"x\":9,\"y\":7,\"w\":86,\"h\":4,\"font\":\"Georgia\",\"fontsize\":42},\"mcq\":{\"options\":{\"layout\":\"table\",\"x\":20,\"y\":15,\"w\":70,\"h\":85,\"cols\":2,\"marginX\":10,\"marginY\":5,\"options\":\"options\"},\"multi_select\":false,\"model\":\"item\"},\"g\":{\"image\":{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"stop\",\"asset_model\":\"item.hints[0].asset\"},{\"type\":\"command\",\"command\":\"toggleShow\",\"asset\":\"hint\"}],\"type\":\"click\"},\"asset\":\"speech_bubble\",\"x\":0,\"y\":0,\"w\":100,\"h\":100},\"text\":[{\"x\":10,\"y\":20,\"w\":80,\"h\":80,\"font\":\"Georgia\",\"weight\":\"bold\",\"fontsize\":150,\"__text\":\"Hint\"},{\"x\":10,\"y\":40,\"w\":80,\"h\":80,\"font\":\"Georgia\",\"fontsize\":120,\"model\":\"item.hints[1].asset\"}],\"x\":9,\"y\":17,\"w\":20,\"h\":20,\"id\":\"hint\",\"visible\":false},\"image\":{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"togglePlay\",\"asset_model\":\"item.hints[0].asset\"},{\"type\":\"command\",\"command\":\"toggleShow\",\"asset\":\"hint\"}],\"type\":\"click\"},\"asset\":\"icon_hint\",\"x\":5,\"y\":35},\"id\":\"mcq_template_1\"},{\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"show\",\"asset\":\"retryDialog\"},\"type\":\"click\"},\"asset\":\"popupTint\",\"x\":-100,\"y\":-150,\"w\":550,\"h\":600,\"visible\":true,\"id\":\"popup-Tint\"},{\"asset\":\"retryBg\",\"x\":0,\"y\":0,\"w\":150,\"h\":150,\"visible\":true,\"id\":\"right\"}],\"shape\":[{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"hide\",\"asset\":\"retryDialog\"},{\"type\":\"command\",\"command\":\"SHOWHTMLELEMENTS\",\"asset\":\"retry\"}],\"type\":\"click\"},\"type\":\"roundrect\",\"x\":72,\"y\":25,\"w\":50,\"h\":65,\"visible\":true,\"id\":\"retry\",\"hitArea\":true},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"type\":\"roundrect\",\"x\":110,\"y\":100,\"w\":25,\"h\":35,\"visible\":true,\"id\":\"continue\",\"hitArea\":true}],\"id\":\"retry\"},{\"g\":{\"image\":[{\"asset\":\"popupTint\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"visible\":true,\"id\":\"popup-Tint\"}],\"text\":[{\"x\":25,\"y\":25,\"w\":50,\"h\":9,\"visible\":true,\"editable\":true,\"model\":\"word.lemma\",\"weight\":\"normal\",\"font\":\"helvetica\",\"color\":\"rgb(0,0,0)\",\"fontstyle\":\"\",\"fontsize\":75,\"align\":\"left\",\"z-index\":1,\"id\":\"lemma\"},{\"x\":25,\"y\":35,\"w\":50,\"h\":40,\"visible\":true,\"editable\":true,\"model\":\"word.gloss\",\"weight\":\"normal\",\"font\":\"helvetica\",\"color\":\"rgb(0,0,0)\",\"fontstyle\":\"\",\"fontsize\":43,\"align\":\"left\",\"z-index\":2,\"id\":\"gloss\"}],\"shape\":[{\"x\":20,\"y\":20,\"w\":60,\"h\":60,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":10,\"opacity\":1,\"fill\":\"#45b3a5\",\"stroke-width\":1,\"z-index\":0,\"id\":\"textBg\"}],\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"SHOWHTMLELEMENTS\",\"asset\":\"textBg\"},{\"type\":\"command\",\"command\":\"hide\",\"parent\":true}],\"type\":\"click\"}},\"id\":\"infoTemplate\"},{\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"show\",\"asset\":\"\"},\"type\":\"click\"},\"asset\":\"popupTint\",\"x\":-100,\"y\":-150,\"w\":550,\"h\":600,\"visible\":true,\"id\":\"popup-Tint\"},{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500}],\"type\":\"click\"},\"asset\":\"goodjobBg\",\"x\":0,\"y\":0,\"w\":150,\"h\":150,\"visible\":true,\"id\":\"continue\"}],\"id\":\"goodjob\"}],\"stage\":[{\"id\":\"recordAudio7\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"previousScreen\",\"value\":\"recordAudio6\"},{\"name\":\"nextScreen\",\"value\":\"recordAudio8\"},{\"name\":\"next\",\"value\":\"scene7b2e5492-b134-4945-9cde-803a094fb430\"},{\"name\":\"previous\",\"value\":\"recordAudio6\"}],\"events\":{\"event\":[{\"action\":[{\"asset\":\"startrec\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"stoprec\",\"command\":\"show\",\"type\":\"command\"},{\"asset\":\"mover\",\"command\":\"show\",\"type\":\"command\"},{\"tween\":{\"to\":[{\"duration\":0,\"ease\":\"linear\",\"__cdata\":{\"x\":47,\"y\":45}},{\"duration\":1000,\"ease\":\"sineInOut\",\"__cdata\":{\"x\":62,\"y\":45}},{\"duration\":1000,\"ease\":\"sineInOut\",\"__cdata\":{\"x\":47,\"y\":45}}],\"id\":\"mover\",\"loop\":true},\"asset\":\"mover\",\"type\":\"animation\"}],\"type\":\"rec_started\"},{\"action\":[{\"asset\":\"stoprec\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"mover\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"playback\",\"command\":\"show\",\"type\":\"command\"}],\"type\":\"rec_stopped\"},{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_4002\",\"loop\":1},\"type\":\"enter\"},{\"action\":{\"type\":\"command\",\"command\":\"stop\",\"asset\":\"domain_4002\",\"loop\":1},\"type\":\"exit\"}]},\"image\":[{\"x\":16.38888888888889,\"y\":20.22222222222222,\"w\":69.44444444444444,\"h\":77.77777777777779,\"visible\":true,\"editable\":true,\"asset\":\"domain_3126\",\"z-index\":0},{\"x\":45,\"y\":80,\"w\":10,\"h\":18,\"visible\":true,\"editable\":true,\"asset\":\"mic\",\"event\":{\"action\":{\"asset\":\"recorder\",\"command\":\"toggleShow\",\"type\":\"command\"},\"type\":\"click\"},\"z-index\":6},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500},\"type\":\"click\"},\"asset\":\"next\",\"x\":93,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"next\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100}],\"text\":[{\"x\":25.27777777777778,\"y\":6.222222222222222,\"w\":71.11111111111111,\"h\":9.322222222222223,\"visible\":true,\"editable\":true,\"__text\":\"बिल्लियाँ देखती ही रह गई |\",\"weight\":\"normal\",\"font\":\"Helvetica\",\"color\":\"rgb(0,0,0)\",\"fontstyle\":\"\",\"fontsize\":85,\"lineHeight\":1.3,\"align\":\"left\",\"z-index\":7}],\"shape\":[],\"hotspot\":[{\"x\":25.208333333333332,\"y\":4.888888888888889,\"w\":14.722222222222223,\"h\":8.444444444444445,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7637\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":1},{\"x\":40.97222222222222,\"y\":5.111111111111112,\"w\":10.277777777777777,\"h\":8,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7638\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":2},{\"x\":52.361111111111114,\"y\":5.111111111111112,\"w\":3.75,\"h\":8.666666666666668,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7639\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":3},{\"x\":57.36111111111111,\"y\":7.333333333333333,\"w\":4.861111111111112,\"h\":6.888888888888889,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7640\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":4},{\"x\":63.47222222222222,\"y\":4.888888888888889,\"w\":5.555555555555555,\"h\":9.777777777777779,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7641\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":5}],\"embed\":[],\"div\":[],\"audio\":[{\"asset\":\"domain_4002\"},{\"asset\":\"domain_7637\"},{\"asset\":\"domain_7638\"},{\"asset\":\"domain_7639\"},{\"asset\":\"domain_7640\"},{\"asset\":\"domain_7641\"}],\"scribble\":[],\"g\":[{\"g\":[{\"image\":{\"event\":{\"action\":{\"asset\":\"recorder\",\"command\":\"hide\",\"type\":\"command\"},\"type\":\"click\"},\"asset\":\"blacktint\",\"h\":100,\"w\":100,\"x\":0,\"y\":0},\"h\":100,\"w\":100,\"x\":0,\"y\":0},{\"shape\":[{\"fill\":\"gray\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":36,\"y\":37},{\"event\":{\"action\":{\"asset\":\"startrec\",\"command\":\"startRecord\",\"failure\":\"rec_start_fail\",\"success\":\"rec_started\",\"type\":\"command\"},\"type\":\"click\"},\"fill\":\"#34e941\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":35,\"y\":35}],\"image\":{\"asset\":\"record\",\"h\":10,\"w\":6,\"x\":38,\"y\":40},\"text\":{\"color\":\"darkgreen\",\"font\":\"sans-serif\",\"fontsize\":50,\"h\":5,\"w\":20,\"x\":47,\"y\":43,\"__text\":\"Start\"},\"h\":100,\"id\":\"startrec\",\"w\":100,\"x\":0,\"y\":0},{\"shape\":[{\"fill\":\"gray\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":36,\"y\":37},{\"event\":{\"action\":{\"asset\":\"stoprec\",\"command\":\"stopRecord\",\"failure\":\"rec_stop_failed\",\"success\":\"rec_stopped\",\"type\":\"command\"},\"type\":\"click\"},\"fill\":\"#e93441\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":35,\"y\":35},{\"fill\":\"#e98888\",\"h\":2,\"id\":\"mover\",\"type\":\"circle\",\"visible\":false,\"w\":2,\"x\":47,\"y\":45}],\"image\":{\"asset\":\"stop_record\",\"h\":10,\"w\":6,\"x\":38,\"y\":40},\"h\":100,\"id\":\"stoprec\",\"visible\":false,\"w\":100,\"x\":0,\"y\":0},{\"shape\":[{\"fill\":\"gray\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":36,\"y\":37},{\"event\":{\"action\":[{\"asset\":\"current_rec\",\"command\":\"TOGGLEPLAY\",\"type\":\"command\"},{\"asset\":\"recorder\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"playback\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"startrec\",\"command\":\"show\",\"type\":\"command\"}],\"type\":\"click\"},\"fill\":\"#34e941\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":35,\"y\":35}],\"image\":{\"asset\":\"icon_sound\",\"h\":10,\"id\":\"story_audio_button\",\"w\":6,\"x\":38,\"y\":40},\"text\":{\"color\":\"darkgreen\",\"font\":\"sans-serif\",\"fontsize\":50,\"h\":5,\"w\":20,\"x\":47,\"y\":43,\"__text\":\"Play\"},\"h\":100,\"id\":\"playback\",\"visible\":false,\"w\":100,\"x\":0,\"y\":0}],\"h\":100,\"id\":\"recorder\",\"z-index\":200,\"visible\":false,\"w\":100,\"x\":0,\"y\":0}],\"appEvents\":{\"list\":\"rec_started,rec_start_fail,rec_stopped,rec_stop_failed\"}},{\"id\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"next\",\"value\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\"},{\"name\":\"previous\",\"value\":\"scene5de719ed-207c-4f01-a7e8-e1eb4abf77ed\"}],\"events\":{\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_11825\",\"loop\":1},\"type\":\"enter\"},{\"action\":{\"type\":\"command\",\"command\":\"stop\",\"asset\":\"domain_11825\",\"loop\":1},\"type\":\"exit\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"retry_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"retryDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut \",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"retryShowAnim\"},\"type\":\"animation\",\"asset\":\"retryDialog\"}],\"type\":\"wrong_response\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"goodjob_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"goodjobDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut\",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"gjShowAnim\"},\"type\":\"animation\",\"asset\":\"goodjobDialog\"}],\"type\":\"correct_response\"}]},\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500},\"type\":\"click\"},\"asset\":\"next\",\"x\":93,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"next\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"eval\",\"asset\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\",\"success\":\"correct_response\",\"failure\":\"wrong_response\"},{\"type\":\"command\",\"command\":\"HIDEHTMLELEMENTS\",\"asset\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\"}],\"type\":\"click\"},\"asset\":\"validate\",\"x\":91,\"y\":15,\"h\":15}],\"text\":[],\"shape\":[],\"hotspot\":[],\"embed\":[],\"div\":[],\"audio\":[{\"asset\":\"domain_11825\"}],\"scribble\":[],\"g\":[{\"embed\":{\"template\":\"item\",\"var-item\":\"item\"},\"x\":10,\"y\":0,\"w\":80,\"h\":90},{\"embed\":{\"template-name\":\"retry\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"retryDialog\",\"visible\":false},{\"embed\":{\"template-name\":\"goodjob\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"goodjobDialog\",\"visible\":false}],\"iterate\":\"assessment_185d692a-1841-4a35-96c1-67cebcde42a2\",\"preload\":true,\"var\":\"item\",\"appEvents\":{\"list\":\"next_item,correct_response,wrong_response\"}},{\"id\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"next\",\"value\":\"endScreen\"},{\"name\":\"previous\",\"value\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\"}],\"events\":{\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_8313\",\"loop\":1},\"type\":\"enter\"},{\"action\":{\"type\":\"command\",\"command\":\"stop\",\"asset\":\"domain_8313\",\"loop\":1},\"type\":\"exit\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"retry_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"retryDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut \",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"retryShowAnim\"},\"type\":\"animation\",\"asset\":\"retryDialog\"}],\"type\":\"wrong_response\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"goodjob_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"goodjobDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut\",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"gjShowAnim\"},\"type\":\"animation\",\"asset\":\"goodjobDialog\"}],\"type\":\"correct_response\"}]},\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500},\"type\":\"click\"},\"asset\":\"next\",\"x\":93,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"next\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"eval\",\"asset\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\",\"success\":\"correct_response\",\"failure\":\"wrong_response\"},{\"type\":\"command\",\"command\":\"HIDEHTMLELEMENTS\",\"asset\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\"}],\"type\":\"click\"},\"asset\":\"validate\",\"x\":91,\"y\":15,\"h\":15}],\"text\":[],\"shape\":[],\"hotspot\":[],\"embed\":[],\"div\":[],\"audio\":[{\"asset\":\"domain_8313\"}],\"scribble\":[],\"g\":[{\"embed\":{\"template\":\"item\",\"var-item\":\"item\"},\"x\":10,\"y\":0,\"w\":80,\"h\":90},{\"embed\":{\"template-name\":\"retry\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"retryDialog\",\"visible\":false},{\"embed\":{\"template-name\":\"goodjob\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"goodjobDialog\",\"visible\":false}],\"iterate\":\"assessment_a3cc8a52-833a-4a9d-b6e6-3be6b266b70f\",\"preload\":true,\"var\":\"item\",\"appEvents\":{\"list\":\"next_item,correct_response,wrong_response\"}},{\"id\":\"endScreen\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"previousScreen\",\"value\":\"recordAudio9\"},{\"name\":\"nextScreen\",\"value\":\"splash\"},{\"name\":\"previous\",\"value\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\"}],\"events\":{\"event\":[]},\"image\":[{\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"visible\":true,\"editable\":true,\"asset\":\"endScreen\",\"z-index\":0},{\"x\":40,\"y\":50,\"w\":8,\"h\":13,\"visible\":false,\"editable\":true,\"asset\":\"previous\",\"event\":{\"action\":{\"asset\":\"theme\",\"command\":\"transitionTo\",\"direction\":\"right\",\"duration\":600,\"ease\":\"linear\",\"effect\":\"scroll\",\"param\":\"previousScreen\",\"type\":\"command\"},\"type\":\"click\"},\"z-index\":1},{\"x\":50,\"y\":50,\"w\":8,\"h\":13,\"visible\":false,\"editable\":true,\"asset\":\"next\",\"event\":{\"action\":{\"asset\":\"theme\",\"command\":\"transitionTo\",\"direction\":\"right\",\"duration\":600,\"ease\":\"linear\",\"effect\":\"scroll\",\"param\":\"nextScreen\",\"type\":\"command\"},\"type\":\"click\"},\"z-index\":2},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100}],\"text\":[],\"shape\":[],\"hotspot\":[],\"embed\":[],\"div\":[],\"audio\":[],\"scribble\":[],\"g\":[]}]}}")) - ) - val result: Map[String, AnyRef] = ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - result.contains("artifactUrl") shouldBe true - result.get("artifactUrl") shouldNot be(null) - result.contains("cloudStorageKey") shouldBe true - result.get("cloudStorageKey") shouldNot be(null) - } - -} diff --git a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/spec/ContentPublishStreamTaskSpec.scala b/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/spec/ContentPublishStreamTaskSpec.scala deleted file mode 100644 index d02e89d7a..000000000 --- a/publish-pipeline/content-publish/src/test/scala/org/sunbird/job/spec/ContentPublishStreamTaskSpec.scala +++ /dev/null @@ -1,119 +0,0 @@ -package org.sunbird.job.spec - -import com.google.gson.Gson -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.anyString -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.content.function.CollectionPublishFunction -import org.sunbird.job.content.publish.domain.Event -import org.sunbird.job.content.task.{ContentPublishConfig, ContentPublishStreamTask} -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{ExtDataConfig, ObjectData} -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.text.SimpleDateFormat -import java.util -import java.util.Date - -class ContentPublishStreamTaskSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val jobConfig: ContentPublishConfig = new ContentPublishConfig(config) - var definitionCache = new DefinitionCache() - implicit val definition: ObjectDefinition = definitionCache.getDefinition("Collection", jobConfig.schemaSupportVersionMap.getOrElse("collection", "1.0").asInstanceOf[String], jobConfig.definitionBasePath) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.hierarchyKeyspaceName, jobConfig.hierarchyTableName, definition.getExternalPrimaryKey, definition.getExternalProps) - - val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - val publishConfig: PublishConfig = new PublishConfig(config, "") - val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - flinkCluster.after() - } - - def initialize(): Unit = { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new ContentPublishEventSource) - } - - def getTimeStamp: String = { - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - sdf.format(new Date()) - } - - - "fetchDialListForContextUpdate" should "fetch the list of added and removed QR codes" in { - val nodeObj = new ObjectData("do_21354027142511820812318.img", Map("objectType" -> "Collection", "identifier" -> "do_21354027142511820812318", "name" -> "DialCodeHierarchy", "lastPublishedOn" -> getTimeStamp, "lastUpdatedOn" -> getTimeStamp, "status" -> "Draft", "pkgVersion" -> 1.asInstanceOf[Number], "versionKey" -> "1652871771396", "channel" -> "0126825293972439041", "contentType" -> "TextBook"), Some(Map()), Some(Map())) - val DIALListMap = new CollectionPublishFunction(jobConfig, mockHttpUtil).fetchDialListForContextUpdate(nodeObj)(mockNeo4JUtil, cassandraUtil, readerConfig, jobConfig) - assert(DIALListMap.nonEmpty) - - val postProcessEvent = new CollectionPublishFunction(jobConfig, mockHttpUtil).getPostProcessEvent(nodeObj, DIALListMap) - assert(postProcessEvent.nonEmpty) - } - - ignore should " publish the content " in { - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(new util.HashMap[String, AnyRef]) - initialize - new ContentPublishStreamTask(jobConfig, mockKafkaUtil, mockHttpUtil).process() - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.contentPublishEventCount}").getValue() should be(1) - } -} - -private class ContentPublishEventSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(jsonToEvent(EventFixture.PDF_EVENT1)) - } - - override def cancel() = {} - - def jsonToEvent(json: String): Event = { - val gson = new Gson() - val data = gson.fromJson(json, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]] - val metadataMap = data.get("edata").asInstanceOf[util.Map[String, Any]].get("metadata").asInstanceOf[util.Map[String, Any]] - metadataMap.put("pkgVersion", metadataMap.get("pkgVersion").asInstanceOf[Double].toInt) - new Event(data, 0, 10) - } -} diff --git a/publish-pipeline/live-node-publisher/pom.xml b/publish-pipeline/live-node-publisher/pom.xml deleted file mode 100644 index 19938fd90..000000000 --- a/publish-pipeline/live-node-publisher/pom.xml +++ /dev/null @@ -1,207 +0,0 @@ - - - - publish-pipeline - org.sunbird - 1.0 - - 4.0.0 - - live-node-publisher - 1.0.0 - jar - live-node-publisher - - Live node publisher job - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - publish-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - jackson-module-scala_${scala.version} - com.fasterxml.jackson.module - - - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.sunbird.job.livenodepublisher.task.LiveNodePublisherStreamTask - - - - reference.conf - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - live-node-publisher-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/main/resources/live-node-publisher.conf b/publish-pipeline/live-node-publisher/src/main/resources/live-node-publisher.conf deleted file mode 100644 index 5c7c1b7af..000000000 --- a/publish-pipeline/live-node-publisher/src/main/resources/live-node-publisher.conf +++ /dev/null @@ -1,176 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.republish.job.request" - live_video_stream.topic = "sunbirddev.live.video.stream.request" - error.topic = "sunbirddev.learning.events.failed" - skipped.topic = "sunbirddev.learning.events.skipped" - groupId = "local-content-publish-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - router.parallelism = 1 -} - -redis { - database { - contentCache.id = 0 - } -} - -content { - bundleLocation = "/tmp/contentBundle" - isECARExtractionEnabled = true - retry_asset_download_count = 1 - keyspace = "dev_content_store" - table = "content_data" - tmp_file_location = "/tmp" - objectType = ["Content", "ContentImage","Collection","CollectionImage"] - mimeType = ["application/pdf", - "application/vnd.ekstep.ecml-archive", - "application/vnd.ekstep.html-archive", - "application/vnd.android.package-archive", - "application/vnd.ekstep.content-archive", - "application/epub", - "application/msword", - "application/vnd.ekstep.h5p-archive", - "video/webm", - "video/mp4", - "application/vnd.ekstep.content-collection", - "video/quicktime", - "application/octet-stream", - "application/json", - "application/javascript", - "application/xml", - "text/plain", - "text/html", - "text/javascript", - "text/xml", - "text/css", - "image/jpeg", - "image/jpg", - "image/png", - "image/tiff", - "image/bmp", - "image/gif", - "image/svg+xml", - "image/x-quicktime", - "video/avi", - "video/mpeg", - "video/quicktime", - "video/3gpp", - "video/mp4", - "video/ogg", - "video/webm", - "video/msvideo", - "video/x-msvideo", - "video/x-qtc", - "video/x-mpeg", - "audio/mp3", - "audio/mp4", - "audio/mpeg", - "audio/ogg", - "audio/webm", - "audio/x-wav", - "audio/wav", - "audio/mpeg3", - "audio/x-mpeg-3", - "audio/vorbis", - "application/x-font-ttf", - "application/vnd.ekstep.plugin-archive", - "video/x-youtube", - "video/youtube", - "text/x-url"] - asset_download_duration = "60 seconds" - - stream { - enabled = true - mimeType = ["video/mp4", "video/webm"] - } - artifact.size.for_online=209715200 - - downloadFiles { - spine = ["appIcon"] - full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"] - } - - nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions"] - -} - -hierarchy { - keyspace = "hierarchy_store" - table = "content_hierarchy" -} - -cloud_storage { - folder { - content = "content" - artifact = "artifact" - } -} - -service { - print.basePath = "http://11.2.6.6/print" -} - - -contentTypeToPrimaryCategory { - ClassroomTeachingVideo: "Explanation Content" - ConceptMap: "Learning Resource" - Course: "Course" - CuriosityQuestionSet: "Practice Question Set" - eTextBook: "eTextbook" - Event: "Event" - EventSet: "Event Set" - ExperientialResource: "Learning Resource" - ExplanationResource: "Explanation Content" - ExplanationVideo: "Explanation Content" - FocusSpot: "Teacher Resource" - LearningOutcomeDefinition: "Teacher Resource" - MarkingSchemeRubric: "Teacher Resource" - PedagogyFlow: "Teacher Resource" - PracticeQuestionSet: "Practice Question Set" - PracticeResource: "Practice Question Set" - SelfAssess: "Course Assessment" - TeachingMethod: "Teacher Resource" - TextBook: "Digital Textbook" - Collection: "Content Playlist" - ExplanationReadingMaterial: "Learning Resource" - LearningActivity: "Learning Resource" - LessonPlan: "Content Playlist" - LessonPlanResource: "Teacher Resource" - PreviousBoardExamPapers: "Learning Resource" - TVLesson: "Explanation Content" - OnboardingResource: "Learning Resource" - ReadingMaterial: "Learning Resource" - Template: "Template" - Asset: "Asset" - Plugin: "Plugin" - LessonPlanUnit: "Lesson Plan Unit" - CourseUnit: "Course Unit" - TextBookUnit: "Textbook Unit" - Asset: "Certificate Template" -} - -max_allowed_content_name = 120 - -service.search.basePath = "http://11.2.6.6/search" - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "CONTENT_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"] -cloudstorage.metadacloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] - -cloud_storage_type="" -cloud_storage_key="" -cloud_storage_secret="" -cloud_storage_container="" -cloud_storage_endpoint="" \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/main/resources/log4j.properties b/publish-pipeline/live-node-publisher/src/main/resources/log4j.properties deleted file mode 100644 index 6eb0a07a7..000000000 --- a/publish-pipeline/live-node-publisher/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=-live-node-publisher.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala deleted file mode 100644 index b776e4206..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala +++ /dev/null @@ -1,218 +0,0 @@ -package org.sunbird.job.livenodepublisher.function - -import akka.dispatch.ExecutionContexts -import com.google.gson.reflect.TypeToken -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.neo4j.driver.v1.exceptions.ClientException -import org.slf4j.LoggerFactory -import org.sunbird.job.cache.{DataCache, RedisConnect} -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.exception.{InvalidInputException, ServerException} -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.livenodepublisher.publish.domain.Event -import org.sunbird.job.livenodepublisher.publish.helpers.LiveCollectionPublisher -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util._ -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type -import java.util -import scala.concurrent.ExecutionContext -import scala.collection.JavaConverters._ - -class LiveCollectionPublishFunction(config: LiveNodePublisherConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var esUtil: ElasticSearchUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var definitionCache: DefinitionCache = null, - @transient var definitionConfig: DefinitionConfig = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with LiveCollectionPublisher with FailedEventHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[LiveCollectionPublishFunction]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - private var cache: DataCache = _ - private val COLLECTION_CACHE_KEY_PREFIX = "hierarchy_" - private val COLLECTION_CACHE_KEY_SUFFIX = ":leafnodes" - private val PUBLISHED_STATUS_LIST = List("Live", "Unlisted") - @transient var ec: ExecutionContext = _ - private val pkgTypes = List(EcarPackageType.SPINE, EcarPackageType.ONLINE) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType) - cloudStorageUtil = new CloudStorageUtil(config) - ec = ExecutionContexts.global - definitionCache = new DefinitionCache() - definitionConfig = DefinitionConfig(config.schemaSupportVersionMap, config.definitionBasePath) - cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) - cache.init() - } - - override def close(): Unit = { - super.close() - cassandraUtil.close() - cache.close() - } - - override def metricsList(): List[String] = { - List(config.collectionPublishEventCount, config.collectionPublishSuccessEventCount, config.collectionPublishFailedEventCount, config.skippedEventCount, config.collectionPostPublishProcessEventCount) - } - - override def processElement(data: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - val definition: ObjectDefinition = definitionCache.getDefinition(data.objectType, config.schemaSupportVersionMap.getOrElse(data.objectType.toLowerCase(), "1.0").asInstanceOf[String], config.definitionBasePath) - val readerConfig = ExtDataConfig(config.hierarchyKeyspaceName, config.hierarchyTableName, definition.getExternalPrimaryKey(), definition.getExternalProps()) - logger.info("Collection publishing started for : " + data.identifier) - metrics.incCounter(config.collectionPublishEventCount) - val obj: ObjectData = getObject(data.identifier, data.pkgVersion, data.mimeType, data.publishType, readerConfig)(neo4JUtil, cassandraUtil, config) - try { - if (obj.pkgVersion > data.pkgVersion || !PUBLISHED_STATUS_LIST.contains(obj.metadata.getOrElse("status", "").asInstanceOf[String])) { - metrics.incCounter(config.skippedEventCount) - logger.info(s"""Either object status is invalid OR Event pkgVersion is not greater than or equal to the obj.pkgVersion for : ${obj.identifier}""") - } else { - val childNodesMetadata: List[String] = obj.metadata.getOrElse("childNodes", new java.util.ArrayList()).asInstanceOf[java.util.List[String]].asScala.toList - val addedResources: List[String] = searchContents(data.identifier, childNodesMetadata.toArray, config, httpUtil) - val addedResourcesMigrationVersion: List[String] = searchContents(data.identifier, childNodesMetadata.toArray, config, httpUtil, true) - - val isCollectionShallowCopy = isContentShallowCopy(obj) - val shallowCopyOriginMigrationVersion: Double = if (isCollectionShallowCopy) { - val originId = obj.metadata.getOrElse("origin", "").asInstanceOf[String] - val originNodeMetadata = neo4JUtil.getNodeProperties(originId) - if (null != originNodeMetadata) { - originNodeMetadata.getOrDefault("migrationVersion", "0").toString.toDouble - } else 0 - } else 0 - - if (obj.pkgVersion > data.pkgVersion) { - metrics.incCounter(config.skippedEventCount) - logger.info(s"""pkgVersion should be greater than or equal to the obj.pkgVersion for : ${obj.identifier}""") - } - else if (isCollectionShallowCopy && shallowCopyOriginMigrationVersion != 1.1) { - pushSkipEvent(data, "Origin node is found to be not migrated", context)(metrics) - } - else if (addedResources.size != addedResourcesMigrationVersion.size) { - val errorMessageIdentifiers = addedResources.filter(rec => !addedResourcesMigrationVersion.contains(rec)).mkString(",") - pushSkipEvent(data, "Non migrated contents found: " + errorMessageIdentifiers, context)(metrics) - } else { - val updObj = new ObjectData(obj.identifier, obj.metadata ++ Map("lastPublishedBy" -> data.lastPublishedBy, "dialcodes" -> obj.metadata.getOrElse("dialcodes", null)), obj.extData, obj.hierarchy) - - // Pre-publish update -// updateProcessingNode(updObj)(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig) - - val updatedObj = if (isCollectionShallowCopy) updateOriginPkgVersion(updObj)(neo4JUtil) else updObj - - // Clear redis cache - cache.del(data.identifier) - cache.del(data.identifier + COLLECTION_CACHE_KEY_SUFFIX) - cache.del(COLLECTION_CACHE_KEY_PREFIX + data.identifier) - - val enrichedObjTemp = enrichObjectMetadata(updatedObj)(neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig) - val enrichedObj = enrichedObjTemp.getOrElse(updatedObj) - logger.info("CollectionPublishFunction:: Collection Object Enriched: " + enrichedObj.identifier) - val objWithEcar = getObjectWithEcar(enrichedObj, pkgTypes)(ec, neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig, httpUtil) - logger.info("CollectionPublishFunction:: ECAR generation completed for Collection Object: " + objWithEcar.identifier) - - val collRelationalMetadata = getRelationalMetadata(obj.identifier, readerConfig)(cassandraUtil).getOrElse(Map.empty[String, AnyRef]) - - val variantsJsonString = ScalaJsonUtil.serialize(objWithEcar.metadata("variants")) - val publishType = objWithEcar.getString("publish_type", "Public") - val successObj = new ObjectData(objWithEcar.identifier, objWithEcar.metadata + ("status" -> (if (publishType.equalsIgnoreCase("Unlisted")) "Unlisted" else "Live"), "variants" -> variantsJsonString, "identifier" -> objWithEcar.identifier), objWithEcar.extData, objWithEcar.hierarchy) - val children = successObj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - - // Collection - update and publish children - line 418 in PublishFinalizer - val updatedChildren = updateHierarchyMetadata(children, successObj.metadata, collRelationalMetadata)(config) - logger.info("CollectionPublishFunction:: Hierarchy Metadata updated for Collection Object: " + successObj.identifier + " || updatedChildren:: " + updatedChildren) - publishHierarchy(updatedChildren, successObj, readerConfig, config)(cassandraUtil) - - if (!isCollectionShallowCopy) syncNodes(successObj, updatedChildren, List.empty)(esUtil, neo4JUtil, cassandraUtil, readerConfig, definition, config) - - metrics.incCounter(config.collectionPublishSuccessEventCount) - logger.info("CollectionPublishFunction:: Collection publishing completed successfully for : " + data.identifier) - - saveOnSuccess(new ObjectData(objWithEcar.identifier, objWithEcar.metadata.-("children"), objWithEcar.extData, objWithEcar.hierarchy))(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig, config) - logger.info("CollectionPublishFunction:: Published Collection Object metadata saved successfully to graph DB: " + objWithEcar.identifier) - } - } - } catch { - case ex@(_: InvalidInputException | _: ClientException) => // ClientException - Invalid input exception. - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - val exMsg = if(ex.getMessage.length>2500) ex.getMessage.substring(0,2500) else ex.getMessage - pushFailedEvent(data, exMsg, null, context)(metrics) - logger.error(s"CollectionPublishFunction::Error while publishing collection :: ${data.partition} and Offset: ${data.offset}. Error : ${ex.getMessage}", ex) - case ex: Exception => - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - logger.error(s"CollectionPublishFunction::Error while processing message for Partition: ${data.partition} and Offset: ${data.offset}. Error : ${ex.getMessage}", ex) - throw ex - } - } - - private def pushFailedEvent(event: Event, errorMessage: String, error: Throwable, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - val failedEvent = if (error == null) getFailedEvent(event.jobName, event.getMap(), errorMessage) else getFailedEvent(event.jobName, event.getMap(), error) - context.output(config.failedEventOutTag, failedEvent) - metrics.incCounter(config.collectionPublishFailedEventCount) - } - - private def pushSkipEvent(event: Event, skipMessage: String, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - val skipEvent = if(skipMessage.length>500) getFailedEvent(event.jobName, event.getMap(), skipMessage.substring(0,500)) else getFailedEvent(event.jobName, event.getMap(), skipMessage) - context.output(config.skippedEventOutTag, skipEvent) - metrics.incCounter(config.skippedEventCount) - } - - private def searchContents(collectionId: String, identifiers: Array[String], config: LiveNodePublisherConfig, httpUtil: HttpUtil, fetchMigratedVersion: Boolean = false): List[String] = { - try { - val migrationVersionList = new util.ArrayList[AnyRef] - migrationVersionList.add(1.asInstanceOf[Number]) - migrationVersionList.add(1.1.asInstanceOf[Number]) - - val reqMap = new java.util.HashMap[String, AnyRef]() { - put("request", new java.util.HashMap[String, AnyRef]() { - put("filters", new java.util.HashMap[String, AnyRef]() { - put("visibility", "Default") - put("identifier", identifiers) - if (fetchMigratedVersion) put("migrationVersion", migrationVersionList.toArray) - }) - put("fields", config.searchFields) - }) - } - - val requestUrl = s"${config.searchServiceBaseUrl}/v3/search" - logger.info("CollectionPublishFunction :: searchContent :: Search Content requestUrl: " + requestUrl) - logger.info("CollectionPublishFunction :: searchContent :: Search Content reqMap: " + reqMap) - val httpResponse = httpUtil.post(requestUrl, JSONUtil.serialize(reqMap)) - if (httpResponse.status == 200) { - val response = JSONUtil.deserialize[Map[String, AnyRef]](httpResponse.body) - val result = response.getOrElse("result", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - val contents = result.getOrElse("content", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - val count = result.getOrElse("count", 0).asInstanceOf[Int] - if (count > 0) { - contents.map(content => { - content.getOrElse("identifier", "").toString - }) - } else { - logger.info("CollectionPublishFunction :: searchContent :: Received 0 count while searching childNodes : ") - List.empty[String] - } - } else { - throw new ServerException("ERR_API_CALL", "Invalid Response received while searching childNodes : " + getErrorDetails(httpResponse)) - } - } catch { - case ex: Exception => throw new InvalidInputException("Exception while searching children data for collection:: " + collectionId + " || ex: " + ex.getMessage) - } - } - - private def getErrorDetails(httpResponse: HTTPResponse): String = { - val response = JSONUtil.deserialize[Map[String, AnyRef]](httpResponse.body) - if (null != response) " | Response Code :" + httpResponse.status + " | Result : " + response.getOrElse("result", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] + " | Error Message : " + response.getOrElse("params", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - else " | Null Response Received." - } - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveContentPublishFunction.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveContentPublishFunction.scala deleted file mode 100644 index 029a2d3a4..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveContentPublishFunction.scala +++ /dev/null @@ -1,152 +0,0 @@ -package org.sunbird.job.livenodepublisher.function - -import akka.dispatch.ExecutionContexts -import com.google.gson.reflect.TypeToken -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.neo4j.driver.v1.exceptions.ClientException -import org.slf4j.LoggerFactory -import org.sunbird.job.cache.{DataCache, RedisConnect} -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.livenodepublisher.publish.domain.Event -import org.sunbird.job.livenodepublisher.publish.helpers.{ExtractableMimeTypeHelper, LiveContentPublisher} -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type -import java.util.UUID -import scala.concurrent.ExecutionContext - -class LiveContentPublishFunction(config: LiveNodePublisherConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var definitionCache: DefinitionCache = null, - @transient var definitionConfig: DefinitionConfig = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with LiveContentPublisher with FailedEventHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[LiveContentPublishFunction]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - private var cache: DataCache = _ - private val readerConfig = ExtDataConfig(config.contentKeyspaceName, config.contentTableName) - private val PUBLISHED_STATUS_LIST = List("Live", "Unlisted") - - @transient var ec: ExecutionContext = _ - private val pkgTypes = List(EcarPackageType.FULL, EcarPackageType.SPINE) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - cloudStorageUtil = new CloudStorageUtil(config) - ec = ExecutionContexts.global - definitionCache = new DefinitionCache() - definitionConfig = DefinitionConfig(config.schemaSupportVersionMap, config.definitionBasePath) - cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) - cache.init() - } - - override def close(): Unit = { - super.close() - cassandraUtil.close() - cache.close() - } - - override def metricsList(): List[String] = { - List(config.contentPublishEventCount, config.contentPublishSuccessEventCount, config.contentPublishFailedEventCount, - config.videoStreamingGeneratorEventCount, config.skippedEventCount) - } - - override def processElement(data: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - logger.info("Content publishing started for : " + data.identifier) - metrics.incCounter(config.contentPublishEventCount) - val obj: ObjectData = getObject(data.identifier, data.pkgVersion, data.mimeType, data.publishType, readerConfig)(neo4JUtil, cassandraUtil, config) - try { - if (obj.pkgVersion > data.pkgVersion || !PUBLISHED_STATUS_LIST.contains(obj.metadata.getOrElse("status", "").asInstanceOf[String])) { - metrics.incCounter(config.skippedEventCount) - logger.info(s"""Either object status is invalid OR Event pkgVersion is not greater than or equal to the obj.pkgVersion for : ${obj.identifier}""") - } else { - val messages: List[String] = validate(obj, obj.identifier, config, validateMetadata) - if (messages.isEmpty) { -// // Pre-publish update -// updateProcessingNode(new ObjectData(obj.identifier, obj.metadata ++ Map("lastPublishedBy" -> data.lastPublishedBy), obj.extData, obj.hierarchy))(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig) - - val ecmlVerifiedObj = if (obj.mimeType.equalsIgnoreCase("application/vnd.ekstep.ecml-archive")) { - val ecarEnhancedObj = ExtractableMimeTypeHelper.processECMLBody(obj, config)(ec, cloudStorageUtil) - new ObjectData(obj.identifier, ecarEnhancedObj, obj.extData, obj.hierarchy) - } else obj - - // Clear redis cache - cache.del(data.identifier) - val enrichedObjTemp = enrichObjectMetadata(ecmlVerifiedObj)(neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig) - val enrichedObj = enrichedObjTemp.getOrElse(ecmlVerifiedObj) - val objWithEcar = getObjectWithEcar(enrichedObj, if (enrichedObj.getString("contentDisposition", "").equalsIgnoreCase("online-only")) List(EcarPackageType.SPINE) else pkgTypes)(ec, neo4JUtil, cloudStorageUtil, config, definitionCache, definitionConfig, httpUtil) - logger.info("Ecar generation done for Content: " + objWithEcar.identifier) - saveOnSuccess(objWithEcar)(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig, config) - pushStreamingUrlEvent(enrichedObj, context)(metrics) - - metrics.incCounter(config.contentPublishSuccessEventCount) - logger.info("Content publishing completed successfully for : " + data.identifier) - logger.info(s"""{ identifier: \"${data.identifier}\", mimetype: \"${data.mimeType}\", status: \"Success\"}""") - } else { - saveOnFailure(obj, messages, data.pkgVersion)(neo4JUtil) - val errorMessages = messages.mkString("; ") - pushFailedEvent(data, errorMessages, null, context)(metrics) - logger.info("Content publishing failed for : " + data.identifier) - logger.info(s"""{ identifier: \"${data.identifier}\", mimetype: \"${data.mimeType}\", status: \"Failed\"}""") - } - } - } catch { - case ex@(_: InvalidInputException | _: ClientException | _:java.lang.IllegalArgumentException) => // ClientException - Invalid input exception. - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - pushFailedEvent(data, null, ex, context)(metrics) - logger.error("Error while publishing content :: " + ex.getMessage) - logger.info(s"""{ identifier: \"${data.identifier}\", mimetype: \"${data.mimeType}\", status: \"Failed\"}""") - case ex: Exception => - ex.printStackTrace() - saveOnFailure(obj, List(ex.getMessage), data.pkgVersion)(neo4JUtil) - logger.error(s"""Error while processing message for Partition: ${data.partition} and Offset: ${data.offset}. Error : ${ex.getMessage}""", ex) - logger.info(s"""{ identifier: \"${data.identifier}\", mimetype: \"${data.mimeType}\", status: \"Failed\"}""") - throw ex - } - } - - private def pushStreamingUrlEvent(obj: ObjectData, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - if (config.isStreamingEnabled && config.streamableMimeType.contains(obj.mimeType)) { - val event = getStreamingEvent(obj) - context.output(config.generateVideoStreamingOutTag, event) - metrics.incCounter(config.videoStreamingGeneratorEventCount) - } - } - - def getStreamingEvent(obj: ObjectData): String = { - val ets = System.currentTimeMillis - val mid = s"""LP.$ets.${UUID.randomUUID}""" - val channelId = obj.getString("channel", "") - val ver = obj.getString("versionKey", "") - val artifactUrl = obj.getString("artifactUrl", "") - val contentType = obj.getString("contentType", "") - val status = obj.getString("status", "") - //TODO: deprecate using contentType in the event. - val event = s"""{"eid":"BE_JOB_REQUEST", "ets": $ets, "mid": "$mid", "actor": {"id": "Live Video Stream Generator", "type": "System"}, "context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"}, "channel":"$channelId","env":"${config.jobEnv}"},"object":{"ver":"$ver","id":"${obj.identifier}"},"edata": {"action":"live-video-stream-generate","iteration":1,"identifier":"${obj.identifier}","channel":"$channelId","artifactUrl":"$artifactUrl","mimeType":"${obj.mimeType}","contentType":"$contentType","pkgVersion":${obj.pkgVersion},"status":"$status"}}""".stripMargin - logger.info(s"Live Video Streaming Event for identifier ${obj.identifier} is : $event") - event - } - - - private def pushFailedEvent(event: Event, errorMessage: String, error: Throwable, context: ProcessFunction[Event, String]#Context)(implicit metrics: Metrics): Unit = { - val failedEvent = if (error == null) getFailedEvent(event.jobName, event.getMap(), errorMessage) else getFailedEvent(event.jobName, event.getMap(), error) - context.output(config.failedEventOutTag, failedEvent) - metrics.incCounter(config.contentPublishFailedEventCount) - } - - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LivePublishEventRouter.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LivePublishEventRouter.scala deleted file mode 100644 index b288d6172..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LivePublishEventRouter.scala +++ /dev/null @@ -1,55 +0,0 @@ -package org.sunbird.job.livenodepublisher.function - -import com.google.gson.reflect.TypeToken -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.livenodepublisher.publish.domain.Event -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type - -class LivePublishEventRouter(config: LiveNodePublisherConfig) extends BaseProcessFunction[Event, String](config) { - - - private[this] val logger = LoggerFactory.getLogger(classOf[LivePublishEventRouter]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def metricsList(): List[String] = { - List(config.skippedEventCount, config.totalEventsCount) - } - - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - logger.info("PublishEventRouter :: Event: " + event) - // Event validation - if (event.validEvent(config)) { - event.objectType match { - case "Content" => { - logger.info("PublishEventRouter :: Sending Content For Publish Having Identifier: " + event.identifier) - context.output(config.contentPublishOutTag, event) - } - case "Collection" => { - logger.info("PublishEventRouter :: Sending Collection For Publish Having Identifier: " + event.identifier) - context.output(config.collectionPublishOutTag, event) - } - case _ => { - metrics.incCounter(config.skippedEventCount) - logger.info("Invalid Object Type Received For Publish.| Identifier : " + event.identifier + " , objectType : " + event.objectType) - } - } - } else { - logger.warn("Event skipped for identifier: " + event.identifier + " objectType: " + event.objectType) - metrics.incCounter(config.skippedEventCount) - } - } -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/domain/Event.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/domain/Event.scala deleted file mode 100644 index dabe9ef80..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/domain/Event.scala +++ /dev/null @@ -1,40 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.domain.reader.JobRequest - -import java.util -import scala.collection.JavaConverters._ - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "live-node-publisher" - - def eData: Map[String, AnyRef] = readOrDefault("edata", new util.HashMap[String, AnyRef]()).asScala.toMap - - def action: String = readOrDefault[String]("edata.action", "") - - def mimeType: String = readOrDefault[String]("edata.metadata.mimeType", "") - - def identifier: String = readOrDefault[String]("edata.metadata.identifier", "") - - def objectType: String = readOrDefault[String]("edata.metadata.objectType", "") - - def contentType: String = readOrDefault[String]("edata.contentType", "") - - def publishType: String = readOrDefault[String]("edata.publish_type", "") - - def lastPublishedBy: String = readOrDefault[String]("edata.metadata.lastPublishedBy", "") - - def pkgVersion: Double = { - val pkgVersion: Number = readOrDefault[Number]("edata.metadata.pkgVersion", 0) - pkgVersion.doubleValue() - } - - def validEvent(config: LiveNodePublisherConfig): Boolean = { - ((StringUtils.equals("republish", action) && StringUtils.isNotBlank(identifier)) - && (config.supportedObjectType.contains(objectType) && config.supportedMimeType.contains(mimeType)) - && !StringUtils.equalsIgnoreCase("Asset", contentType)) - } -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/ECMLExtractor.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/ECMLExtractor.scala deleted file mode 100644 index 8d96c8578..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/ECMLExtractor.scala +++ /dev/null @@ -1,8 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers - -import org.sunbird.job.livenodepublisher.publish.processor.{BaseProcessor, MissingAssetValidatorProcessor} -import org.sunbird.job.util.CloudStorageUtil - -class ECMLExtractor(basePath: String, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil) extends BaseProcessor(basePath, identifier) with MissingAssetValidatorProcessor { - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/ExtractableMimeTypeHelper.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/ExtractableMimeTypeHelper.scala deleted file mode 100644 index 3a75fbd7b..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/ExtractableMimeTypeHelper.scala +++ /dev/null @@ -1,254 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers - -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.livenodepublisher.publish.processor.{JsonParser, Media, Plugin, XmlParser} -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.{CloudStorageUtil, FileUtils, Slug} -import org.xml.sax.{InputSource, SAXException} - -import java.io._ -import javax.xml.parsers.{DocumentBuilderFactory, ParserConfigurationException} -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext, Future} - -object ExtractableMimeTypeHelper { - - private[this] val logger = LoggerFactory.getLogger("ExtractableMimeTypeHelper") - private val extractablePackageExtensions = List(".zip", ".h5p", ".epub") - - def getCloudStoreURL(obj: ObjectData, cloudStorageUtil: CloudStorageUtil, config: LiveNodePublisherConfig): String = { - val path = getExtractionPath(obj, config, "latest") - cloudStorageUtil.getURI(path, Option.apply(config.extractableMimeTypes.contains(obj.mimeType))) - } - - private def getExtractionPath(obj: ObjectData, config: LiveNodePublisherConfig, suffix: String): String = { - obj.mimeType match { - case "application/vnd.ekstep.ecml-archive" => config.contentFolder + File.separator + "ecml" + File.separator + obj.identifier + "-" + suffix - case "application/vnd.ekstep.html-archive" => config.contentFolder + File.separator + "html" + File.separator + obj.identifier + "-" + suffix - case "application/vnd.ekstep.h5p-archive" => config.contentFolder + File.separator + "h5p" + File.separator + obj.identifier + "-" + suffix - case _ => "" - } - } - - def getBasePath(objectId: String, tempFileLocation: String): String = { - if (!StringUtils.isBlank(objectId)) tempFileLocation + File.separator + System.currentTimeMillis + "_temp" + File.separator + objectId else "" - } - - def copyExtractedContentPackage(obj: ObjectData, contentConfig: LiveNodePublisherConfig, extractionType: String, cloudStorageUtil: CloudStorageUtil): Unit = { - if (!isExtractedSnapshotExist(obj)) throw new InvalidInputException("Error! Snapshot Type Extraction doesn't Exists.") - val sourcePrefix = getExtractionPath(obj, contentConfig, "snapshot") - val destinationPrefix = getExtractionPath(obj, contentConfig, extractionType) - cloudStorageUtil.copyObjectsByPrefix(sourcePrefix, destinationPrefix, isFolder = true) - } - - private def isExtractedSnapshotExist(obj: ObjectData): Boolean = { - extractablePackageExtensions.exists(key => StringUtils.endsWithIgnoreCase(obj.getString("artifactUrl", null), key)) - } - - def processECMLBody(obj: ObjectData, config: LiveNodePublisherConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Map[String, AnyRef] = { - try { - val basePath = config.bundleLocation + "/" + System.currentTimeMillis + "_tmp" + "/" + obj.identifier - val ecmlBody = obj.extData.get.getOrElse("body", "").toString - val ecmlType: String = getECMLType(ecmlBody) - val ecrfObj: Plugin = getEcrfObject(ecmlType, ecmlBody) - - // localize assets - download assets to local base path (tmp folder) for validation - localizeAssets(obj.identifier, ecrfObj, basePath, config) - - // validate assets - val processedEcrf: Plugin = new ECMLExtractor(basePath, obj.identifier).process(ecrfObj) - - // getECMLString - val processedEcml: String = getEcmlStringFromEcrf(processedEcrf, ecmlType) - - // write ECML String to basePath - writeECMLFile(basePath, processedEcml, ecmlType) - - // create zip package - val zipFileName: String = basePath + File.separator + System.currentTimeMillis + "_" + Slug.makeSlug(obj.identifier) + ".zip" - FileUtils.createZipPackage(basePath, zipFileName) - - // upload zip file to blob and set artifactUrl - val result: Array[String] = uploadArtifactToCloud(new File(zipFileName), obj.identifier, None, config) - - // upload local extracted directory to blob - extractPackageInCloud(new File(zipFileName), obj, "snapshot", slugFile = true, basePath, config) - - val contentSize = (new File(zipFileName)).length - - // delete local folder - FileUtils.deleteQuietly(basePath) - - obj.metadata ++ Map("artifactUrl" -> result(1), "cloudStorageKey" -> result(0), "size" -> contentSize.asInstanceOf[AnyRef]) - } catch { - case ex@(_: org.sunbird.cloud.storage.exception.StorageServiceException | _: java.lang.NullPointerException | _:java.io.FileNotFoundException | _:java.io.IOException) => { - ex.printStackTrace() - throw new InvalidInputException(s"ECML Asset Files not found For $obj.identifier") - } - case anyEx: Exception => throw anyEx - } - } - - private def getEcrfObject(ecmlType: String, ecmlBody: String): Plugin = { - ecmlType match { - case "ecml" => XmlParser.parse(ecmlBody) - case "json" => JsonParser.parse(ecmlBody) - case _ => classOf[Plugin].newInstance() - } - } - - private def getEcmlStringFromEcrf(processedEcrf: Plugin, ecmlType: String): String = { - ecmlType match { - case "ecml" => XmlParser.toString(processedEcrf) - case "json" => JsonParser.toString(processedEcrf) - case _ => "" - } - } - - private def getECMLType(contentBody: String): String = { - if (!StringUtils.isBlank(contentBody)) { - if (isValidJSON(contentBody)) "json" - else if (isValidXML(contentBody)) "ecml" - else throw new InvalidInputException("Invalid Content Body") - } - else throw new InvalidInputException("Invalid Content Body. ECML content should have body.") - } - - private def isValidJSON(contentBody: String): Boolean = { - if (!StringUtils.isBlank(contentBody)) try { - val objectMapper = new ObjectMapper - objectMapper.enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) - objectMapper.readTree(contentBody) - true - } catch { - case ex@(_: IOException) => - logger.error("isValidJSON - Exception when validating the JSON :: ", ex) - false - } - else false - } - - private def isValidXML(contentBody: String): Boolean = { - if (!StringUtils.isBlank(contentBody)) try { - val dbFactory = DocumentBuilderFactory.newInstance - val dBuilder = dbFactory.newDocumentBuilder - dBuilder.parse(new InputSource(new StringReader(contentBody))) - true - } catch { - case ex@(_: ParserConfigurationException | _: SAXException | _: IOException) => - logger.error("isValidXML - Exception when validating the XML :: ", ex) - false - } else false - } - - private def writeECMLFile(basePath: String, ecml: String, ecmlType: String): Unit = { - try { - if (StringUtils.isBlank(ecml)) throw new InvalidInputException("[Unable to write Empty ECML File.]") - if (StringUtils.isBlank(ecmlType)) throw new InvalidInputException("[System is in a fix between (XML & JSON) ECML Type.]") - val file = new File(basePath + "/" + "index." + ecmlType) - FileUtils.writeStringToFile(file, ecml) - } catch { - case e: Exception => - logger.error("writeECMLFile - Exception when write ECML file :: ", e) - throw new Exception("[Unable to Write ECML File.]", e) - } - } - - private def extractPackageInCloud(uploadFile: File, obj: ObjectData, extractionType: String, slugFile: Boolean, basePath: String, config: LiveNodePublisherConfig)(implicit cloudStorageUtil: CloudStorageUtil) = { - val file = Slug.createSlugFile(uploadFile) - val mimeType = obj.mimeType - validationForCloudExtraction(file, extractionType, mimeType, config) - if (config.extractableMimeTypes.contains(mimeType)) { - cloudStorageUtil.uploadDirectory(getExtractionPath(obj, config, extractionType), new File(basePath), Option(slugFile)) - } - } - - private def uploadArtifactToCloud(uploadedFile: File, identifier: String, filePath: Option[String] = None, config: LiveNodePublisherConfig)(implicit cloudStorageUtil: CloudStorageUtil): Array[String] = { - val urlArray = { - try { - val folder = if (filePath.isDefined) filePath.get + File.separator + config.contentFolder + File.separator + Slug.makeSlug(identifier, isTransliterate = true) + File.separator + config.artifactFolder - else config.contentFolder + File.separator + Slug.makeSlug(identifier, isTransliterate = true) + File.separator + config.artifactFolder - cloudStorageUtil.uploadFile(folder, uploadedFile) - } catch { - case e: Exception => - cloudStorageUtil.deleteFile(uploadedFile.getAbsolutePath, Option(false)) - logger.error("Error while uploading the Artifact file.", e) - throw new Exception("Error while uploading the Artifact File.", e) - } - } - urlArray - } - - private def validationForCloudExtraction(file: File, extractionType: String, mimeType: String, config: LiveNodePublisherConfig): Unit = { - if (!file.exists() || (!extractablePackageExtensions.contains("." + FilenameUtils.getExtension(file.getName)) && config.extractableMimeTypes.contains(mimeType))) - throw new InvalidInputException("Error! File doesn't Exist.") - if (extractionType == null) - throw new InvalidInputException("Error! Invalid Content Extraction Type.") - } - - - private def localizeAssets(contentId: String, ecrfObj: Plugin, basePath: String, config: LiveNodePublisherConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Unit = { - val medias: List[Media] = if (null != ecrfObj && null != ecrfObj.manifest) ecrfObj.manifest.medias else List.empty - if (null != medias && medias.nonEmpty) processAssetsDownload(contentId, medias, basePath, config) - } - - private def processAssetsDownload(contentId: String, medias: List[Media], basePath: String, config: LiveNodePublisherConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Map[String, String] = { - val downloadResultMap = Await.result(downloadAssetFiles(contentId, medias, basePath, config), Duration.apply(config.assetDownloadDuration)) - downloadResultMap.filter(record => record.nonEmpty).flatten.toMap - } - - private def downloadAssetFiles(identifier: String, mediaFiles: List[Media], basePath: String, config: LiveNodePublisherConfig)(implicit ec: ExecutionContext, cloudStorageUtil: CloudStorageUtil): Future[List[Map[String, String]]] = { - val futures = mediaFiles.map(mediaFile => { - logger.info(s"ExtractableMimeTypeHelper ::: downloadAssetFiles ::: Processing file: ${mediaFile.id} for : " + identifier) - if (!StringUtils.equals("youtube", mediaFile.`type`) && !StringUtils.isBlank(mediaFile.src) && !StringUtils.isBlank(mediaFile.`type`)) { - val downloadPath = if (isWidgetTypeAsset(mediaFile.`type`)) basePath + "/" + "widgets" else basePath + "/" + "assets" - val subFolder = { - if (!mediaFile.src.startsWith("http") && !mediaFile.src.startsWith("https")) { - val f = new File(mediaFile.src) - if (f.exists) f.delete - StringUtils.stripStart(f.getParent, "/") - } else "" - } - val fDownloadPath = if (StringUtils.isNotBlank(subFolder)) downloadPath + File.separator + subFolder + File.separator else downloadPath + File.separator - createDirectoryIfNeeded(fDownloadPath) - logger.info(s"ExtractableMimeTypeHelper ::: downloadAssetFiles ::: fDownloadPath: $fDownloadPath & src : ${mediaFile.src}") - - if (mediaFile.src.startsWith("https://") || mediaFile.src.startsWith("http://")) { - FileUtils.downloadFile(mediaFile.src, fDownloadPath) - } - else { - if (mediaFile.src.contains("assets/public")) { - try { - cloudStorageUtil.downloadFile(fDownloadPath, StringUtils.replace(mediaFile.src, "//", "/").substring(mediaFile.src.indexOf("assets/public") + 14)) - } catch { - case _: Exception => cloudStorageUtil.downloadFile(fDownloadPath, mediaFile.src.substring(mediaFile.src.indexOf("assets/public") + 13)) - } - } - else if (mediaFile.src.startsWith(File.separator)) { - cloudStorageUtil.downloadFile(fDownloadPath, StringUtils.replace(mediaFile.src.substring(1), "//", "/")) - } else { - cloudStorageUtil.downloadFile(fDownloadPath, StringUtils.replace(mediaFile.src, "//", "/")) - } - } - val downloadedFile = new File(fDownloadPath + mediaFile.src.split("/").last) - logger.info("Downloaded file : " + mediaFile.src + " - " + downloadedFile + " | [Content Id '" + identifier + "']") - - Map(mediaFile.id -> downloadedFile.getName) - } else Map.empty[String, String] - }) - Future(futures) - } - - private def isWidgetTypeAsset(assetType: String): Boolean = StringUtils.equalsIgnoreCase(assetType, "js") || StringUtils.equalsIgnoreCase(assetType, "css") || StringUtils.equalsIgnoreCase(assetType, "json") || StringUtils.equalsIgnoreCase(assetType, "plugin") - - private def createDirectoryIfNeeded(directoryName: String): Unit = { - val theDir = new File(directoryName) - if (!theDir.exists) theDir.mkdirs - } - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveCollectionPublisher.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveCollectionPublisher.scala deleted file mode 100644 index 8dd9c6cde..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveCollectionPublisher.scala +++ /dev/null @@ -1,649 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.{Insert, QueryBuilder, Select} -import com.fasterxml.jackson.core.JsonProcessingException -import org.apache.commons.io.FileUtils -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers._ -import org.sunbird.job.util._ - -import java.io.{File, IOException} -import java.util -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.collection.mutable.ListBuffer -import scala.concurrent.ExecutionContext -import java.text.{DecimalFormat, DecimalFormatSymbols, SimpleDateFormat} -import java.util.{Date, Locale} - -trait LiveCollectionPublisher extends LiveObjectReader with SyncMessagesGenerator with ObjectValidator with ObjectEnrichment with EcarGenerator with LiveObjectUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[LiveCollectionPublisher]) - private val level4ContentTypes = List("Course", "CourseUnit", "LessonPlan", "LessonPlanUnit") - private val EXPANDABLE_OBJECTS = List("Collection", "QuestionSet") - private val EXCLUDE_LEAFNODE_OBJECTS = List("Collection", "Question") - private val INCLUDE_LEAFNODE_OBJECTS = List("QuestionSet") - private val INCLUDE_CHILDNODE_OBJECTS = List("Collection") - private val PUBLISHED_STATUS_LIST = List("Live", "Unlisted") - private val COLLECTION_MIME_TYPE = "application/vnd.ekstep.content-collection" - - override def getExtData(identifier: String, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] = None - - override def getHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = { - try { - val row: Row = getCollectionHierarchy(identifier, readerConfig) - if (null != row) { - val hierarchy = row.getString("hierarchy") - val updatedHierarchy = if (config.asInstanceOf[LiveNodePublisherConfig].isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(hierarchy) else hierarchy - val data: Map[String, AnyRef] = if(updatedHierarchy.nonEmpty) ScalaJsonUtil.deserialize[Map[String, AnyRef]](updatedHierarchy) else Map.empty[String, AnyRef] - Option(data) - } else Option(Map.empty[String, AnyRef]) - } catch { - case _: Exception => throw new InvalidInputException("Exception while reading Hierarchy for collection:: " + identifier) - } - } - - private def getCollectionHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Row = { - val selectWhere: Select.Where = QueryBuilder.select().all() - .from(readerConfig.keyspace, readerConfig.table). - where() - selectWhere.and(QueryBuilder.eq("identifier", identifier)) - cassandraUtil.findOne(selectWhere.toString) - } - - def getRelationalMetadata(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = { - val row: Row = getCollectionHierarchy(identifier, readerConfig) - if (null != row && row.getString("relational_metadata") != null && row.getString("relational_metadata").nonEmpty) { - val data: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](row.getString("relational_metadata")) - Option(data) - } else Option(Map.empty[String, AnyRef]) - } - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig): Option[ObjectData] = { - val contentConfig = config.asInstanceOf[LiveNodePublisherConfig] - val extraMeta = Map("pkgVersion" -> (obj.pkgVersion + 1).asInstanceOf[AnyRef], "lastPublishedOn" -> getTimeStamp, - "flagReasons" -> null, "body" -> null, "publishError" -> null, "variants" -> null, "downloadUrl" -> null) - val contentSize = obj.metadata.getOrElse("size", 0).toString.toDouble - val configSize = contentConfig.artifactSizeForOnline - val updatedMeta: Map[String, AnyRef] = if (contentSize > configSize) obj.metadata ++ extraMeta ++ Map("contentDisposition" -> "online-only") else obj.metadata ++ extraMeta - - val publishType = obj.getString("publish_type", "Public") - val status = if (StringUtils.equalsIgnoreCase("Unlisted", publishType)) "Unlisted" else "Live" - val updatedCompatibilityLevelMeta: Map[String, AnyRef] = setCompatibilityLevel(obj, updatedMeta).get + ("status" -> status) - - val isCollectionShallowCopy = isContentShallowCopy(obj) - - // Collection - Enrich Children - line 345 - val collectionHierarchy: Map[String, AnyRef] = if (isCollectionShallowCopy) { - val originData: Map[String, AnyRef] = obj.metadata.getOrElse("originData", "").asInstanceOf[Map[String, AnyRef]] - getHierarchy(obj.metadata("origin").asInstanceOf[String], readerConfig).get - } else getHierarchy(obj.identifier, readerConfig).get - logger.info("LiveCollectionPublisher:: enrichObjectMetadata:: collectionHierarchy:: " + collectionHierarchy) - val children = if (collectionHierarchy.nonEmpty) { - collectionHierarchy.getOrElse("children", List.empty[Map[String, AnyRef]]).asInstanceOf[List[Map[String, AnyRef]]] - } else List.empty[Map[String, AnyRef]] - val toEnrichChildren = children.to[ListBuffer] - val enrichedObj: ObjectData = if (collectionHierarchy.nonEmpty && !isCollectionShallowCopy) { - val childNodesToRemove = ListBuffer.empty[String] - val collectionResourceChildNodes: mutable.HashSet[String] = mutable.HashSet.empty[String] - val enrichedChildrenData = enrichChildren(toEnrichChildren, collectionResourceChildNodes, childNodesToRemove) - val collectionChildNodes = (updatedCompatibilityLevelMeta.getOrElse("childNodes", new java.util.ArrayList()).asInstanceOf[java.util.List[String]].asScala.toList ++ collectionResourceChildNodes).distinct - new ObjectData(obj.identifier, updatedCompatibilityLevelMeta ++ Map("childNodes" -> collectionChildNodes.filter(rec => !childNodesToRemove.contains(rec))), obj.extData, Option(collectionHierarchy + ("children" -> enrichedChildrenData.toList))) - } else new ObjectData(obj.identifier, updatedCompatibilityLevelMeta, obj.extData, Option(collectionHierarchy ++ Map("children" -> toEnrichChildren.toList))) - - logger.info("LiveCollectionPublisher:: enrichObjectMetadata:: Collection data after processing for : " + enrichedObj.identifier + " | Metadata : " + enrichedObj.metadata) - logger.debug("LiveCollectionPublisher:: enrichObjectMetadata:: Collection children data after processing : " + enrichedObj.hierarchy.get("children")) - - Some(enrichedObj) - } - - override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = { - val hChildren: List[Map[String, AnyRef]] = obj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - Some(getFlatStructure(List(obj.metadata ++ obj.extData.getOrElse(Map()) ++ Map("children" -> hChildren)), List())) - } - - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val identifier = obj.identifier.replace(".img", "") - val query: Insert = QueryBuilder.insertInto(readerConfig.keyspace, readerConfig.table) - query.value(readerConfig.primaryKey(0), identifier) - query.value("relational_metadata", null) - val result = cassandraUtil.upsert(query.toString) - if (result) { - logger.info(s"relational_metadata emptied successfully for ${identifier}") - } else { - val msg = s"relational_metadata emptying Failed For $identifier" - logger.error(msg) - } - } - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - def getObjectWithEcar(obj: ObjectData, pkgTypes: List[String])(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig, httpUtil: HttpUtil): ObjectData = { - val collRelationalMetadata = getRelationalMetadata(obj.identifier, readerConfig).getOrElse(Map.empty[String, AnyRef]) - // Line 1107 in PublishFinalizer - val children = obj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - val updatedChildren = updateHierarchyMetadata(children, obj.metadata, collRelationalMetadata)(config) - val enrichedObj = processCollection(obj, updatedChildren) - val updatedObj = updateRootChildrenList(enrichedObj, updatedChildren) - val nodes = ListBuffer.empty[ObjectData] - val nodeIds = ListBuffer.empty[String] - nodes += obj - nodeIds += obj.identifier - - val ecarMap: Map[String, String] = try{ - generateEcar(updatedObj, pkgTypes) - } catch { - case ex@(_: org.sunbird.cloud.storage.exception.StorageServiceException | _: java.lang.NullPointerException | _:java.io.FileNotFoundException | _:java.io.IOException) => { - ex.printStackTrace() - throw new InvalidInputException(s"ECAR bundling failed for ${obj.identifier}:: " + ex.getMessage) - } - case anyEx: Exception => throw anyEx - } - - val variants: java.util.Map[String, java.util.Map[String, String]] = ecarMap.map { case (key, value) => key.toLowerCase -> Map[String, String]("ecarUrl" -> value, "size" -> httpUtil.getSize(value).toString).asJava }.asJava - logger.info("CollectionPulisher ::: getObjectWithEcar ::: variants ::: " + variants) - - val meta: Map[String, AnyRef] = Map("downloadUrl" -> ecarMap.getOrElse(EcarPackageType.SPINE, ""), "variants" -> variants, "size" -> httpUtil.getSize(ecarMap.getOrElse(EcarPackageType.SPINE, "")).asInstanceOf[AnyRef]) - new ObjectData(updatedObj.identifier, updatedObj.metadata ++ meta, updatedObj.extData, updatedObj.hierarchy) - } - - private def setCompatibilityLevel(obj: ObjectData, updatedMeta: Map[String, AnyRef]): Option[Map[String, AnyRef]] = { - if (level4ContentTypes.contains(obj.getString("contentType", ""))) { - logger.info("LiveCollectionPublisher:: setCompatibilityLevel:: setting compatibility level for content id : " + obj.identifier + " as 4.") - Some(updatedMeta ++ Map("compatibilityLevel" -> 4.asInstanceOf[AnyRef])) - } else Some(updatedMeta) - } - - def getUnitsFromLiveContent(obj: ObjectData)(implicit cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, config: PublishConfig): List[String] = { - logger.info("LiveCollectionPublisher:: getUnitsFromLiveContent:: identifier: " + obj.identifier + " || pkgVersion: " + obj.metadata.getOrElse("pkgVersion", 1).asInstanceOf[Number]) - val objHierarchy = getHierarchy(obj.identifier, readerConfig).get - val children = objHierarchy.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]] - getUnits(children) - } - - private def getUnits(children: List[Map[String, AnyRef]]): List[String] = { - if (children.nonEmpty) { - children.flatMap(child => { - if (child.getOrElse("visibility", "").asInstanceOf[String].equalsIgnoreCase("Parent")) { - val childUnits = if(child.contains("children")) getUnits(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]]) else List.empty - childUnits ++ List(child.getOrElse("identifier", "").asInstanceOf[String]) - } else List.empty[String] - }).filter(rec => rec.nonEmpty) - } else List.empty[String] - } - - def isContentShallowCopy(obj: ObjectData): Boolean = { - val originData: Map[String, AnyRef] = if(obj.metadata.contains("originData")) { - obj.metadata("originData") match { - case strValue: String => ScalaJsonUtil.deserialize[Map[String, AnyRef]](strValue) - case mapValue:util.Map[String, AnyRef] => mapValue.asScala.toMap[String, AnyRef] - case _ => obj.metadata("originData").asInstanceOf[Map[String,AnyRef]] - } - } else Map.empty[String, AnyRef] - originData.nonEmpty && originData.getOrElse("copyType", "").asInstanceOf[String].equalsIgnoreCase("shallow") - } - - def updateOriginPkgVersion(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil): ObjectData = { - val originId = obj.metadata.getOrElse("origin", "").asInstanceOf[String] - val originNodeMetadata = Option(neo4JUtil.getNodeProperties(originId)).getOrElse(neo4JUtil.getNodeProperties(originId)) - if (null != originNodeMetadata) { - val originPkgVer: Double = originNodeMetadata.getOrDefault("pkgVersion", "0").toString.toDouble - if (originPkgVer != 0) { - val originData = obj.metadata("originData") match { - case propVal: String => ScalaJsonUtil.deserialize[Map[String, AnyRef]](propVal) + ("pkgVersion" -> originPkgVer) - case _ => obj.metadata.getOrElse("originData", Map.empty[String, AnyRef]).asInstanceOf[Map[String, AnyRef]] + ("pkgVersion" -> originPkgVer) - } - new ObjectData(obj.identifier, obj.metadata ++ Map("originData" -> originData), obj.extData, obj.hierarchy) - } else obj - } else obj - } - - private def enrichChildren(toEnrichChildren: ListBuffer[Map[String, AnyRef]], collectionResourceChildNodes: mutable.HashSet[String], childNodesToRemove: ListBuffer[String])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, config: PublishConfig): ListBuffer[Map[String, AnyRef]] = { - val newChildren = toEnrichChildren.toList - newChildren.map(child => { - logger.info("LiveCollectionPublisher:: enrichChildren:: child identifier:: " + child.getOrElse("identifier", "") + " || visibility:: " + child.getOrElse("visibility", "") + " || mimeType:: " + child.getOrElse("mimeType", "") + " || objectType:: " + child.getOrElse("objectType", "")) - if (StringUtils.equalsIgnoreCase(child.getOrElse("visibility", "").asInstanceOf[String], "Parent") && StringUtils.equalsIgnoreCase(child.getOrElse("mimeType", "").asInstanceOf[String], COLLECTION_MIME_TYPE)) { - val updatedChildrenData = enrichChildren(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]].to[ListBuffer], collectionResourceChildNodes, childNodesToRemove) - toEnrichChildren(newChildren.indexOf(child)) = child + ("children" -> updatedChildrenData.toList) - } - - if (StringUtils.equalsIgnoreCase(child.getOrElse("visibility", "").asInstanceOf[String], "Default") && EXPANDABLE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String])) { - val pkgVersion = child.getOrElse("pkgVersion", 0) match { - case _: Integer => child.getOrElse("pkgVersion", 0).asInstanceOf[Integer].doubleValue() - case _: Double => child.getOrElse("pkgVersion", 0).asInstanceOf[Double].doubleValue() - case _ => child.getOrElse("pkgVersion", "0").toString.toDouble - } - val childCollectionHierarchy = getHierarchy(child.getOrElse("identifier", "").asInstanceOf[String], readerConfig).get - if (childCollectionHierarchy.nonEmpty) { - val childNodes = childCollectionHierarchy.getOrElse("childNodes", List.empty).asInstanceOf[List[String]] - if (childNodes.nonEmpty && INCLUDE_CHILDNODE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String])) collectionResourceChildNodes ++= childNodes.toSet[String] - toEnrichChildren(newChildren.indexOf(child)) = childCollectionHierarchy ++ Map("index" -> child.getOrElse("index", 0).asInstanceOf[AnyRef], "depth" -> child.getOrElse("depth", 0).asInstanceOf[AnyRef], "parent" -> child.getOrElse("parent", ""), "objectType" -> child.getOrElse("objectType", "Collection").asInstanceOf[String]) - } - } - - if (StringUtils.equalsIgnoreCase(child.getOrElse("visibility", "").asInstanceOf[String], "Default") && !EXPANDABLE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String])) { - val childNode = Option(neo4JUtil.getNodeProperties(child.getOrElse("identifier", "").asInstanceOf[String])).getOrElse(neo4JUtil.getNodeProperties(child.getOrElse("identifier", "").asInstanceOf[String])).asScala.toMap - if (PUBLISHED_STATUS_LIST.contains(childNode.getOrElse("status", "").asInstanceOf[String])) { - logger.info("LiveCollectionPublisher:: enrichChildren:: fetched child node:: " + childNode.getOrElse("IL_UNIQUE_ID", "").asInstanceOf[String] + " || objectType:: " + childNode.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String]) - toEnrichChildren(newChildren.indexOf(child)) = childNode ++ Map("identifier" ->childNode.getOrElse("IL_UNIQUE_ID", "").asInstanceOf[String], "objectType" ->childNode.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String], "index" -> child.getOrElse("index", 0).asInstanceOf[AnyRef], "parent" -> child.getOrElse("parent", "").asInstanceOf[String], "depth" -> child.getOrElse("depth", 0).asInstanceOf[AnyRef]) - ("collections", "children", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE","IL_UNIQUE_ID") - } else childNodesToRemove += child.getOrElse("identifier", "").asInstanceOf[String] - } - }) - - toEnrichChildren - } - - - private def processCollection(obj: ObjectData, children: List[Map[String, AnyRef]])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig): ObjectData = { - val dataMap: mutable.Map[String, AnyRef] = processChildren(children) - logger.info("LiveCollectionPublisher:: processCollection:: dataMap: " + dataMap) - val updatedObj: ObjectData = if (dataMap.nonEmpty) { - val updatedMetadataMap: Map[String, AnyRef] = dataMap.flatMap(record => { - if (!"concepts".equalsIgnoreCase(record._1) && !"keywords".equalsIgnoreCase(record._1)) { - Map(record._1 -> record._2.asInstanceOf[Set[String]].toArray[String]) - } else Map.empty[String, AnyRef] - }).filter(record => record._1.nonEmpty).toMap[String, AnyRef] - val keywords = dataMap.getOrElse("keywords", Set.empty).asInstanceOf[Set[String]].toArray[String] - val finalKeywords: Array[String] = if (null != keywords && keywords.nonEmpty) { - val updatedKeywords: Array[String] = if (obj.metadata.contains("keywords")) { - obj.metadata("keywords") match { - case _: Array[String] => keywords ++ obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[Array[String]] - case kwValue: String => keywords ++ Array[String](kwValue) - case _: util.Collection[String] => keywords ++ obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[util.Collection[String]].asScala.toArray[String] - case _ => keywords - } - } else keywords - updatedKeywords.filter(record => record.trim.nonEmpty).distinct - } else if(obj.metadata.contains("keywords")) { - obj.metadata("keywords") match { - case _: Array[String] => obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[Array[String]] - case kwValue: String => Array[String](kwValue) - case _: util.Collection[String] => obj.metadata.getOrElse("keywords", Array.empty).asInstanceOf[util.Collection[String]].asScala.toArray[String] - case _ => Array.empty[String] - } - } else Array.empty[String] - new ObjectData(obj.identifier, obj.metadata ++ updatedMetadataMap + ("keywords" -> finalKeywords), obj.extData, obj.hierarchy) - } else obj - - val enrichedObject = if(children.nonEmpty) enrichCollection(updatedObj) else updatedObj - // addResourceToCollection(enrichedObject, children.to[ListBuffer]) - TODO - enrichedObject - } - - private def processChildren(children: List[Map[String, AnyRef]]): mutable.Map[String, AnyRef] = { - val dataMap: mutable.Map[String, AnyRef] = mutable.Map.empty - processChildren(children, dataMap) - dataMap - } - - private def processChildren(children: List[Map[String, AnyRef]], dataMap: mutable.Map[String, AnyRef]): Unit = { - if (null != children && children.nonEmpty) { - for (child <- children) { - mergeMap(dataMap, processChild(child)) - if (child.contains("children")) processChildren(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]], dataMap) - } - } - } - - private def processChild(childMetadata: Map[String, AnyRef]): Map[String, AnyRef] = { - val taggingProperties = List("language", "domain", "ageGroup", "genre", "theme", "keywords") - val result: Map[String, AnyRef] = childMetadata.flatMap(prop => { - if (taggingProperties.contains(prop._1)) { - childMetadata(prop._1) match { - case propStrValue: String => Map(prop._1 -> Set(propStrValue)) - case propListValue: List[_] => Map(prop._1 -> propListValue.toSet) - case propVal: java.util.List[String] => Map(prop._1 -> propVal.asScala.toSet[String]) - case _ => Map.empty[String, AnyRef] - } - } else Map.empty[String, AnyRef] - }).filter(rec => rec._1.nonEmpty) - result - } - - private def mergeMap(dataMap: mutable.Map[String, AnyRef], childDataMap: Map[String, AnyRef]): mutable.Map[String, AnyRef] = { - if (dataMap.isEmpty) dataMap ++= childDataMap - else { - dataMap.map(record => { - dataMap += (record._1 -> (if (childDataMap.contains(record._1)) childDataMap(record._1).asInstanceOf[Set[String]] ++ record._2.asInstanceOf[Set[String]] else record._2.asInstanceOf[Set[String]])) - }) - if (!dataMap.equals(childDataMap)) { - childDataMap.map(record => { - if (!dataMap.contains(record._1)) dataMap += record - }) - } - } - dataMap - } - - def enrichCollection(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig): ObjectData = { - val nodeMetadata = mutable.Map.empty[String, AnyRef] ++ obj.metadata - val contentId = obj.identifier - logger.info("LiveCollectionPublisher:: enrichCollection:: Processing Collection Content :" + contentId) - val content = obj.hierarchy.get - if (content.isEmpty) return obj - val leafCount = getLeafNodeCount(content) - val totalCompressedSize = getTotalCompressedSize(content, 0.0) - - val df = new DecimalFormat("0", DecimalFormatSymbols.getInstance(Locale.ENGLISH)) - df.setMaximumFractionDigits(0) - - nodeMetadata.put("leafNodesCount", leafCount.asInstanceOf[AnyRef]) - nodeMetadata.put("totalCompressedSize", df.format(totalCompressedSize).toLong.asInstanceOf[AnyRef]) - - nodeMetadata.put("leafNodes", updateLeafNodeIds(content)) - val mimeTypeMap: mutable.Map[String, AnyRef] = mutable.Map.empty[String, AnyRef] - val contentTypeMap: mutable.Map[String, AnyRef] = mutable.Map.empty[String, AnyRef] - getTypeCount(content, "mimeType", mimeTypeMap) - getTypeCount(content, "contentType", contentTypeMap) - - val updatedContent = content ++ Map("leafNodesCount" -> leafCount, "totalCompressedSize" -> df.format(totalCompressedSize).toLong, "mimeTypesCount" -> ScalaJsonUtil.serialize(mimeTypeMap), "contentTypesCount" -> ScalaJsonUtil.serialize(contentTypeMap)).asInstanceOf[Map[String, AnyRef]] - nodeMetadata.put("mimeTypesCount", ScalaJsonUtil.serialize(mimeTypeMap)) - nodeMetadata.put("contentTypesCount", ScalaJsonUtil.serialize(contentTypeMap)) - val uploadedFileUrl: Array[String] = generateTOC(obj, nodeMetadata.toMap) - if(uploadedFileUrl.nonEmpty) { - nodeMetadata.put("toc_url", uploadedFileUrl(1)) - nodeMetadata.put("s3Key", uploadedFileUrl(0)) - } - - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ") - val updatedOn = sdf.format(new Date()) - nodeMetadata.put("sYS_INTERNAL_LAST_UPDATED_ON", updatedOn) - val updatedMetadata: Map[String, AnyRef] = try { - setContentAndCategoryTypes(nodeMetadata.toMap) - } catch { - case e: Exception => logger.error("LiveCollectionPublisher:: enrichCollection:: Error while stringify mimeTypeCount or contentTypesCount:", e) - nodeMetadata.toMap - } - - new ObjectData(obj.identifier, updatedMetadata, obj.extData, Option(updatedContent)) - } - - private def updateLeafNodeIds(content: Map[String, AnyRef]): Array[String] = { - val leafNodeIds: mutable.Set[String] = mutable.Set.empty[String] - getLeafNodesIds(content, leafNodeIds) - leafNodeIds.toArray - } - - private def getTypeCount(data: Map[String, AnyRef], `type`: String, typeMap: mutable.Map[String, AnyRef]): Unit = { - val children = data.getOrElse("children", List.empty).asInstanceOf[List[AnyRef]] - if (null != children && children.nonEmpty) { - for (child <- children) { - val childMap = child.asInstanceOf[Map[String, AnyRef]] - val typeValue = childMap.getOrElse(`type`, "").asInstanceOf[String] - if (null != typeValue) if (typeMap.contains(typeValue)) { - var count = typeMap.getOrElse(typeValue, 0).asInstanceOf[Int] - count += 1 - typeMap.put(typeValue, count.asInstanceOf[AnyRef]) - } - else typeMap.put(typeValue, 1.asInstanceOf[AnyRef]) - if (childMap.contains("children")) getTypeCount(childMap, `type`, typeMap) - } - } - } - - @SuppressWarnings(Array("unchecked")) - private def getLeafNodeCount(data: Map[String, AnyRef]): Int = { - val leafNodeIds: mutable.Set[String] = mutable.Set.empty[String] - getLeafNodesIds(data, leafNodeIds) - leafNodeIds.size - } - - private def getLeafNodesIds(data: Map[String, AnyRef], leafNodeIds: mutable.Set[String]): Unit = { - if (INCLUDE_LEAFNODE_OBJECTS.contains(data.getOrElse("objectType", "")) && StringUtils.equals(data.getOrElse("visibility", "").asInstanceOf[String], "Default")) leafNodeIds += data.getOrElse("identifier", "").asInstanceOf[String] - val children = data.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]] - if (children.nonEmpty) { - for (child <- children) { - getLeafNodesIds(child, leafNodeIds) - } - } - else if (!EXCLUDE_LEAFNODE_OBJECTS.contains(data.getOrElse("objectType", "").asInstanceOf[String])) leafNodeIds.add(data.getOrElse("identifier", "").asInstanceOf[String]) - } - - private def getTotalCompressedSize(data: Map[String, AnyRef], totalCompressed: Double): Double = { - val children = data.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]] - if (children.nonEmpty) { - val childrenSizes = children.map(child => { - val childSize = - if (!EXPANDABLE_OBJECTS.contains(child.getOrElse("objectType", "").asInstanceOf[String]) && StringUtils.equals(child.getOrElse("visibility", "").asInstanceOf[String], "Default")) { - child.getOrElse("totalCompressedSize", child.getOrElse("size", 0).asInstanceOf[Number].doubleValue).asInstanceOf[Number].doubleValue - } else 0 - - getTotalCompressedSize(child, childSize) - }).sum - totalCompressed + childrenSizes - } - else totalCompressed - } - - def generateTOC(obj: ObjectData, content: Map[String, AnyRef])(implicit cloudStorageUtil: CloudStorageUtil, config: PublishConfig): Array[String] = { - logger.info("LiveCollectionPublisher:: generateTOC:: Write hierarchy to JSON File :" + obj.identifier) - val file = new File(getTOCBasePath(obj.identifier) + "_toc.json") - try { - val data = ScalaJsonUtil.serialize(content) - FileUtils.writeStringToFile(file, data, "UTF-8") - if (file.exists) { - logger.debug("LiveCollectionPublisher:: generateTOC:: Upload File to cloud storage :" + file.getName) - val uploadedFileUrl = cloudStorageUtil.uploadFile(getAWSPath(obj.identifier), file, Option.apply(true)) - if (null != uploadedFileUrl && uploadedFileUrl.length > 1) { - logger.info("LiveCollectionPublisher:: generateTOC:: Update cloud storage url to node" + uploadedFileUrl(1)) - uploadedFileUrl - } else Array.empty - } else Array.empty - } catch { - case e: JsonProcessingException => logger.error("LiveCollectionPublisher:: generateTOC:: Error while parsing map object to string.", e) - throw new InvalidInputException("LiveCollectionPublisher:: generateTOC:: Error while parsing map object to string.", e) - case e: Exception => logger.error("LiveCollectionPublisher:: generateTOC:: Error while uploading file ", e) - throw new InvalidInputException("LiveCollectionPublisher:: generateTOC:: Error while uploading file", e) - } finally try { - logger.info("LiveCollectionPublisher:: generateTOC:: Deleting Uploaded files") - FileUtils.deleteDirectory(file.getParentFile) - } catch { - case e: IOException => - logger.error("LiveCollectionPublisher:: generateTOC::Error while deleting file ", e) - } - } - - private def getTOCBasePath(contentId: String)(implicit cloudStorageUtil: CloudStorageUtil, config: PublishConfig): String = { - if (contentId.nonEmpty) "/tmp" + File.separator + System.currentTimeMillis + "_temp" + File.separator + contentId else "" - } - - private def getAWSPath(identifier: String)(implicit cloudStorageUtil: CloudStorageUtil, config: PublishConfig): String = { - val contentConfig = config.asInstanceOf[LiveNodePublisherConfig] - val folderName = contentConfig.contentFolder - if (folderName.nonEmpty) folderName + File.separator + Slug.makeSlug(identifier, isTransliterate = true) + File.separator + contentConfig.artifactFolder else folderName - } - - def setContentAndCategoryTypes(input: Map[String, AnyRef])(implicit config: PublishConfig): Map[String, AnyRef] = { - val contentConfig = config.asInstanceOf[LiveNodePublisherConfig] - val categoryMap = contentConfig.categoryMap - val contentType = input.getOrElse("contentType", "").asInstanceOf[String] - val primaryCategory = input.getOrElse("primaryCategory","").asInstanceOf[String] - val (updatedContentType, updatedPrimaryCategory): (String, String) = - if(contentType.nonEmpty && (primaryCategory.isEmpty || primaryCategory.isBlank)) { (contentType, categoryMap.getOrDefault(contentType,"").asInstanceOf[String]) } - else if((contentType.isEmpty || contentType.isBlank) && primaryCategory.nonEmpty) { (categoryMap.asScala.filter(entry => StringUtils.equalsIgnoreCase(entry._2.asInstanceOf[String], primaryCategory)).keys.headOption.getOrElse(""), primaryCategory) } - else (contentType, primaryCategory) - - input ++ Map("contentType" -> updatedContentType, "primaryCategory" -> updatedPrimaryCategory) - } - - def updateHierarchyMetadata(children: List[Map[String, AnyRef]], objMetadata: Map[String, AnyRef], collRelationalMetadata: Map[String, AnyRef])(implicit config: PublishConfig): List[Map[String, AnyRef]] = { - if (children.nonEmpty) { - children.map(child => { - if (StringUtils.equalsIgnoreCase("Parent", child.getOrElse("visibility", "").asInstanceOf[String])) { //set child metadata -- compatibilityLevel, appIcon, posterImage, lastPublishedOn, pkgVersion, status - val updatedChild = populatePublishMetadata(child, objMetadata) - updatedChild + ("children" -> updateHierarchyMetadata(updatedChild.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]], objMetadata, collRelationalMetadata)) - } else { - //TODO: Populate relationalMetadata here for child contents - if (collRelationalMetadata.nonEmpty) { - val parent = child.getOrElse("parent", "").asInstanceOf[String] - val unitRelationalMetadata = collRelationalMetadata(parent).asInstanceOf[Map[String, AnyRef]].getOrElse("relationalMetadata", Map.empty).asInstanceOf[Map[String, AnyRef]] - if (unitRelationalMetadata.nonEmpty) { - val childRelationalMetadata = unitRelationalMetadata.getOrElse(child.getOrElse("identifier","").asInstanceOf[String], Map.empty).asInstanceOf[Map[String, AnyRef]] - if(childRelationalMetadata.nonEmpty) { - child + ("relationalMetadata" -> childRelationalMetadata) - } else child - } else child - } else child - } - }) - } else children - } - - private def populatePublishMetadata(content: Map[String, AnyRef], objMetadata: Map[String, AnyRef])(implicit config: PublishConfig): Map[String, AnyRef] = { - //TODO: For appIcon, posterImage and screenshot createThumbNail method has to be implemented. - val leafNodeIds: mutable.Set[String] = mutable.Set.empty[String] - getLeafNodesIds(content, leafNodeIds) - - val updatedContent = content ++ - Map("compatibilityLevel" -> (if (null != content.get("compatibilityLevel")) content.getOrElse("compatibilityLevel", 1).asInstanceOf[Number].intValue else 1), - "lastPublishedOn" -> objMetadata("lastPublishedOn"), "pkgVersion" -> objMetadata.getOrElse("pkgVersion", 1).asInstanceOf[Number].intValue, "leafNodesCount" -> getLeafNodeCount(content), - "leafNodes" -> leafNodeIds.toArray[String], "status" -> objMetadata("status"), "lastUpdatedOn" -> objMetadata("lastUpdatedOn"), - "downloadUrl" -> objMetadata("downloadUrl"), "variants" -> objMetadata("variants")).asInstanceOf[Map[String, AnyRef]] - - // PRIMARY CATEGORY MAPPING IS DONE - setContentAndCategoryTypes(updatedContent) - } - - def publishHierarchy(children: List[Map[String, AnyRef]], obj: ObjectData, readerConfig: ExtDataConfig, config: PublishConfig)(implicit cassandraUtil: CassandraUtil): Boolean = { - val identifier = obj.identifier.replace(".img", "") - val contentConfig = config.asInstanceOf[LiveNodePublisherConfig] - val nestedFields = contentConfig.nestedFields.asScala.toList - val nodeMetadata = obj.metadata.map(property => { - property._2 match { - case propVal: String => if (nestedFields.contains(property._1)) (property._1 -> ScalaJsonUtil.deserialize[AnyRef](propVal)) else property - case _ => property - } - }) - val hierarchy: Map[String, AnyRef] = nodeMetadata ++ Map("children" -> children) - val data = Map("hierarchy" -> hierarchy) ++ obj.extData.getOrElse(Map()) - val query: Insert = QueryBuilder.insertInto(readerConfig.keyspace, readerConfig.table) - query.value(readerConfig.primaryKey.head, identifier) - data.map(d => { - readerConfig.propsMapping.getOrElse(d._1, "") match { - case "blob" => query.value(d._1.toLowerCase, QueryBuilder.fcall("textAsBlob", d._2)) - case "string" => d._2 match { - case value: String => query.value(d._1.toLowerCase, value) - case _ => query.value(d._1.toLowerCase, ScalaJsonUtil.serialize(d._2)) - } - case _ => query.value(d._1, d._2) - } - }) - logger.info(s"LiveCollectionPublisher:: publishHierarchy:: Publishing Hierarchy data for $identifier | Query : ${query.toString}") - val result = cassandraUtil.upsert(query.toString) - if (result) { - logger.info(s"LiveCollectionPublisher:: publishHierarchy:: Hierarchy data saved successfully for $identifier") - } else { - val msg = s"LiveCollectionPublisher:: publishHierarchy:: Hierarchy Data Insertion Failed For $identifier" - logger.error(msg) - throw new InvalidInputException(msg) - } - result - } - - private def updateRootChildrenList(obj: ObjectData, nextLevelNodes: List[Map[String, AnyRef]]): ObjectData = { - val childrenMap: List[Map[String, AnyRef]] = - nextLevelNodes.map(record => { - Map("identifier" -> record.getOrElse("identifier", "").asInstanceOf[String], - "name" -> record.getOrElse("name", "").asInstanceOf[String], - "objectType" -> record.getOrElse("objectType", "").asInstanceOf[String], - "description" -> record.getOrElse("description", "").asInstanceOf[String], - "index" -> record.getOrElse("index", 0).asInstanceOf[AnyRef]) - }) - - new ObjectData(obj.identifier, obj.metadata ++ Map("children" -> childrenMap, "objectType" -> "content"), obj.extData, Option(obj.hierarchy.get + ("children" -> nextLevelNodes))) - } - - def syncNodes(successObj: ObjectData, children: List[Map[String, AnyRef]], unitNodes: List[String])(implicit esUtil: ElasticSearchUtil, neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definition: ObjectDefinition, config: PublishConfig): Map[String, Map[String, AnyRef]] = { - val contentConfig = config.asInstanceOf[LiveNodePublisherConfig] - val nestedFields = contentConfig.nestedFields.asScala.toList - val nodes = ListBuffer.empty[ObjectData] - val nodeIds = ListBuffer.empty[String] - - getNodeForSyncing(children, nodes, nodeIds) - logger.info("LiveCollectionPublisher:: syncNodes:: after getNodeForSyncing:: nodes:: " + nodes.toList + " || nodeIds:: " + nodeIds) - logger.info("LiveCollectionPublisher:: syncNodes:: unitNodes:: " + unitNodes) - - // Filtering for removed nodes from Live version. nodeIds is list of nodes from Draft version. unitNodes is list of nodes from Live version. - val orphanUnitNodes = if (unitNodes.nonEmpty) unitNodes.filter(unitNode => !nodeIds.contains(unitNode)) else unitNodes - logger.info("LiveCollectionPublisher:: syncNodes:: after getNodeForSyncing:: orphanUnitNodes:: " + orphanUnitNodes) - - if (nodes.isEmpty && orphanUnitNodes.isEmpty) return Map.empty - - val errors = mutable.Map.empty[String, String] - val messages: Map[String, Map[String, AnyRef]] = getMessages(nodes.toList, definition, nestedFields, errors)(esUtil) - logger.info("LiveCollectionPublisher:: syncNodes:: after getMessages:: messages:: " + messages) - if (errors.nonEmpty) logger.error("LiveCollectionPublisher:: syncNodes:: Error! while forming ES document data from nodes, below nodes are ignored: " + errors) - if (messages.nonEmpty) - try { - logger.info("LiveCollectionPublisher:: syncNodes:: Number of units to be synced : " + messages.size + " || " + messages.keySet) - esUtil.bulkIndexWithIndexId(contentConfig.compositeSearchIndexName, contentConfig.compositeSearchIndexType, messages) - logger.info("LiveCollectionPublisher:: syncNodes:: UnitIds synced : " + messages.keySet) - } catch { - case e: Exception => e.printStackTrace() - logger.error("LiveCollectionPublisher:: syncNodes:: Elastic Search indexing failed: " + e) - } - - try //Unindexing not utilized units - if (orphanUnitNodes.nonEmpty) orphanUnitNodes.map(unitNodeId => esUtil.deleteDocument(unitNodeId)) - catch { - case e: Exception => - logger.error("LiveCollectionPublisher:: syncNodes:: Elastic Search indexing failed: " + e) - } - - // Syncing collection metadata - val doc: Map[String, AnyRef] = getDocument(new ObjectData(successObj.identifier, successObj.metadata.-("children"), successObj.extData, successObj.hierarchy), true, nestedFields)(esUtil) - val jsonDoc: String = ScalaJsonUtil.serialize(doc) - logger.info("LiveCollectionPublisher:: syncNodes:: collection doc: " + jsonDoc) - esUtil.addDocument(successObj.identifier, jsonDoc) - - messages - } - - private def getNodeForSyncing(children: List[Map[String, AnyRef]], nodes: ListBuffer[ObjectData], nodeIds: ListBuffer[String])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig): Unit = { - if (children.nonEmpty) { - children.foreach((child: Map[String, AnyRef]) => { - try { - if (StringUtils.equalsIgnoreCase("Parent", child.getOrElse("visibility", "").asInstanceOf[String])) { - logger.info("LiveCollectionPublisher:: getNodeForSyncing:: child identifier: " + child.getOrElse("identifier", "").asInstanceOf[String]) - - val nodeMetadata = mutable.Map() ++ child - - // TODO - Relation related CODE is MISSING - Line 735 in Publish Finalizer - - if (nodeMetadata.getOrElse("objectType", "").asInstanceOf[String].isEmpty) { - nodeMetadata += ("objectType" -> "Collection") - } - if (nodeMetadata.getOrElse("graphId", "").asInstanceOf[String].isEmpty) { - nodeMetadata += ("graph_id" -> "domain") - } - - if (nodeMetadata.contains("children")) nodeMetadata.remove("children") - - logger.info("LiveCollectionPublisher:: getNodeForSyncing:: nodeMetadata: " + nodeMetadata) - - if (!nodeIds.contains(child.getOrElse("identifier", "").asInstanceOf[String])) { - nodes += new ObjectData(child.getOrElse("identifier", "").asInstanceOf[String], nodeMetadata.toMap[String, AnyRef], Option(Map.empty[String, AnyRef]), Option(Map.empty[String, AnyRef])) - nodeIds += child.getOrElse("identifier", "").asInstanceOf[String] - } - - getNodeForSyncing(child.getOrElse("children", List.empty).asInstanceOf[List[Map[String, AnyRef]]], nodes, nodeIds) - } - } catch { - case e: Exception => logger.error("LiveCollectionPublisher:: getNodeForSyncing:: Error while generating node map. ", e) - } - }) - } - } - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveContentPublisher.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveContentPublisher.scala deleted file mode 100644 index d74ba18ee..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveContentPublisher.scala +++ /dev/null @@ -1,230 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers - -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils -import org.apache.tika.Tika -import org.neo4j.driver.v1.StatementResult -import org.slf4j.LoggerFactory -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers._ -import org.sunbird.job.util.CSPMetaUtil.updateRelativePath -import org.sunbird.job.util._ - -import java.io.{File, IOException} -import java.nio.file.Files -import java.util -import java.util.regex.Pattern -import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer -import scala.concurrent.ExecutionContext - -trait LiveContentPublisher extends LiveObjectReader with ObjectValidator with ObjectEnrichment with EcarGenerator with LiveObjectUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[LiveContentPublisher]) - private val level4MimeTypes = List(MimeType.X_Youtube, MimeType.PDF, MimeType.MSWORD, MimeType.EPUB, MimeType.H5P_Archive, MimeType.X_URL) - private val level4ContentTypes = List("Course", "CourseUnit", "LessonPlan", "LessonPlanUnit") - private val pragmaMimeTypes = List(MimeType.X_Youtube, MimeType.PDF) // Ne to check for other mimetype - private val youtubeMimetypes = List(MimeType.X_Youtube, MimeType.Youtube) - private val validateArtifactUrlMimetypes = List(MimeType.PDF, MimeType.EPUB, MimeType.MSWORD) - private val ignoreValidationMimeType = List(MimeType.Collection, MimeType.Plugin_Archive, MimeType.ASSETS) - private val YOUTUBE_REGEX = "^(http(s)?:\\/\\/)?((w){3}.)?youtu(be|.be)?(\\.com)?\\/.+" - - def getExtData(identifier: String, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] = { - mimeType match { - case MimeType.ECML_Archive => - val ecmlBody = getContentBody(identifier, readerConfig) - Some(ObjectExtData(Some(Map[String, AnyRef]("body" -> ecmlBody)))) - case _ => - None - } - } - - override def getHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = None - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, - cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, - definitionConfig: DefinitionConfig): Option[ObjectData] = { - val contentConfig = config.asInstanceOf[LiveNodePublisherConfig] - val extraMeta = Map("pkgVersion" -> (obj.pkgVersion + 1).asInstanceOf[AnyRef], "lastPublishedOn" -> getTimeStamp, - "flagReasons" -> null, "body" -> null, "publishError" -> null, "variants" -> null, "downloadUrl" -> null) - val contentSize = obj.metadata.getOrElse("size", 0).toString.toDouble - val configSize = contentConfig.artifactSizeForOnline - val publishType = obj.getString("publish_type", "Public") - val status = if (StringUtils.equalsIgnoreCase("Unlisted", publishType)) "Unlisted" else "Live" - val updatedMeta: Map[String, AnyRef] = if (contentSize > configSize) obj.metadata ++ extraMeta ++ Map("contentDisposition" -> "online-only", "status" -> status) else obj.metadata ++ extraMeta ++ Map("status" -> status) - - val updatedCompatibilityLevel = setCompatibilityLevel(obj, updatedMeta).getOrElse(updatedMeta) - val updatedPragma = setPragma(obj, updatedCompatibilityLevel).getOrElse(updatedCompatibilityLevel) - - //delete basePath if exists - Files.deleteIfExists(new File(ExtractableMimeTypeHelper.getBasePath(obj.identifier, contentConfig.bundleLocation)).toPath) - - try { - if (contentConfig.isECARExtractionEnabled && contentConfig.extractableMimeTypes.contains(obj.mimeType)) { - ExtractableMimeTypeHelper.copyExtractedContentPackage(obj, contentConfig, "version", cloudStorageUtil) - ExtractableMimeTypeHelper.copyExtractedContentPackage(obj, contentConfig, "latest", cloudStorageUtil) - } - } catch { - case _:Exception => neo4JUtil.executeQuery(s"""MATCH (n:domain{IL_UNIQUE_ID:"${obj.identifier}"}) SET n.migrationVersion=0.5;""") - throw new InvalidInputException(s"Invalid input found For $obj.identifier") - } - val updatedPreviewUrl = updatePreviewUrl(obj, updatedPragma, neo4JUtil, cloudStorageUtil, contentConfig).getOrElse(updatedPragma) - Some(new ObjectData(obj.identifier, updatedPreviewUrl, obj.extData, obj.hierarchy)) - } - - override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = { - Some(List(obj.metadata ++ obj.extData.getOrElse(Map()).filter(p => !excludeBundleMeta.contains(p._1)))) - } - - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - def validateMetadata(obj: ObjectData, identifier: String, config: PublishConfig): List[String] = { - logger.info("Validating Content metadata for : " + obj.identifier) - val messages = ListBuffer[String]() - val artifactUrl = obj.getString("artifactUrl", "") - if (ignoreValidationMimeType.contains(obj.mimeType)) { - // Validation not required. Nothing to do. - } else if (obj.mimeType.equalsIgnoreCase(MimeType.ECML_Archive)) { // Either 'body' or 'artifactUrl' is needed - if ((obj.extData.isEmpty || !obj.extData.get.contains("body") || obj.extData.get.getOrElse("body",null) == null || obj.extData.get.getOrElse("body", "").asInstanceOf[String].isEmpty || obj.extData.get.getOrElse("body", "").asInstanceOf[String].isBlank) && (artifactUrl.isEmpty || artifactUrl.isBlank)) { - messages += s"""Either 'body' or 'artifactUrl' are required for processing of ECML content for : $identifier""" - } - } else { - val allowedExtensionsWord: util.List[String] = config.asInstanceOf[LiveNodePublisherConfig].allowedExtensionsWord - - if (StringUtils.isBlank(artifactUrl)) - messages += s"""There is no artifactUrl available for : $identifier""" - else if (youtubeMimetypes.contains(obj.mimeType) && !isValidYouTubeUrl(artifactUrl)) - messages += s"""Invalid youtube Url = $artifactUrl for : $identifier""" - else if (validateArtifactUrlMimetypes.contains(obj.mimeType) && !isValidUrl(artifactUrl, obj.mimeType, allowedExtensionsWord)) { // valid url check by downloading the file and then delete it - // artifactUrl + valid url check by downloading the file - obj.mimeType match { - case MimeType.PDF => - messages += s"""Error! Invalid File Extension. Uploaded file $artifactUrl is not a pdf file for : $identifier""" - case MimeType.EPUB => - messages += s"""Error! Invalid File Extension. Uploaded file $artifactUrl is not a epub file for : $identifier""" - case MimeType.MSWORD => - messages += s"""Error! Invalid File Extension. | Uploaded file $artifactUrl should be among the Allowed_file_extensions for mimeType doc $allowedExtensionsWord for : $identifier""" - } - } - } - messages.toList - } - - private def isValidYouTubeUrl(artifactUrl: String): Boolean = { - logger.info(s"Validating if the given youtube url = $artifactUrl is valid or not.") - Pattern.matches(YOUTUBE_REGEX, artifactUrl) - } - - private def isValidUrl(url: String, mimeType: String, allowedExtensionsWord: util.List[String]): Boolean = { - val destPath = s"""${File.separator}tmp${File.separator}validUrl""" - var isValid = false - try { - val file: File = FileUtils.downloadFile(url, destPath) - if (exceptionChecks(mimeType, file, allowedExtensionsWord)) isValid = true - } catch { - case e: Exception => - logger.error("isValidUrl: Error while checking mimeType.") - e.printStackTrace() - throw new InvalidInputException("isValidUrl: Error while checking mimeType ", e) - } finally { - FileUtils.deleteQuietly(destPath) - } - isValid - } - - private def exceptionChecks(mimeType: String, file: File, allowedExtensionsWord: util.List[String]): Boolean = { - try { - val extension = FilenameUtils.getExtension(file.getPath) - logger.info("Validating File For MimeType: " + file.getName) - if (extension.nonEmpty) { - val tika = new Tika - val fileType = tika.detect(file) - mimeType match { - case MimeType.PDF => - if (StringUtils.equalsIgnoreCase(extension, "pdf") && fileType == MimeType.PDF) return true - case MimeType.EPUB => - if (StringUtils.equalsIgnoreCase(extension, "epub") && fileType == "application/epub+zip") return true - case MimeType.MSWORD => - if (allowedExtensionsWord.contains(extension)) return true - } - } - } catch { - case e: IOException => - logger.error("exceptionChecks: Error while checking mimeType.") - e.printStackTrace() - throw new InvalidInputException("exceptionChecks: Error while checking mimeType ", e) - } - false - } - - def getObjectWithEcar(data: ObjectData, pkgTypes: List[String])(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig, httpUtil: HttpUtil): ObjectData = { - try { - logger.info("LiveContentPublisher :: getObjectWithEcar:: Ecar generation done for Content: " + data.identifier) - val ecarMap: Map[String, String] = generateEcar(data, pkgTypes) - val variants: java.util.Map[String, java.util.Map[String, String]] = ecarMap.map { case (key, value) => key.toLowerCase -> Map[String, String]("ecarUrl" -> value, "size" -> httpUtil.getSize(value).toString).asJava }.asJava - logger.info("LiveContentPublisher :: getObjectWithEcar :: ecar map :: " + ecarMap) - val meta: Map[String, AnyRef] = Map("downloadUrl" -> ecarMap.getOrElse(EcarPackageType.FULL, ""), "variants" -> variants) - new ObjectData(data.identifier, data.metadata ++ meta, data.extData, data.hierarchy) - } catch { - case _: java.lang.IllegalArgumentException => throw new InvalidInputException(s"Invalid input found For $data.identifier") - case iex@(_: java.lang.InterruptedException | _:java.io.FileNotFoundException) => - neo4JUtil.executeQuery(s"""MATCH (n:domain{IL_UNIQUE_ID:"${data.identifier}"}) SET n.migrationVersion=0.5;""") - throw new InvalidInputException(s"Invalid input found For $data.identifier") - } - } - - private def setCompatibilityLevel(obj: ObjectData, updatedMeta: Map[String, AnyRef]): Option[Map[String, AnyRef]] = { - if (level4MimeTypes.contains(obj.mimeType) - || level4ContentTypes.contains(obj.getString("contentType", ""))) { - logger.info("setting compatibility level for content id : " + obj.identifier + " as 4.") - Some(updatedMeta ++ Map("compatibilityLevel" -> 4.asInstanceOf[AnyRef])) - } else None - } - - private def setPragma(obj: ObjectData, updatedMeta: Map[String, AnyRef]): Option[Map[String, AnyRef]] = { - if (pragmaMimeTypes.contains(obj.mimeType)) { - val pgm: java.util.List[String] = obj.metadata.getOrElse("pragma", new java.util.ArrayList[String]()).asInstanceOf[java.util.List[String]] - val pragma: List[String] = pgm.asScala.toList - val value = "external" - if (!pragma.contains(value)) { - Some(updatedMeta ++ Map("pragma" -> (pragma ++ List(value)))) - } else None - } else None - } - - private def updatePreviewUrl(obj: ObjectData, updatedMeta: Map[String, AnyRef], neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, config: LiveNodePublisherConfig): Option[Map[String, AnyRef]] = { - try { - if (StringUtils.isNotBlank(obj.mimeType)) { - logger.debug("Checking Required Fields For: " + obj.mimeType) - obj.mimeType match { - case MimeType.Collection | MimeType.Plugin_Archive | MimeType.Android_Package | MimeType.ASSETS => - None - case MimeType.ECML_Archive | MimeType.HTML_Archive | MimeType.H5P_Archive => - val latestFolderS3Url = ExtractableMimeTypeHelper.getCloudStoreURL(obj, cloudStorageUtil, config) - val relativeLatestFolder = if(config.isrRelativePathEnabled) StringUtils.replaceEach(latestFolderS3Url, config.config.getStringList("cloudstorage.write_base_path").asScala.toArray, Array(config.getString("cloudstorage.read_base_path", ""))) else latestFolderS3Url - val updatedPreviewUrl = updatedMeta ++ Map("previewUrl" -> relativeLatestFolder, "streamingUrl" -> latestFolderS3Url) - Some(updatedPreviewUrl) - case _ => - val artifactUrl = obj.getString("artifactUrl", null) - val updatedPreviewUrl = updatedMeta ++ Map("previewUrl" -> artifactUrl) - if (config.isStreamingEnabled && !config.streamableMimeType.contains(obj.mimeType)) Some(updatedPreviewUrl ++ Map("streamingUrl" -> artifactUrl)) else Some(updatedPreviewUrl) - } - } else None - } catch { - case ex: Exception => logger.debug("Exception for updatePreviewUrl: " + ex.getMessage) - val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"${obj.identifier}"}) SET n.migrationVersion=0.5;""" - val result: StatementResult = neo4JUtil.executeQuery(query) - None - } - } -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveObjectReader.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveObjectReader.scala deleted file mode 100644 index c7c331981..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveObjectReader.scala +++ /dev/null @@ -1,39 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} - -import scala.collection.JavaConverters._ - -trait LiveObjectReader { - - private[this] val logger = LoggerFactory.getLogger(classOf[LiveObjectReader]) - - def getObject(identifier: String, pkgVersion: Double, mimeType: String, publishType: String, readerConfig: ExtDataConfig)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, config: PublishConfig): ObjectData = { - logger.info("Reading editable object data for: " + identifier + " with pkgVersion: " + pkgVersion) - val metadata = getMetadata(identifier, mimeType, publishType, pkgVersion) - logger.info("Reading metadata for: " + identifier + " with metadata: " + metadata) - val extData = getExtData(identifier, mimeType, readerConfig) - logger.info("Reading extData for: " + identifier + " with extData: " + extData) - new ObjectData(identifier, metadata, extData.getOrElse(ObjectExtData()).data, extData.getOrElse(ObjectExtData()).hierarchy) - } - - private def getMetadata(identifier: String, mimeType: String, publishType: String, pkgVersion: Double)(implicit neo4JUtil: Neo4JUtil): Map[String, AnyRef] = { - val metaData = neo4JUtil.getNodeProperties(identifier).asScala.toMap - val id = metaData.getOrElse("IL_UNIQUE_ID", identifier).asInstanceOf[String] - val objType = metaData.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String] - logger.info("ObjectReader:: getMetadata:: identifier: " + identifier + " with objType: " + objType) - metaData ++ Map[String, AnyRef]("identifier" -> id, "objectType" -> objType, "publish_type" -> publishType) - ("IL_UNIQUE_ID", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE") - } - - def getExtData(identifier: String, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] - - def getHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] - - def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] - - def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveObjectUpdater.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveObjectUpdater.scala deleted file mode 100644 index b9d852ffe..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/LiveObjectUpdater.scala +++ /dev/null @@ -1,147 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers - -import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} -import org.apache.commons.lang3.StringUtils -import org.neo4j.driver.v1.StatementResult -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.util.{CSPMetaUtil, CassandraUtil, JSONUtil, Neo4JUtil, ScalaJsonUtil} - -import java.text.SimpleDateFormat -import java.util -import java.util.Date - -trait LiveObjectUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[LiveObjectUpdater]) - - @throws[Exception] - def saveOnSuccess(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig, config: PublishConfig): Unit = { - val publishType = obj.getString("publish_type", "Public") - val status = if (StringUtils.equalsIgnoreCase("Unlisted", publishType)) "Unlisted" else "Live" - val identifier = obj.identifier - val migrationVersion: Double = if(config.getConfig().hasPath("migrationVersion")) config.getConfig().getDouble("migrationVersion") else 1.0 + 0.1 - val metadataUpdateQuery = metaDataQuery(obj)(definitionCache, definitionConfig) - val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"$identifier"}) SET n.status="$status",n.pkgVersion=${obj.pkgVersion},n.prevStatus="Processing",n.migrationVersion=$migrationVersion,$metadataUpdateQuery,$auditPropsUpdateQuery;""" - logger.info("ObjectUpdater:: saveOnSuccess:: Query: " + query) - logger.info(s"ObjectUpdater:: saveOnSuccess:: DB ID for ${obj.identifier} is : ${obj.dbId} | pkgVersion : ${obj.pkgVersion}" ) - - if (obj.mimeType.equalsIgnoreCase("application/vnd.ekstep.ecml-archive")) { - val ecmlBody = getContentBody(identifier, readerConfig) - updateContentBody(identifier,ecmlBody,readerConfig) - } - - val result: StatementResult = neo4JUtil.executeQuery(query) - if (null != result && result.hasNext) - logger.info(s"ObjectUpdater:: saveOnSuccess:: statement result : ${result.next().asMap()}") - saveExternalData(obj, readerConfig) - } - -// @throws[Exception] -// def updateProcessingNode(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definitionCache: DefinitionCache, config: DefinitionConfig): Unit = { -// val status = "Processing" -// val prevState = obj.getString("status", "Draft") -// val identifier = obj.dbId -// val metadataUpdateQuery = metaDataQuery(obj)(definitionCache, config) -// val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"$identifier"}) SET n.status="$status",n.prevState="$prevState",$metadataUpdateQuery,$auditPropsUpdateQuery;""" -// logger.info("ObjectUpdater:: updateProcessingNode:: Query: " + query) -// val result: StatementResult = neo4JUtil.executeQuery(query) -// if (null != result && result.hasNext) -// logger.info(s"ObjectUpdater:: updateProcessingNode:: statement result : ${result.next().asMap()}") -// } - - def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit - - def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit - - @throws[Exception] - def saveOnFailure(obj: ObjectData, messages: List[String], pkgVersion: Double)(implicit neo4JUtil: Neo4JUtil): Unit = { - val errorMessages = messages.mkString("; ") - val nodeId = obj.dbId - val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"$nodeId"}) SET n.pkgVersion=${obj.pkgVersion},n.migrationVersion=0.2, n.publishError="$errorMessages", $auditPropsUpdateQuery;""" - logger.info("ObjectUpdater:: saveOnFailure:: Query: " + query) - neo4JUtil.executeQuery(query) - } - - def metaDataQuery(obj: ObjectData)(definitionCache: DefinitionCache, config: DefinitionConfig): String = { - val version = config.supportedVersion.getOrElse(obj.dbObjType.toLowerCase(), "1.0").asInstanceOf[String] - val definition = definitionCache.getDefinition(obj.dbObjType, version, config.basePath) - val metadata = obj.metadata - ("IL_UNIQUE_ID", "identifier", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE", "pkgVersion", "lastStatusChangedOn", "lastUpdatedOn", "status", "objectType", "publish_type", "migrationVersion") - metadata.map(prop => { - if (null == prop._2) s"n.${prop._1}=${prop._2}" - else if (definition.objectTypeProperties.contains(prop._1)) { - prop._2 match { - case _: Map[String, AnyRef] => - val strValue = JSONUtil.serialize(ScalaJsonUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: util.Map[String, AnyRef] => - val strValue = JSONUtil.serialize(JSONUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _ => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - } - } else { - prop._2 match { - case _: Map[String, AnyRef] => - val strValue = JSONUtil.serialize(ScalaJsonUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: util.Map[String, AnyRef] => - val strValue = JSONUtil.serialize(JSONUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: List[String] => - val strValue = ScalaJsonUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - case _: util.List[String] => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - case _ => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - } - } - }).mkString(",") - } - - private def auditPropsUpdateQuery(): String = { - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ") - val updatedOn = sdf.format(new Date()) - s"""n.lastUpdatedOn="$updatedOn",n.lastStatusChangedOn="$updatedOn"""" - } - - def getContentBody(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): String = { - // fetch content body from cassandra - val selectId = QueryBuilder.select() - selectId.fcall("blobAsText", QueryBuilder.column("body")).as("body") - val selectWhereId: Select.Where = selectId.from(readerConfig.keyspace, readerConfig.table).where().and(QueryBuilder.eq("content_id", identifier)) - logger.info("ObjectUpdater:: getContentBody:: Cassandra Fetch Query :: " + selectWhereId.toString) - val rowId = cassandraUtil.findOne(selectWhereId.toString) - if (null != rowId) { - val body = rowId.getString("body") - val updatedBody = if (isrRelativePathEnabled(config)) CSPMetaUtil.updateAbsolutePath(body) else body - updatedBody - } else "" - } - - private def isrRelativePathEnabled(config: PublishConfig): Boolean = { - config.getBoolean("cloudstorage.metadata.replace_absolute_path", false) - } - - def updateContentBody(identifier: String, ecmlBody: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val updateQuery = QueryBuilder.update(readerConfig.keyspace, readerConfig.table) - .where(QueryBuilder.eq("content_id", identifier)) - .`with`(QueryBuilder.set("body", QueryBuilder.fcall("textAsBlob", ecmlBody))) - logger.info(s"ObjectUpdater:: updateContentBody:: Updating Content Body in Cassandra For $identifier : ${updateQuery.toString}") - val result = cassandraUtil.upsert(updateQuery.toString) - if (result) logger.info(s"ObjectUpdater:: updateContentBody:: Content Body Updated Successfully For $identifier") - else { - logger.error(s"ObjectUpdater:: updateContentBody:: Content Body Update Failed For $identifier") - throw new InvalidInputException(s"Content Body Update Failed For $identifier") - } - } - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/SyncMessagesGenerator.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/SyncMessagesGenerator.scala deleted file mode 100644 index 51a7a92f1..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/helpers/SyncMessagesGenerator.scala +++ /dev/null @@ -1,167 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.ObjectDefinition -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} - -import scala.collection.mutable -import scala.collection.mutable.ListBuffer - -trait SyncMessagesGenerator { - - private[this] val logger = LoggerFactory.getLogger(classOf[SyncMessagesGenerator]) - - private def getIndexDocument(identifier: String)(esUtil: ElasticSearchUtil): scala.collection.mutable.Map[String, AnyRef] = { - val documentJson: String = esUtil.getDocumentAsString(identifier) - if (documentJson != null && documentJson.nonEmpty) ScalaJsonUtil.deserialize[scala.collection.mutable.Map[String, AnyRef]](documentJson) else scala.collection.mutable.Map[String, AnyRef]() - } - - private def getJsonMessage(message: Map[String, Any], definition: ObjectDefinition, nestedFields: List[String], ignoredFields: List[String]): Map[String, AnyRef] = { - val indexDocument = scala.collection.mutable.Map[String, AnyRef]() - val transactionData = message.getOrElse("transactionData", Map[String, Any]()).asInstanceOf[Map[String, Any]] - logger.debug("SyncMessagesGenerator:: getJsonMessage:: transactionData:: " + transactionData) - if (transactionData.nonEmpty) { - val addedProperties = transactionData.getOrElse("properties", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - logger.debug("SyncMessagesGenerator:: getJsonMessage:: definition.externalProperties:: " + definition.externalProperties) - addedProperties.foreach(property => { - if (!definition.externalProperties.contains(property._1)) { - val propertyNewValue: AnyRef = property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) - if (propertyNewValue == null) indexDocument.remove(property._1) else indexDocument.put(property._1, addMetadataToDocument(property._1, propertyNewValue, nestedFields)) - } - }) - -// val addedRelations = transactionData.getOrElse("addedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] -// if (addedRelations.nonEmpty) { -// addedRelations.foreach(rel => { -// val direction = rel.getOrElse("dir", "").asInstanceOf[String] -// val relationType = rel.getOrElse("rel", "").asInstanceOf[String] -// val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] -// val title = definition.relationLabel(targetObjType, direction, relationType) -// if (title.nonEmpty) { -// val list = indexDocument.getOrElse(title.get, List[String]()).asInstanceOf[List[String]] -// val id = rel.getOrElse("id", "").asInstanceOf[String] -// if (!list.contains(id)) indexDocument.put(title.get, (id :: list).asInstanceOf[AnyRef]) -// } -// }) -// } -// -// val removedRelations = transactionData.getOrElse("removedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] -// removedRelations.foreach(rel => { -// val direction = rel.getOrElse("dir", "").asInstanceOf[String] -// val relationType = rel.getOrElse("rel", "").asInstanceOf[String] -// val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] -// val title = definition.relationLabel(targetObjType, direction, relationType) -// if (title.nonEmpty) { -// val list = indexDocument.getOrElse(title.get, List[String]()).asInstanceOf[List[String]] -// val id = rel.getOrElse("id", "").asInstanceOf[String] -// if (list.contains(id)) { -// val updatedList = list diff List(id) -// indexDocument.put(title.get, updatedList.asInstanceOf[AnyRef]) -// } -// } -// }) - } - - //Ignored fields are removed-> it can be a propertyName or relation Name - indexDocument --= ignoredFields - - indexDocument.put("graph_id", message.getOrElse("graphId", "domain").asInstanceOf[String]) - indexDocument.put("node_id", message.getOrElse("nodeGraphId",0).asInstanceOf[AnyRef]) - indexDocument.put("identifier", message.getOrElse("nodeUniqueId", "").asInstanceOf[String]) - indexDocument.put("objectType", message.getOrElse("objectType", "").asInstanceOf[String]) - indexDocument.put("nodeType", message.getOrElse("nodeType", "").asInstanceOf[String]) - - logger.info("SyncMessagesGenerator:: getJsonMessage:: final indexDocument:: " + indexDocument) - - indexDocument.toMap - } - - private def addMetadataToDocument(propertyName: String, propertyValue: AnyRef, nestedFields: List[String]): AnyRef = { - if (nestedFields.contains(propertyName)) { - propertyValue match { - case propVal: String => ScalaJsonUtil.deserialize[AnyRef](propVal) - case _ => propertyValue - } - } else propertyValue - } - - def getMessages(nodes: List[ObjectData], definition: ObjectDefinition, nestedFields: List[String], errors: mutable.Map[String, String])(esUtil: ElasticSearchUtil): Map[String, Map[String, AnyRef]] = { - val messages = collection.mutable.Map.empty[String, Map[String, AnyRef]] - for (node <- nodes) { - try { - if (definition.getRelationLabels() != null) { - val nodeMap = getNodeMap(node) - logger.debug("SyncMessagesGenerator:: getMessages:: nodeMap:: " + nodeMap) - val message = getJsonMessage(nodeMap, definition, nestedFields, List.empty) - logger.debug("SyncMessagesGenerator:: getMessages:: message:: " + message) - messages.put(node.identifier, message) - } - } catch { - case e: Exception => e.printStackTrace() - errors.put(node.identifier, e.getMessage) - } - } - messages.toMap - } - - - private def getNodeMap(node: ObjectData): Map[String, AnyRef] = { - val transactionData = collection.mutable.Map.empty[String, AnyRef] - if (null != node.metadata && node.metadata.nonEmpty) { - val propertyMap = collection.mutable.Map.empty[String, AnyRef] - for (key <- node.metadata.keySet) { - if (StringUtils.isNotBlank(key)) { - val valueMap = collection.mutable.Map.empty[String, AnyRef] - valueMap.put("ov", null) // old value - valueMap.put("nv", node.metadata(key)) // new value - - // temporary check to not sync body and editorState - if (!StringUtils.equalsIgnoreCase("body", key) && !StringUtils.equalsIgnoreCase("editorState", key)) propertyMap.put(key, valueMap.toMap) - } - } - transactionData.put("properties", propertyMap.toMap) - } - else transactionData.put("properties", Map.empty[String,AnyRef]) - - val relations = ListBuffer.empty[Map[String, AnyRef]] - // add IN relations -// if (null != node.metadata.inRelations && node.metadata.inRelations.nonEmpty) { -// for (rel <- node.metadata.inRelations) { -// val relMap = Map("rel" -> rel.relationType, "id" -> rel.startNodeId, "dir" -> "IN", "type" -> rel.startNodeObjectType, "label" -> getLabel(rel.getStartNodeMetadata)) -// relations += relMap -// } -// } -// // add OUT relations -// if (null != node.getOutRelations && !node.getOutRelations.isEmpty) { -// for (rel <- node.getOutRelations) { -// val relMap = Map("rel" -> rel.getRelationType, "id" -> rel.getEndNodeId, "dir" -> "OUT", "type" -> rel.getEndNodeObjectType, "label" -> getLabel(rel.getEndNodeMetadata)) -// relations += relMap -// } -// } - transactionData.put("addedRelations", relations.toList) - - Map("operationType"-> "UPDATE", "graphId" -> node.metadata.getOrElse("graphId","domain").asInstanceOf[String], "nodeGraphId"-> 0.asInstanceOf[AnyRef], "nodeUniqueId"-> node.identifier, "objectType"-> node.dbObjType, - "nodeType"-> "DATA_NODE", "transactionData" -> transactionData.toMap, "syncMessage" -> true.asInstanceOf[AnyRef]) - } - - - def getDocument(node: ObjectData, updateRequest: Boolean, nestedFields: List[String])(esUtil: ElasticSearchUtil): Map[String, AnyRef] = { - val message = getNodeMap(node) - val identifier: String = message.getOrElse("nodeUniqueId", "").asInstanceOf[String] - val indexDocument = mutable.Map[String, AnyRef]() - val transactionData: Map[String, AnyRef] = message.getOrElse("transactionData", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - if (transactionData.nonEmpty) { - val addedProperties: Map[String, AnyRef] = transactionData.getOrElse("properties", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - addedProperties.foreach(property => { - val propertyNewValue: AnyRef = property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) - if (propertyNewValue == null) indexDocument.remove(property._1) else indexDocument.put(property._1, addMetadataToDocument(property._1, propertyNewValue, nestedFields)) - }) - } - indexDocument.put("identifier", message.getOrElse("nodeUniqueId", "").asInstanceOf[String]) - indexDocument.put("objectType", message.getOrElse("objectType", "").asInstanceOf[String]) - indexDocument.toMap - } - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/BaseProcessor.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/BaseProcessor.scala deleted file mode 100644 index 4599b62df..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/BaseProcessor.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.processor - -import org.sunbird.job.util.CloudStorageUtil - -class BaseProcessor(basePath: String, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil) extends IProcessor(basePath, identifier) { - override def process(ecrf: Plugin): Plugin = { - ecrf - } -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/EcrfObject.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/EcrfObject.scala deleted file mode 100644 index 59e7500c9..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/EcrfObject.scala +++ /dev/null @@ -1,19 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.processor - -case class Plugin(id: String, data: Map[String, AnyRef], innerText: String, cData: String, childrenPlugin: List[Plugin], manifest: Manifest, controllers: List[Controller], events: List[Event]) { - def this() = this("", null, "", "", null, null, null, null) -} -case class Manifest(id: String, data: Map[String, AnyRef], innerText: String, cData: String, medias: List[Media]) { - def this() = this("", null, "", "", null) -} -case class Controller(id: String, data: Map[String, AnyRef], innerText: String, cData: String) { - def this() = this("", null, "", "") -} -case class Media(id: String, data: Map[String, AnyRef], innerText: String, cData: String, src: String, `type`: String, childrenPlugin: List[Plugin]) { - def this() = this("", null, "", "", "", "", null) -} -case class Event(id: String, data: Map[String, AnyRef], innerText: String, cData: String, childrenPlugin: List[Plugin]) { - def this() = this("", null, "", "", null) -} - - diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/IProcessor.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/IProcessor.scala deleted file mode 100644 index 839d55aa2..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/IProcessor.scala +++ /dev/null @@ -1,16 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.processor - -import org.sunbird.job.util.CloudStorageUtil - -abstract class IProcessor(basePath: String, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil) { - - implicit val ss = cloudStorageUtil.getService - - val widgetTypeAssets: List[String] = List("js", "css", "json", "plugin") - - def process(ecrf: Plugin): Plugin - - def getBasePath(): String = basePath - - def getIdentifier(): String = identifier -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/JsonParser.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/JsonParser.scala deleted file mode 100644 index cc3110f1b..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/JsonParser.scala +++ /dev/null @@ -1,218 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.processor - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.util.ScalaJsonUtil - -import scala.collection.mutable.ListBuffer - -object JsonParser { - val nonPluginElements: List[String] = List("manifest", "controller", "media", "events", "event", "__cdata", "__text") - - def parse(jsonString: String): Plugin = { - val jsonMap: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](jsonString) - processDocument(jsonMap) - } - - def processDocument(json: Map[String, AnyRef]): Plugin = { - if (json.keySet.contains("theme")) { - val root = json("theme").asInstanceOf[Map[String, AnyRef]] - Plugin(getId(root), getData(root, "theme"), getInnerText(root), getCdata(root), getChildrenPlugin(root), getManifest(root, validateMedia = true), getControllers(root), getEvents(root)) - } else classOf[Plugin].newInstance() - } - - private def getDataFromMap(map: Map[String, AnyRef], key: String): String = { - if (null != map && map.keySet.contains(key)) { - map(key) match { - case str: String => str - case _ => map(key).toString - } - } else "" - } - - def getId(jsonObject: Map[String, AnyRef]): String = getDataFromMap(jsonObject, "id") - - def getData(jsonObject: Map[String, AnyRef], elementName: String): Map[String, AnyRef] = { - if (null != jsonObject && StringUtils.isNotBlank(elementName)) { - var result = jsonObject.filter(p => !p._1.equalsIgnoreCase("__cdata") && !p._1.equalsIgnoreCase("__text")) - result += ("cwp_element_name" -> elementName) - result - } else Map[String, AnyRef]() - } - - def getInnerText(jsonObject: Map[String, AnyRef]): String = getDataFromMap(jsonObject, "__text") - - def getCdata(jsonObject: Map[String, AnyRef]): String = ScalaJsonUtil.serialize(jsonObject.getOrElse("__cdata", "")) - - def getChildrenPlugin(jsonObject: Map[String, AnyRef]): List[Plugin] = { - val childPluginList: ListBuffer[Plugin] = ListBuffer() - val filteredObject = jsonObject.filter(entry => null != entry._2) - childPluginList ++= filteredObject.filter(entry => entry._2.isInstanceOf[List[Map[String, AnyRef]]] && !nonPluginElements.contains(entry._1)).map(entry => { - val objectList: List[Map[String, AnyRef]] = entry._2.asInstanceOf[List[Map[String, AnyRef]]] - objectList.map(obj => Plugin(getId(obj), getData(obj, entry._1), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj), getManifest(obj, validateMedia = false), getControllers(obj), getEvents(obj))) - }).toList.flatten - childPluginList ++= filteredObject.filter(entry => entry._2.isInstanceOf[Map[String, AnyRef]] && !nonPluginElements.contains(entry._2)).map(entry => { - val obj = entry._2.asInstanceOf[Map[String, AnyRef]] - Plugin(getId(obj), getData(obj, entry._1), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj), getManifest(obj, validateMedia = false), getControllers(obj), getEvents(obj)) - }).toList - childPluginList.toList - } - - def getManifest(jsonObject: Map[String, AnyRef], validateMedia: Boolean): Manifest = { - if (null != jsonObject && jsonObject.keySet.contains("manifest") && jsonObject("manifest").isInstanceOf[List[Map[String, AnyRef]]]) throw new Exception("Error! JSON Object is Expected for the Element. manifest") - else if (null != jsonObject && jsonObject.keySet.contains("manifest") && jsonObject("manifest").isInstanceOf[Map[String, AnyRef]] && jsonObject("manifest").asInstanceOf[Map[String, AnyRef]].keySet.contains("media")) { - val manifestObject = jsonObject("manifest").asInstanceOf[Map[String, AnyRef]] - Manifest(getId(manifestObject), getData(manifestObject, "manifest"), getInnerText(manifestObject), getCdata(manifestObject), getMedias(manifestObject("media"), validateMedia)) - } else classOf[Manifest].newInstance() - } - - def getControllers(jsonObject: Map[String, AnyRef]): List[Controller] = { - if (null != jsonObject && jsonObject.keySet.contains("controller") && jsonObject("controller").isInstanceOf[List[Map[String, Object]]]) { - val controllerList: List[Map[String, AnyRef]] = jsonObject("controller").asInstanceOf[List[Map[String, Object]]] - controllerList.map(obj => { - validateController(obj) - Controller(getId(obj), getData(obj, "controller"), getInnerText(obj), getCdata(obj)) - }) - } else if (null != jsonObject && jsonObject.keySet.contains("controller") && jsonObject("controller").isInstanceOf[Map[String, Object]]) { - val obj = jsonObject("controller").asInstanceOf[Map[String, Object]] - validateController(obj) - List(Controller(getId(obj), getData(obj, "controller"), getInnerText(obj), getCdata(obj))) - } else List() - } - - def validateController(obj: Map[String, AnyRef]): Unit = { - if (StringUtils.isBlank(getDataFromMap(obj, "id"))) - throw new InvalidInputException("Error! Invalid Controller ('id' is required.)") - - val `type` = getDataFromMap(obj, "type") - if (StringUtils.isBlank(getDataFromMap(obj, "type"))) - throw new InvalidInputException("Error! Invalid Controller ('type' is required.)") - if (!"items".equalsIgnoreCase(`type`) && !"data".equalsIgnoreCase(`type`)) - throw new InvalidInputException("Error! Invalid Controller ('type' should be either 'items' or 'data')") - } - - private def getEventsFromObject(jsonObject: AnyRef): List[Event] = { - if (null != jsonObject && jsonObject.isInstanceOf[List[Map[String, AnyRef]]]) { - val jsonList: List[Map[String, AnyRef]] = jsonObject.asInstanceOf[List[Map[String, AnyRef]]] - jsonList.map(obj => Event(getId(obj), getData(obj, "event"), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj))) - } else if (null != jsonObject && jsonObject.isInstanceOf[Map[String, AnyRef]]) { - List(Event(getId(jsonObject.asInstanceOf[Map[String, AnyRef]]), getData(jsonObject.asInstanceOf[Map[String, AnyRef]], "event"), getInnerText(jsonObject.asInstanceOf[Map[String, AnyRef]]), getCdata(jsonObject.asInstanceOf[Map[String, AnyRef]]), getChildrenPlugin(jsonObject.asInstanceOf[Map[String, AnyRef]]))) - } else List() - } - - def getEvents(jsonObject: Map[String, AnyRef]): List[Event] = { - val eventList: ListBuffer[Event] = ListBuffer() - if (jsonObject != null) { - if (jsonObject.keySet.contains("events")) { - val value = jsonObject("events") - value match { - case jsonList: List[Map[String, AnyRef]] => - eventList ++= jsonList.map(obj => Event(getId(obj), getData(obj, "event"), getInnerText(obj), getCdata(obj), getChildrenPlugin(obj))) - case _: Map[String, AnyRef] => - eventList ++= getEventsFromObject(value) - case _ => - } - } else if (jsonObject.keySet.contains("event")) { - eventList ++= getEventsFromObject(jsonObject("event")) - } - } - eventList.toList - } - - def getMedias(manifestObject: AnyRef, validateMedia: Boolean): List[Media] = { - if (null != manifestObject && manifestObject.isInstanceOf[List[Map[String, AnyRef]]]) { - val jsonList: List[Map[String, AnyRef]] = manifestObject.asInstanceOf[List[Map[String, AnyRef]]] - jsonList.map(json => getMedia(json, validateMedia)) - } else if (null != manifestObject && manifestObject.isInstanceOf[Map[String, AnyRef]]) { - List(getMedia(manifestObject.asInstanceOf[Map[String, AnyRef]], validateMedia)) - } else List() - } - - def getMedia(mediaJson: Map[String, AnyRef], validateMedia: Boolean): Media = { - if (null != mediaJson) { - val id = getDataFromMap(mediaJson, "id") - val src = getDataFromMap(mediaJson, "src") - val `type` = getDataFromMap(mediaJson, "type") - if (validateMedia) { - if (StringUtils.isBlank(id) && isMediaIdRequiredForMediaType(`type`)) - throw new InvalidInputException("Error! Invalid Media ('id' is required.)") - if (StringUtils.isBlank(`type`)) - throw new InvalidInputException("Error! Invalid Media ('type' is required.)") - if (StringUtils.isBlank(src)) - throw new InvalidInputException("Error! Invalid Media ('src' is required.)") - } - Media(id, getData(mediaJson, "media"), getInnerText(mediaJson), getCdata(mediaJson), src, `type`, getChildrenPlugin(mediaJson)) - } else classOf[Media].newInstance() - } - - private def isMediaIdRequiredForMediaType(`type`: String): Boolean = { - !(StringUtils.isNotBlank(`type`) && (`type`.equalsIgnoreCase("js") || `type`.equalsIgnoreCase("css"))) - } - - /** - * serialize - * - * @param plugin - * @return - */ - def toString(plugin: Plugin): String = { - val map = plugin.data ++ Map[String, AnyRef]("__text" -> plugin.innerText, "__cdata" -> plugin.cData) ++ getManifestMap(plugin.manifest) ++ getControllersMap(plugin.controllers) ++ getEventsMap(plugin.events) - ScalaJsonUtil.serialize(Map[String, AnyRef]("theme" -> map)) - } - - def getManifestMap(manifest: Manifest): Map[String, AnyRef] = { - if (null != manifest && null != manifest.medias && manifest.medias.nonEmpty) { - val map: Map[String, AnyRef] = manifest.data ++ Map[String, AnyRef]("__text" -> manifest.innerText, "__cdata" -> manifest.cData) ++ getMediaMap(manifest.medias) - Map[String, AnyRef]("manifest" -> map) - } else Map[String, AnyRef]() - } - - def getControllersMap(controllers: List[Controller]): Map[String, AnyRef] = { - if (null != controllers && controllers.nonEmpty) { - val controllerMap: List[Map[String, AnyRef]] = controllers.map(controller => { - controller.data ++ Map[String, AnyRef]("__text" -> controller.innerText, "__cdata" -> controller.cData) - }) - Map[String, AnyRef]("controller" -> controllerMap) - } else Map[String, AnyRef]() - } - - def getEventsMap(events: List[Event]): Map[String, AnyRef] = { - if (null != events && events.nonEmpty) { - val eventsList: List[Map[String, AnyRef]] = events.map(event => event.data ++ Map[String, AnyRef]("__text" -> event.innerText, "__cdata" -> event.cData) ++ getChildPluginMap(event.childrenPlugin)) - - if (eventsList.length > 1) { - Map[String, AnyRef]("events" -> Map[String, AnyRef]("event" -> eventsList)) - } else { - Map[String, AnyRef]("event" -> eventsList.head) - } - } else Map[String, AnyRef]() - } - - def getMediaMap(medias: List[Media]): Map[String, AnyRef] = { - val mediasMap = medias.map(media => { - - val updatedMediaData = media.data.map(mediaInfo => { - if(mediaInfo._1.equalsIgnoreCase("src")) - ("src" -> (StringUtils.stripStart(mediaInfo._2.toString, "/"))) - else mediaInfo - }) - - updatedMediaData ++ Map[String, AnyRef]("__text" -> media.innerText, "__cdata" -> media.cData) ++ getChildPluginMap(media.childrenPlugin) - }) - Map[String, AnyRef]("media" -> mediasMap) - } - - def getChildPluginMap(plugins: List[Plugin]): Map[String, AnyRef] = { - if (null != plugins && plugins.nonEmpty) { - plugins.filter(plugin => StringUtils.isNotBlank(getDataFromMap(plugin.data, "cwp_element_name"))) - .groupBy(plugin => getDataFromMap(plugin.data, "cwp_element_name")) - .map(entry => { - if (entry._2.size == 1) - entry._1 -> entry._2.map(plugin => plugin.data ++ Map[String, AnyRef]("__text" -> plugin.innerText, "__cdata" -> plugin.cData) ++ getManifestMap(plugin.manifest) ++ getControllersMap(plugin.controllers) ++ getEventsMap(plugin.events)).head - else - entry._1 -> entry._2.map(plugin => plugin.data ++ Map[String, AnyRef]("__text" -> plugin.innerText, "__cdata" -> plugin.cData) ++ getManifestMap(plugin.manifest) ++ getControllersMap(plugin.controllers) ++ getEventsMap(plugin.events)) - }) - } else Map[String, AnyRef]() - } - -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/MissingAssetValidatorProcessor.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/MissingAssetValidatorProcessor.scala deleted file mode 100644 index f79dd7f3b..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/MissingAssetValidatorProcessor.scala +++ /dev/null @@ -1,45 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.processor - -import org.sunbird.job.exception.InvalidInputException - -import java.io.File - -trait MissingAssetValidatorProcessor extends IProcessor { - - abstract override def process(ecrf: Plugin): Plugin = { - validateMissingAssets(ecrf) - super.process(ecrf) - } - - def getMediaId(media: Media): String = { - if(null != media.data && media.data.nonEmpty){ - val plugin = media.data.get("plugin") - val ver = media.data.get("version") - if((null != plugin && plugin.toString.nonEmpty) && (null != ver && ver.toString.nonEmpty)) - media.id + "_" + plugin+ "_" + ver - else media.id - }else media.id - } - - def validateMissingAssets(ecrf: Plugin): Any = { - if(null != ecrf.manifest){ - val medias:List[Media] = ecrf.manifest.medias - if(null != medias){ - val mediaIds = medias.map(media => getMediaId(media)).toList - if(mediaIds.size != mediaIds.distinct.size) - throw new InvalidInputException("Error! Duplicate Asset Id used in the manifest. Asset Ids are: " - + mediaIds.groupBy(identity).mapValues(_.size).filter(p => p._2 > 1).keySet) - - val nonYoutubeMedias = medias.filter(media => !"youtube".equalsIgnoreCase(media.`type`)) - nonYoutubeMedias.map(media => { - if(widgetTypeAssets.contains(media.`type`) && !new File(getBasePath() + File.separator + "widgets" + File.separator + media.src).exists()) - throw new InvalidInputException("Error! Missing Asset. | [Asset Id '" + media.id) - else if(!widgetTypeAssets.contains(media.`type`) && !media.src.startsWith("http") && !media.src.startsWith("https") && !new File(getBasePath() + File.separator + "assets" + File.separator + media.src).exists()) - throw new InvalidInputException("Error! Missing Asset. | [Asset Id '" + media.id) - else if (!widgetTypeAssets.contains(media.`type`) && (media.src.startsWith("http") || media.src.startsWith("https")) && !new File(getBasePath() + File.separator + "assets" + File.separator + media.src.split("/").last).exists()) - throw new InvalidInputException("Error! Missing Asset. | [Asset Id '" + media.id) - }) - } - } - } -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/XMLLoaderWithCData.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/XMLLoaderWithCData.scala deleted file mode 100644 index f27e8f07b..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/XMLLoaderWithCData.scala +++ /dev/null @@ -1,36 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.processor - -import org.xml.sax.InputSource -import org.xml.sax.ext.{DefaultHandler2, LexicalHandler} - -import scala.xml.factory.XMLLoader -import scala.xml.parsing.FactoryAdapter -import scala.xml.{Elem, PCData, SAXParser, TopScope} - -object XMLLoaderWithCData extends XMLLoader[Elem] { - def lexicalHandler(adapter: FactoryAdapter): LexicalHandler = - new DefaultHandler2 { - def captureCData(): Unit = { - adapter.hStack push PCData(adapter.buffer.toString) - adapter.buffer.clear() - } - - override def startCDATA(): Unit = adapter.captureText() - override def endCDATA(): Unit = captureCData() - } - - override def loadXML(source: InputSource, parser: SAXParser): Elem = { - val newAdapter = adapter - - val xmlReader = parser.getXMLReader - xmlReader.setProperty( - "http://xml.org/sax/properties/lexical-handler", - lexicalHandler(newAdapter)) - - newAdapter.scopeStack push TopScope - parser.parse(source, newAdapter) - newAdapter.scopeStack.pop() - - newAdapter.rootElem.asInstanceOf[Elem] - } -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/XmlParser.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/XmlParser.scala deleted file mode 100644 index 90b7c7246..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/publish/processor/XmlParser.scala +++ /dev/null @@ -1,232 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.processor - -import org.apache.commons.lang3.{StringEscapeUtils, StringUtils} -import org.sunbird.job.exception.InvalidInputException - -import scala.collection.mutable.ListBuffer -import scala.xml._ - -object XmlParser { - - val nonPluginElements: List[String] = List("manifest", "controller", "media", "events", "event", "__cdata", "__text") - val START_TAG_OPENING: String = "<" - val END_TAG_OPENING: String = "" - val ATTRIBUTE_KEY_VALUE_SEPARATOR: String = "=" - val BLANK_SPACE: String = " " - val DOUBLE_QUOTE: String = "\"" - - def parse(xml: String): Plugin = { - val xmlObj: Node = XMLLoaderWithCData.loadString(xml) - processDocument(xmlObj) - } - - def processDocument(root: Node): Plugin = { - if (null != root) { - Plugin(getId(root), getData(root), "", getCdata(root), getChildrenPlugin(root), getManifest(root, validateNode = true), getControllers(root \ "controllers"), getEvents(root)) - } else classOf[Plugin].newInstance() - } - - def getAttributesMap(node: Node): Map[String, AnyRef] = { - node.attributes.asAttrMap - } - - def getId(node: Node): String = { - getAttributesMap(node).getOrElse("id", "").asInstanceOf[String] - } - - def getData(node: Node): Map[String, AnyRef] = { - if (null != node) Map("cwp_element_name" -> node.label) ++ getAttributesMap(node) else Map() - } - - //TODO: Review the below code, this is as per the existing logic - def getCdata(node: Node): String = { - if (null != node && node.child.nonEmpty) { - val childNodes = node.child - var cdata = "" - childNodes.toList.filter(childNode => childNode.isInstanceOf[PCData]).map(childNode => { - cdata = childNode.text - }) - cdata - } else "" - } - - def getChildrenPlugin(node: Node): List[Plugin] = { - if (null != node && node.child.nonEmpty) { - val nodeList = node.child - nodeList.toList.filter(childNode => childNode.isInstanceOf[Elem] && !nonPluginElements.contains(childNode.label) && !"event".equalsIgnoreCase(childNode.label)) - .map(chilNode => Plugin(getId(chilNode), getData(chilNode), getInnerText(chilNode), getCdata(chilNode), getChildrenPlugin(chilNode), getManifest(chilNode, validateNode = false), getControllers(chilNode \ "controllers"), getEvents(chilNode))) - } else { - List() - } - } - - def getInnerText(node: Node): String = { - if (null != node && node.isInstanceOf[Elem] && node.child.nonEmpty) { - val childNodes = node.child - val innerTextlist = childNodes.toList.filter(childNode => childNode.isInstanceOf[Text]).map(item => item.text) - if (innerTextlist.nonEmpty) innerTextlist.head else "" - } else "" - } - - def getMedia(node: Node, validateNode: Boolean): Media = { - if (null != node) { - val attributeMap = getAttributesMap(node) - val id: String = attributeMap.getOrElse("id", "").asInstanceOf[String] - val `type`: String = attributeMap.getOrElse("type", "").asInstanceOf[String] - val src: String = attributeMap.getOrElse("src", "").asInstanceOf[String] - if (validateNode) { - if (StringUtils.isBlank(id) && !(StringUtils.isNotBlank(`type`) && (StringUtils.equalsIgnoreCase(`type`, "js") || StringUtils.equalsIgnoreCase(`type`, "css")))) - throw new InvalidInputException("Error! Invalid Media ('id' is required.) in '" + node.buildString(true) + "' ...") - if (StringUtils.isBlank(`type`)) - throw new InvalidInputException("Error! Invalid Media ('type' is required.) in '" + node.buildString(true) + "' ...") - if (StringUtils.isBlank(src)) - throw new InvalidInputException("Error! Invalid Media ('src' is required.) in '" + node.buildString(true) + "' ...") - } - Media(id, getData(node), getInnerText(node), getCdata(node), src, `type`, getChildrenPlugin(node)) - } else classOf[Media].newInstance() - } - - def getManifest(node: Node, validateNode: Boolean): Manifest = { - val childNodes = node.child - var manifestNode: Node = null - childNodes.toList.filter(childNode => StringUtils.equalsIgnoreCase(childNode.label, "manifest")).map(childNode => manifestNode = childNode) - val mediaList = { - if (null != manifestNode && manifestNode.child.nonEmpty) { - manifestNode.child.toList.filter(childNode => childNode.isInstanceOf[Elem] && "media".equalsIgnoreCase(childNode.label)).map(childNode => getMedia(childNode, validateNode)) - } else List() - } - if (null != manifestNode) { - Manifest(getId(manifestNode), getData(manifestNode), getInnerText(manifestNode), getCdata(manifestNode), mediaList) - } else classOf[Manifest].newInstance() - } - - def getControllers(nodeList: Seq[Node]): List[Controller] = { - if (null != nodeList && nodeList.length > 0) { - nodeList.toList.filter(node => node.isInstanceOf[Elem]).map(node => Controller(node.\@("id"), getData(node), getInnerText(node), getCdata(node))) - } else { - List() - } - } - - def getEvents(node: Node): List[Event] = { - var eventsList: ListBuffer[Event] = ListBuffer() - if (null != node && node.child.nonEmpty) { - val childNodes = node.child - childNodes.toList.map(childNode => { - if (childNode.isInstanceOf[Elem] && "events".equalsIgnoreCase(childNode.label)) { - eventsList ++= getEvents(childNode) - } - if (childNode.isInstanceOf[Elem] && "event".equalsIgnoreCase(childNode.label)) { - eventsList += Event(getId(childNode), getData(childNode), getInnerText(childNode), getCdata(childNode), getChildrenPlugin(childNode)) - } - }) - } - eventsList.toList - } - - /** - * serialize - * - */ - def toString(plugin: Plugin): String = { - val strBuilder = StringBuilder.newBuilder - if (null != plugin) { - strBuilder.append(getElementXml(plugin.data)) - .append(getInnerTextXml(plugin.innerText)) - .append(getCdataXml(plugin.cData)) - .append(getContentManifestXml(plugin.manifest)) - .append(getContentControllersXml(plugin.controllers)) - .append(getPluginsXml(plugin.childrenPlugin)) - .append(getEventsXml(plugin.events)) - .append(getEndTag(plugin.data.getOrElse("cwp_element_name", "").asInstanceOf[String])) - } - strBuilder.toString() - } - - def getElementXml(data: Map[String, AnyRef]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != data) { - strBuilder.append(START_TAG_OPENING + data("cwp_element_name")) - data.filterKeys(key => !StringUtils.equalsIgnoreCase("cwp_element_name", key)).map(entry => strBuilder.append(BLANK_SPACE + entry._1 + ATTRIBUTE_KEY_VALUE_SEPARATOR + DOUBLE_QUOTE + entry._2 + DOUBLE_QUOTE)) - strBuilder.append(TAG_CLOSING) - } - strBuilder - } - - def getInnerTextXml(innerText: String): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (StringUtils.isNotBlank(innerText)) strBuilder.append(StringEscapeUtils.escapeXml(innerText)) - strBuilder - } - - def getCdataXml(cData: String): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (StringUtils.isNotBlank(cData)) strBuilder.append("") - strBuilder - } - - def getContentManifestXml(manifest: Manifest): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != manifest && null != manifest.medias && manifest.medias.nonEmpty) { - strBuilder.append(getElementXml(manifest.data)).append(getInnerTextXml(manifest.innerText)) - .append(getCdataXml(manifest.cData)) - .append(getMediaXml(manifest.medias)) - .append(getEndTag(manifest.data.getOrElse("cwp_element_name", "").asInstanceOf[String])) - } - strBuilder - } - - def getPluginsXml(childrenPlugin: List[Plugin]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != childrenPlugin && childrenPlugin.nonEmpty) { - childrenPlugin.map(plugin => strBuilder.append(toString(plugin))) - } - strBuilder - } - - def getContentControllersXml(controllers: List[Controller]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != controllers && controllers.nonEmpty) { - controllers.map(controller => { - strBuilder.append(getElementXml(controller.data)) - .append(getInnerTextXml(controller.innerText)) - .append(getCdataXml(controller.cData)) - .append(getEndTag(controller.data.getOrElse("cwp_element_name", "").asInstanceOf[String])) - }) - } - strBuilder - } - - def getEventsXml(events: List[Event]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != events && events.nonEmpty) { - if (events.size > 1) strBuilder.append(START_TAG_OPENING + "events" + TAG_CLOSING) - events.map(event => strBuilder.append(getElementXml(event.data)).append(getInnerTextXml(event.innerText)).append(getCdataXml(event.cData)).append(getPluginsXml(event.childrenPlugin)).append(getEndTag("event"))) - if (events.size > 1) strBuilder.append(getEndTag("events")) - } - strBuilder - } - - def getEndTag(str: String): String = { - if (StringUtils.isNotBlank(str)) END_TAG_OPENING + str + TAG_CLOSING - else "" - } - - def getMediaXml(medias: List[Media]): StringBuilder = { - val strBuilder = StringBuilder.newBuilder - if (null != medias && medias.nonEmpty) { - medias.map(media => { - - val updatedMediaData = media.data.map(mediaInfo => { - if(mediaInfo._1.equalsIgnoreCase("src")) - ("src" -> (StringUtils.stripStart(mediaInfo._2.toString, "/"))) - else mediaInfo - }) - - strBuilder.append(getElementXml(updatedMediaData)).append(getInnerTextXml(media.innerText)).append(media.cData).append(getPluginsXml(media.childrenPlugin)).append(getEndTag("media")) - }) - } - strBuilder - } -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/task/LiveNodePublisherConfig.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/task/LiveNodePublisherConfig.scala deleted file mode 100644 index 066d36d84..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/task/LiveNodePublisherConfig.scala +++ /dev/null @@ -1,105 +0,0 @@ -package org.sunbird.job.livenodepublisher.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.livenodepublisher.publish.domain.Event - -import java.util -import scala.collection.JavaConverters._ - -class LiveNodePublisherConfig(override val config: Config) extends PublishConfig(config, "live-node-publisher") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val publishMetaTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Job Configuration - val jobEnv: String = config.getString("job.env") - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val liveVideoStreamTopic: String = config.getString("kafka.live_video_stream.topic") - val kafkaErrorTopic: String = config.getString("kafka.error.topic") - val kafkaSkippedTopic: String = config.getString("kafka.skipped.topic") - val inputConsumerName = "live-node-publisher-consumer" - - // Parallelism - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val eventRouterParallelism: Int = config.getInt("task.router.parallelism") - - // Metric List - val totalEventsCount = "total-events-count" - val skippedEventCount = "skipped-event-count" - val contentPublishEventCount = "content-publish-count" - val contentPublishSuccessEventCount = "content-publish-success-count" - val contentPublishFailedEventCount = "content-publish-failed-count" - val videoStreamingGeneratorEventCount = "video-streaming-event-count" - val collectionPublishEventCount = "collection-publish-count" - val collectionPublishSuccessEventCount = "collection-publish-success-count" - val collectionPublishFailedEventCount = "collection-publish-failed-count" - val collectionPostPublishProcessEventCount = "collection-post-publish-process-count" - val mvProcessorEventCount = "mvc-processor-event-count" - val dialcodeContextUpdaterEventCount = "dialcode-context-updater-event-count" - - // Cassandra Configurations - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - val contentKeyspaceName: String = config.getString("content.keyspace") - val contentTableName: String = config.getString("content.table") - val hierarchyKeyspaceName: String = config.getString("hierarchy.keyspace") - val hierarchyTableName: String = config.getString("hierarchy.table") - - // Neo4J Configurations - val graphRoutePath: String = config.getString("neo4j.routePath") - val graphName: String = config.getString("neo4j.graph") - - // Redis Configurations - val nodeStore: Int = config.getInt("redis.database.contentCache.id") - - // Out Tags - val contentPublishOutTag: OutputTag[Event] = OutputTag[Event]("live-content-publish") - val collectionPublishOutTag: OutputTag[Event] = OutputTag[Event]("live-collection-publish") - val generateVideoStreamingOutTag: OutputTag[String] = OutputTag[String]("live-video-streaming-generator-request") - val failedEventOutTag: OutputTag[String] = OutputTag[String]("failed-event") - val skippedEventOutTag: OutputTag[String] = OutputTag[String]("skipped-event") - val generatePostPublishProcessTag: OutputTag[String] = OutputTag[String]("post-publish-process-request") - - // Service Urls - val printServiceBaseUrl: String = config.getString("service.print.basePath") - - val definitionBasePath: String = if (config.hasPath("schema.basePath")) config.getString("schema.basePath") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - val schemaSupportVersionMap: Map[String, AnyRef] = if (config.hasPath("schema.supportedVersion")) config.getObject("schema.supportedVersion").unwrapped().asScala.toMap else Map[String, AnyRef]() - - val supportedObjectType: util.List[String] = if (config.hasPath("content.objectType")) config.getStringList("content.objectType") else util.Arrays.asList[String]("Content", "ContentImage") - val supportedMimeType: util.List[String] = if (config.hasPath("content.mimeType")) config.getStringList("content.mimeType") else util.Arrays.asList[String]("application/pdf") - val streamableMimeType: util.List[String] = if (config.hasPath("content.stream.mimeType")) config.getStringList("content.stream.mimeType") else util.Arrays.asList[String]("video/mp4") - val isStreamingEnabled: Boolean = if (config.hasPath("content.stream.enabled")) config.getBoolean("content.stream.enabled") else false - val assetDownloadDuration: String = if (config.hasPath("content.asset_download_duration")) config.getString("content.asset_download_duration") else "60 seconds" - - val isECARExtractionEnabled: Boolean = if (config.hasPath("content.isECARExtractionEnabled")) config.getBoolean("content.isECARExtractionEnabled") else true - val contentFolder: String = if (config.hasPath("cloud_storage.folder.content")) config.getString("cloud_storage.folder.content") else "content" - val artifactFolder: String = if (config.hasPath("cloud_storage.folder.artifact")) config.getString("cloud_storage.folder.artifact") else "artifact" - val retryAssetDownloadsCount: Integer = if (config.hasPath("content.retry_asset_download_count")) config.getInt("content.retry_asset_download_count") else 1 - val artifactSizeForOnline: Double = if (config.hasPath("content.artifact.size.for_online")) config.getDouble("content.artifact.size.for_online") else 209715200 - val bundleLocation: String = if (config.hasPath("content.bundleLocation")) config.getString("content.bundleLocation") else "/data/contentBundle/" - - val extractableMimeTypes = List("application/vnd.ekstep.ecml-archive", "application/vnd.ekstep.html-archive", "application/vnd.ekstep.plugin-archive", "application/vnd.ekstep.h5p-archive") - - val categoryMap: java.util.Map[String, AnyRef] = if (config.hasPath("contentTypeToPrimaryCategory")) config.getAnyRef("contentTypeToPrimaryCategory").asInstanceOf[java.util.Map[String, AnyRef]] else new util.HashMap[String, AnyRef]() - - val esConnectionInfo: String = config.getString("es.basePath") - val compositeSearchIndexName: String = if (config.hasPath("compositesearch.index.name")) config.getString("compositesearch.index.name") else "compositesearch" - val compositeSearchIndexType: String = if (config.hasPath("search.document.type")) config.getString("search.document.type") else "cs" - val nestedFields: util.List[String] = if (config.hasPath("content.nested.fields")) config.getStringList("content.nested.fields") else util.Arrays.asList[String]("badgeAssertions","targets","badgeAssociations") - - val allowedExtensionsWord: util.List[String] = if (config.hasPath("mimetype.allowed_extensions.word")) config.getStringList("mimetype.allowed_extensions.word") else util.Arrays.asList[String]("doc", "docx", "ppt", "pptx", "key", "odp", "pps", "odt", "wpd", "wps", "wks") - - val searchServiceBaseUrl : String = config.getString("service.search.basePath") - val searchFields: util.List[String] = util.Arrays.asList[String]("identifier","migratedVersion") - - val isrRelativePathEnabled: Boolean = if (config.hasPath("cloudstorage.metadata.replace_absolute_path")) config.getBoolean("cloudstorage.metadata.replace_absolute_path") else false - val migrationVersion: Double = if(config.hasPath("migrationVersion")) config.getDouble("migrationVersion") else 1.0 -} diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/task/LiveNodePublisherStreamTask.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/task/LiveNodePublisherStreamTask.scala deleted file mode 100644 index 503aaf128..000000000 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/task/LiveNodePublisherStreamTask.scala +++ /dev/null @@ -1,62 +0,0 @@ -package org.sunbird.job.livenodepublisher.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.livenodepublisher.function.{LiveCollectionPublishFunction, LiveContentPublishFunction, LivePublishEventRouter} -import org.sunbird.job.livenodepublisher.publish.domain.Event -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - -import java.io.File -import java.util - -class LiveNodePublisherStreamTask(config: LiveNodePublisherConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) -// implicit val env: StreamExecutionEnvironment = StreamExecutionEnvironment.createLocalEnvironment() - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - val processStreamTask = env.addSource(source).name(config.inputConsumerName) - .uid(config.inputConsumerName).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new LivePublishEventRouter(config)) - .name("publish-event-router").uid("publish-event-router") - .setParallelism(config.eventRouterParallelism) - - val contentPublish = processStreamTask.getSideOutput(config.contentPublishOutTag).process(new LiveContentPublishFunction(config, httpUtil)) - .name("live-content-publish-process").uid("live-content-publish-process").setParallelism(config.kafkaConsumerParallelism) - - contentPublish.getSideOutput(config.generateVideoStreamingOutTag).addSink(kafkaConnector.kafkaStringSink(config.liveVideoStreamTopic)) - contentPublish.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaErrorTopic)) - - val collectionPublish = processStreamTask.getSideOutput(config.collectionPublishOutTag).process(new LiveCollectionPublishFunction(config, httpUtil)) - .name("live-collection-publish-process").uid("live-collection-publish-process").setParallelism(config.kafkaConsumerParallelism) - collectionPublish.getSideOutput(config.skippedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaSkippedTopic)) - collectionPublish.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaErrorTopic)) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object LiveNodePublisherStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("live-node-publisher.conf").withFallback(ConfigFactory.systemEnvironment())) - val publishConfig = new LiveNodePublisherConfig(config) - val kafkaUtil = new FlinkKafkaConnector(publishConfig) - val httpUtil = new HttpUtil - val task = new LiveNodePublisherStreamTask(publishConfig, kafkaUtil, httpUtil) - task.process() - } -} \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/test/resources/logback-test.xml b/publish-pipeline/live-node-publisher/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/publish-pipeline/live-node-publisher/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/test/resources/test.conf b/publish-pipeline/live-node-publisher/src/test/resources/test.conf deleted file mode 100644 index 70c1a6b4c..000000000 --- a/publish-pipeline/live-node-publisher/src/test/resources/test.conf +++ /dev/null @@ -1,140 +0,0 @@ -include "base-test.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.publish.job.request" - live_video_stream.topic = "sunbirddev.live.video.stream.request" - error.topic = "sunbirddev.learning.events.failed" - skipped.topic = "sunbirddev.learning.events.skipped" - groupId = "local-content-publish-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - router.parallelism = 1 -} - -redis { - database { - contentCache.id = 0 - } -} - -content { - bundleLocation = "/tmp/contentBundle" - isECARExtractionEnabled = true - retry_asset_download_count = 1 - keyspace = "dev_content_store" - table = "content_data" - tmp_file_location = "/tmp" - objectType = ["Content", "ContentImage","Collection","CollectionImage"] - mimeType = ["application/pdf", "video/avi", "video/mpeg", "video/quicktime", "video/3gpp", "video/mpeg", "video/mp4", "video/ogg", "video/webm", "application/vnd.ekstep.html-archive","application/vnd.ekstep.ecml-archive","application/vnd.ekstep.content-collection" - "application/vnd.ekstep.ecml-archive", - "application/vnd.ekstep.html-archive", - "application/vnd.android.package-archive", - "application/vnd.ekstep.content-archive", - "application/octet-stream", - "application/json", - "application/javascript", - "application/xml", - "text/plain", - "text/html", - "text/javascript", - "text/xml", - "text/css", - "image/jpeg", "image/jpg", "image/png", "image/tiff", "image/bmp", "image/gif", "image/svg+xml", - "image/x-quicktime", - "video/avi", "video/mpeg", "video/quicktime", "video/3gpp", "video/mpeg", "video/mp4", "video/ogg", "video/webm", - "video/msvideo", - "video/x-msvideo", - "video/x-qtc", - "video/x-mpeg", - "audio/mp3", "audio/mp4", "audio/mpeg", "audio/ogg", "audio/webm", "audio/x-wav", "audio/wav", - "audio/mpeg3", - "audio/x-mpeg-3", - "audio/vorbis", - "application/x-font-ttf", - "application/pdf", "application/epub", "application/msword", - "application/vnd.ekstep.h5p-archive", - "application/vnd.ekstep.plugin-archive", - "video/x-youtube", "video/youtube", - "text/x-url"] - asset_download_duration = "60 seconds" - - stream { - enabled = true - mimeType = ["video/mp4", "video/webm"] - } - artifact.size.for_online=209715200 - - downloadFiles { - spine = ["appIcon"] - full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"] - } - - nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions"] -} - -hierarchy { - keyspace = "dev_hierarchy_store" - table = "content_hierarchy" -} - -cloud_storage { - folder { - content = "content" - artifact = "artifact" - } -} - -service { - print.basePath = "http://11.2.6.6/print" -} - - -contentTypeToPrimaryCategory { - ClassroomTeachingVideo: "Explanation Content" - ConceptMap: "Learning Resource" - Course: "Course" - CuriosityQuestionSet: "Practice Question Set" - eTextBook: "eTextbook" - Event: "Event" - EventSet: "Event Set" - ExperientialResource: "Learning Resource" - ExplanationResource: "Explanation Content" - ExplanationVideo: "Explanation Content" - FocusSpot: "Teacher Resource" - LearningOutcomeDefinition: "Teacher Resource" - MarkingSchemeRubric: "Teacher Resource" - PedagogyFlow: "Teacher Resource" - PracticeQuestionSet: "Practice Question Set" - PracticeResource: "Practice Question Set" - SelfAssess: "Course Assessment" - TeachingMethod: "Teacher Resource" - TextBook: "Digital Textbook" - Collection: "Content Playlist" - ExplanationReadingMaterial: "Learning Resource" - LearningActivity: "Learning Resource" - LessonPlan: "Content Playlist" - LessonPlanResource: "Teacher Resource" - PreviousBoardExamPapers: "Learning Resource" - TVLesson: "Explanation Content" - OnboardingResource: "Learning Resource" - ReadingMaterial: "Learning Resource" - Template: "Template" - Asset: "Asset" - Plugin: "Plugin" - LessonPlanUnit: "Lesson Plan Unit" - CourseUnit: "Course Unit" - TextBookUnit: "Textbook Unit" - Asset: "Certificate Template" -} - -enableDIALContextUpdate = "Yes" - - -service.search.basePath = "http://11.2.6.6/search" \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/test/resources/test.cql b/publish-pipeline/live-node-publisher/src/test/resources/test.cql deleted file mode 100644 index 3416fa82a..000000000 --- a/publish-pipeline/live-node-publisher/src/test/resources/test.cql +++ /dev/null @@ -1,58 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS dev_content_store WITH replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS dev_content_store.content_data ( - content_id text, - body blob, - PRIMARY KEY (content_id) -); - -CREATE KEYSPACE IF NOT EXISTS dev_hierarchy_store WITH replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS dev_hierarchy_store.content_hierarchy ( - identifier text, - hierarchy text, - relational_metadata text, - PRIMARY KEY (identifier) -); - -INSERT INTO dev_content_store.content_data(content_id, body) VALUES ( -'do_11321328578759884811663', -textAsBlob('

A person having less ‘haemoglobin’ is suffering from:

Jaundice

Anaemia

Malaria

Chikungunya

\",\"media\":[],\"responseDeclaration\":{\"responseValue\":{\"cardinality\":\"single\",\"type\":\"integer\",\"correct_response\":{\"value\":\"2\"}}},\"options\":[{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Jaundice

\",\"resvalue\":0,\"resindex\":0}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Anaemia

\",\"resvalue\":1,\"resindex\":1}},{\"answer\":true,\"value\":{\"type\":\"text\",\"body\":\"

Malaria

\",\"resvalue\":2,\"resindex\":2}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Chikungunya

\",\"resvalue\":3,\"resindex\":3}}],\"questionCount\":0}]]>') -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_2133950809948078081503', -'{"ownershipType":["createdBy"],"copyright":"firstorg","se_gradeLevelIds":["tn_k-12_5_gradelevel_class1"],"keywords":["pdf"],"subject":["English"],"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","organisation":["First Org","ST MARYS SCHOOL BHATHUBASTI"],"language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142","do_2132926691727114241157"],"objectType":"Content","se_mediums":["English"],"gradeLevel":["Class 1"],"appIcon":"","primaryCategory":"Digital Textbook","children":[{"ownershipType":["createdBy"],"parent":"do_2133950809948078081503","code":"76abafa2a0c2cfef90b52db1ef41fb82","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"This chapter describes about human body","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142","do_2132926691727114241157"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.254+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816031047681516","code":"20cc1f31e62f924c6e47bf04c994376b","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":[],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.252+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816030883841508","code":"7991af5c2e51e4d3d7b83167aaac8829","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"xyz","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":[],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.253+0000","objectType":"Content","primaryCategory":"Textbook Unit","contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030965761510","lastStatusChangedOn":"2021-10-25T06:03:17.253+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797253","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":3,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":0,"name":"5.1.1 Key parts in the head","status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950816030883841508","code":"3bf70f06d3e8dba010d8806fd94259b1","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":[],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.228+0000","objectType":"Content","primaryCategory":"Textbook Unit","contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816028917761504","lastStatusChangedOn":"2021-10-25T06:03:17.228+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797228","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":3,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":0,"name":"5.1.2 Other parts","status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030883841508","lastStatusChangedOn":"2021-10-25T06:03:17.252+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797252","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":2,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":0,"name":"5.1 Parts of Body","status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950816031047681516","code":"40a1ed37e0fad94eca76b2a96fe086ab","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.254+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816031047681514","code":"b186b1bbcc9c58db865f75e34345179e","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.251+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816030801921506","code":"9cf84ff2fb08f9af4c23eb09df9b2520","credentials":{"enabled":"No"},"channel":"0126825293972439041","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"online":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950127_do_2133950809948078081503_1.0_online.ecar","size":8755.0},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","size":12426.0}},"leafNodes":["do_2133916718042972161142"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:03:17.253+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133950816030965761512","copyright":"Tamil Nadu, P P MAHAPUR","previewUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133916718042972161142/sample-mp4-file.mp4","subject":["Home Science"],"channel":"01269878797503692810","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133916718042972161142/oct20-mp4_1634728008255_do_2133916718042972161142_2.ecar","organisation":["Tamil Nadu","P P MAHAPUR"],"showNotification":true,"language":["English"],"mimeType":"video/mp4","variants":{"full":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133916718042972161142/oct20-mp4_1634728008255_do_2133916718042972161142_2.ecar","size":"10504301"},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133916718042972161142/oct20-mp4_1634728009035_do_2133916718042972161142_2_SPINE.ecar","size":"1465"}},"objectType":"Content","se_mediums":["English","Tamil"],"gradeLevel":["Class 4","Class 2"],"primaryCategory":"Explanation Content","appId":"preprod.diksha.app","contentEncoding":"identity","artifactUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133916718042972161142/sample-mp4-file.mp4","me_totalPlaySessionCount":{"portal":4},"lockKey":"68bcaed4-8eaf-4f01-b5b4-a34a08887de7","sYS_INTERNAL_LAST_UPDATED_ON":"2021-10-25T05:46:53.202+0000","contentType":"Resource","se_gradeLevels":["Class 4","Class 2"],"identifier":"do_2133916718042972161142","lastUpdatedBy":"4cd4c690-eab6-4938-855a-447c7b1b8ea9","audience":["Teacher"],"me_totalTimeSpentInSec":{"portal":325},"visibility":"Default","author":"Adarsh","consumerId":"962257b5-dab6-44a6-b8db-6933b3963dc7","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"08631a74-4b94-4cf7-a818-831135248a4a","version":2,"se_subjects":["Home Science"],"license":"CC BY 4.0","prevState":"Review","size":1.054662E7,"lastPublishedOn":"2021-10-20T11:06:48.254+0000","name":"Oct20 Mp4","attributions":["Tester"],"status":"Live","code":"9194086b-4c4c-45d6-92c9-98828a4aae74","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Review","streamingUrl":"https://ntppreprodmedia-inct.streaming.media.azure.net/29886fa7-95d5-4cc3-bf5f-be9c66fb78cb/sample-mp4-file.ism/manifest(format=m3u8-aapl-v3)","medium":["English","Tamil"],"idealScreenSize":"normal","createdOn":"2021-10-20T10:26:02.051+0000","se_boards":["State (Tamil Nadu)"],"copyrightYear":2021,"contentDisposition":"inline","additionalCategories":["Textbook"],"licenseterms":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.","lastUpdatedOn":"2021-10-20T11:36:17.429+0000","dialcodeRequired":"No","lastStatusChangedOn":"2021-10-20T11:06:49.079+0000","createdFor":["01269878797503692810","0133480174794670081832"],"creator":"ContentcreatorTN","os":["All"],"se_FWIds":["tn_k-12_5"],"pkgVersion":2,"versionKey":"1634729777429","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":5,"lastSubmittedOn":"2021-10-20T10:30:29.896+0000","createdBy":"4cd4c690-eab6-4938-855a-447c7b1b8ea9","compatibilityLevel":1,"board":"State (Tamil Nadu)","resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030965761512","lastStatusChangedOn":"2021-10-25T06:03:17.253+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797253","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":4,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":1,"name":"dsffgdg","status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816030801921506","lastStatusChangedOn":"2021-10-25T06:03:17.251+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797251","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":3,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":1,"name":"5.2.1 Respiratory System","status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T06:05:49.619+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816031047681514","lastStatusChangedOn":"2021-10-25T06:03:17.254+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797254","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":2,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":1,"name":"5.2 Organ Systems","status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950816031047681516","unitIdentifiers":["do_21329099910853427215263"],"copyright":"Kendriya_Vidyalaya,2020","organisationId":"32625405-bc59-4622-89bb-d06934d690ef","previewUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","keywords":["pdf"],"subject":["Hindi"],"channel":"013085024460783616158023","downloadUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/pdf_1.pdf_1634813465403_do_2132926691727114241157_2.ecar","language":["English"],"source":"https://dock.preprod.ntp.net.in/api/content/v1/read/do_21329119747088384015821","mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/pdf_1.pdf_1634813465403_do_2132926691727114241157_2.ecar","size":"259984"},"spine":{"ecarUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/pdf_1.pdf_1634813465462_do_2132926691727114241157_2_SPINE.ecar","size":"5422"}},"objectType":"Content","se_mediums":["English"],"gradeLevel":["Class 10"],"appIcon":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/content.thumb.jpg","primaryCategory":"Teacher Resource","appId":"preprod.diksha.portal","contentEncoding":"identity","artifactUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","me_totalPlaySessionCount":{"portal":3},"sYS_INTERNAL_LAST_UPDATED_ON":"2021-10-25T05:46:52.419+0000","contentType":"MarkingSchemeRubric","se_gradeLevels":["Class 10"],"identifier":"do_2132926691727114241157","audience":["Administrator"],"me_totalTimeSpentInSec":{"portal":221},"visibility":"Default","consumerId":"962257b5-dab6-44a6-b8db-6933b3963dc7","discussionForum":{"enabled":"Yes"},"index":3,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"fcc5bf38-810b-4543-aa00-e96a1d59573a","version":2,"pragma":["external"],"se_subjects":["Hindi"],"license":"CC BY 4.0","prevState":"Review","size":256184.0,"lastPublishedOn":"2021-10-21T10:51:03.459+0000","name":"PDF_1.pdf","attributions":["kanmani"],"status":"Live","code":"c644316c-0a39-7c95-9ff4-c4f027d4c024","credentials":{"enabled":"No"},"prevStatus":"Processing","origin":"do_21329119747088384015821","description":"pdf","streamingUrl":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","medium":["English"],"posterImage":"https://drive.google.com/uc?export=download&id=1z2kHz_wfjcOcDKfenkxWwqIlwtro6uv0","idealScreenSize":"normal","createdOn":"2021-06-02T13:24:39.874+0000","se_boards":["CBSE"],"processId":"2bb3d2ed-a0b5-4502-831b-e7e0e3423e4c","contentDisposition":"inline","lastUpdatedOn":"2021-10-21T10:51:05.506+0000","originData":{"identifier":"do_21329119747088384015821","repository":"https://dock.preprod.ntp.net.in/api/content/v1/read/do_21329119747088384015821"},"collectionId":"do_21329099910788710415244","dialcodeRequired":"No","lastStatusChangedOn":"2021-10-21T10:51:05.506+0000","creator":"కాయల్ నధి","os":["All"],"cloudStorageKey":"content/do_2132926691727114241157/artifact/do_2132926691727114241157_1622640280065_do_21329119747088384015821_1622462440773_pdf_1.pdf","se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":2,"versionKey":"1622641657267","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"s3Key":"ecar_files/do_2132926691727114241157/pdf_1.pdf_1622641665371_do_2132926691727114241157_1.0.ecar","lastSubmittedOn":"2021-06-02T13:24:41.290+0000","createdBy":"4096d5cf-af62-48d6-88a3-e007219b8e59","compatibilityLevel":4,"board":"CBSE","programId":"013c04f0-c1c9-11eb-be1d-e5048cd700e9"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T10:29:28.137+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133950816031047681516","lastStatusChangedOn":"2021-10-25T06:03:17.254+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"pkgVersion":1.0,"versionKey":"1635141797254","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":1,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","compatibilityLevel":1,"leafNodesCount":2,"name":"5. Human Body","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_2133950809948078081503","code":"c1bed6fb-7e28-c033-d87a-365511c10614","credentials":{"enabled":"No"},"channel":"0126825293972439041","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-10-25T10:29:28.133+0000","objectType":"Content","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_2133952124365455361558","code":"2ebb6e7c-e01b-5be1-7c33-d00334fcdd46","credentials":{"enabled":"No"},"channel":"0126825293972439041","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-10-25T10:29:28.137+0000","objectType":"Content","primaryCategory":"Textbook Unit","contentDisposition":"inline","lastUpdatedOn":"2021-10-25T10:29:28.137+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133952124365783041560","lastStatusChangedOn":"2021-10-25T10:29:28.137+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1635157768137","license":"CC BY 4.0","idealScreenDensity":"hdpi","depth":2,"compatibilityLevel":1,"name":"6.1 Medula Oblongata","attributions":[],"status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2021-10-25T10:29:28.133+0000","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_2133952124365455361558","lastStatusChangedOn":"2021-10-25T10:29:28.133+0000","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1635157768133","license":"CC BY 4.0","idealScreenDensity":"hdpi","depth":1,"compatibilityLevel":1,"name":"6. Inhuman Mind","attributions":[],"status":"Draft"}],"contentEncoding":"gzip","lockKey":"02bcb42d-d22d-4b08-a872-b8a22372d2a0","generateDIALCodes":"Yes","totalCompressedSize":1.0802804E7,"mimeTypesCount":"{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":7,\"video/mp4\":1}","sYS_INTERNAL_LAST_UPDATED_ON":"2021-10-25T06:05:50.280+0000","contentType":"TextBook","se_gradeLevels":["Class 1"],"trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_2133950809948078081503","audience":["Student"],"se_boardIds":["tn_k-12_5_board_statetamilnadu"],"subjectIds":["tn_k-12_5_subject_english"],"toc_url":"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/do_2133950809948078081503/artifact/do_2133950809948078081503_toc.json","visibility":"Default","contentTypesCount":"{\"TextBookUnit\":7,\"MarkingSchemeRubric\":1,\"Resource\":1}","author":"Content 1 Creator1","consumerId":"962257b5-dab6-44a6-b8db-6933b3963dc7","childNodes":["do_2133950816030965761510","do_2133950816030883841508","do_2133950816031047681516","do_2133950816028917761504","do_2133916718042972161142","do_2133950816030965761512","do_2133950816030801921506","do_2133950816031047681514","do_2132926691727114241157","do_2133952124365783041560","do_2133952124365455361558"],"discussionForum":{"enabled":"No"},"mediaType":"content","osId":"org.ekstep.quiz.app","lastPublishedBy":"30b2571f-08f9-49ce-b97a-c643df0c82f7","languageCode":["en"],"version":2,"se_subjects":["English"],"license":"CC BY 4.0","prevState":"Review","size":12426.0,"lastPublishedOn":"2021-10-25T06:05:49.988+0000","name":"Test collection publishing","mediumIds":["tn_k-12_5_medium_english"],"status":"Draft","code":"org.sunbird.nV84DB","credentials":{"enabled":"No"},"prevStatus":"Processing","description":"Enter description for TextBook","medium":["English"],"idealScreenSize":"normal","createdOn":"2021-10-25T06:02:03.002+0000","se_boards":["State (Tamil Nadu)"],"se_mediumIds":["tn_k-12_5_medium_english"],"copyrightYear":2021,"contentDisposition":"inline","additionalCategories":["Textbook"],"lastUpdatedOn":"2021-10-25T06:05:49.619+0000","dialcodeRequired":"No","lastStatusChangedOn":"2021-10-25T10:29:28.562+0000","createdFor":["0126825293972439041","01323953134532198492165"],"creator":"Content 1 Creator1","os":["All"],"se_subjectIds":["tn_k-12_5_subject_english"],"se_FWIds":["tn_k-12_5"],"pkgVersion":1.0,"versionKey":"1635157768888","idealScreenDensity":"hdpi","framework":"tn_k-12_5","depth":0,"s3Key":"ecar_files/do_2133950809948078081503/test-collection-publishing_1635141950055_do_2133950809948078081503_1.0_spine.ecar","boardIds":["tn_k-12_5_board_statetamilnadu"],"lastSubmittedOn":"2021-10-25T06:04:24.346+0000","createdBy":"0b96635f-fe2b-4ab0-a511-05cfce8faa3f","compatibilityLevel":1,"leafNodesCount":2,"userConsent":"Yes","gradeLevelIds":["tn_k-12_5_gradelevel_class1"],"board":"State (Tamil Nadu)","resourceType":"Book"}', -'{"do_2133950809948078081503":{"name":"Collection Publish T21","children":["do_2133916718042972161142"],"root":true},"do_2133916718042972161142":{"name":"Collection Parent","children":["do_11340096165525094411"],"root":false,"relationalMetadata":{"do_11340096165525094411":{"name":"Test Name RM L1 - R1","keywords":["Overwriting content KW1"]}}},"do_11340096165525094411":{"name":"PDF Content","children":[],"root":false}' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_123', -'{"identifier":"do_123","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}', -'{"do_123":{"name":"Collection Publish T21","children":["do_11340511137112064018","do_11340511137080934412"],"root":true},"do_11340511137112064018":{"name":"Collection Parent","children":["do_11340096165525094411"],"root":false,"relationalMetadata":{"do_11340096165525094411":{"name":"Test Name RM L1 - R1","keywords":["Overwriting content KW1"]}}},"do_11340511137080934412":{"name":"Collection Parent","children":["do_11340096165525094411"],"root":false,"relationalMetadata":{"do_11340096165525094411":{"name":"Test Name RM L1 - R1","keywords":["Overwriting content KW1"]}}},"do_11340096165525094411":{"name":"PDF Content","children":[],"root":false},"do_113405111371145216110":{"name":"test","children":[], "root":false}}' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy) VALUES ( -'do_21354027142511820812318.img', -'{ "ownershipType": [ "createdBy" ], "copyright": "tn", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "keywords": [ "வாக்கியங்கள்", "വളരെ", "बतख़!", "Drop", "മലയാളം" ], "subject": [ "English" ], "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "organisation": [ "Tamil Nadu" ], "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "size": "16926" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar", "size": "8313" } }, "leafNodes": [ "do_21353901951355289618149" ], "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 1" ], "appIcon": "", "primaryCategory": "Digital Textbook", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "80986053-eb44-0c35-1c3b-c84806d4775a", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "size": "16926" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar", "size": "8313" } }, "leafNodes": [ "do_21353901951355289618149" ], "idealScreenSize": "normal", "createdOn": "2022-05-18T10:50:55.849+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354031968951500812320", "copyright": "tn", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "keywords": [ "Drop" ], "subject": [ "Accountancy" ], "targetMediumIds": [ "tn_k-12_5_medium_english" ], "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar", "organisation": [ "Tamil Nadu", "MPPS GYARAGONDANAHALLI" ], "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar", "size": "13797" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862667_do_21354022673739776012312_1_ONLINE.ecar", "size": "5181" } }, "leafNodes": [ "do_21353901951355289618149" ], "targetGradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "objectType": "Content", "se_mediums": [ "English" ], "primaryCategory": "Course", "appId": "staging.sunbird.portal", "contentEncoding": "gzip", "lockKey": "3efeaad1-490b-4044-8185-52a421057844", "generateDIALCodes": "No", "totalCompressedSize": 209850, "mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":1}", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T07:45:16.264+0000", "contentType": "Course", "se_gradeLevels": [ "Class 1" ], "trackable": { "enabled": "Yes", "autoBatch": "No" }, "identifier": "do_21354022673739776012312", "audience": [ "Student" ], "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "subjectIds": [ "tn_k-12_5_subject_accountancy" ], "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "visibility": "Default", "contentTypesCount": "{\"eTextBook\":1,\"CourseUnit\":1}", "author": "Guest name changed", "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "childNodes": [ "do_21353901951355289618149", "do_21354022740915814412313" ], "discussionForum": { "enabled": "Yes" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "version": 2, "se_subjects": [ "Accountancy" ], "license": "CC BY 4.0", "prevState": "Review", "size": 13797, "lastPublishedOn": "2022-05-18T07:44:22.530+0000", "name": "newcert", "targetBoardIds": [ "tn_k-12_5_board_statetamilnadu" ], "status": "Live", "code": "org.sunbird.11SAbB", "credentials": { "enabled": "Yes" }, "prevStatus": "Processing", "description": "Enter description for Course", "idealScreenSize": "normal", "createdOn": "2022-05-18T07:41:49.160+0000", "reservedDialcodes": { "U5X4C5": 0 }, "batches": [ { "createdFor": [ "01269878797503692810", "01275630040485068839017" ], "endDate": "2022-05-20", "name": "nc", "batchId": "013540229187338240118", "enrollmentType": "open", "enrollmentEndDate": "2022-05-19", "startDate": "2022-05-18", "status": 1 } ], "se_boards": [ "State (Tamil Nadu)" ], "targetSubjectIds": [ "tn_k-12_5_subject_english" ], "se_mediumIds": [ "tn_k-12_5_medium_english" ], "copyrightYear": 2022, "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "lastUpdatedOn": "2022-05-18T07:44:22.749+0000", "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-18T07:44:22.749+0000", "createdFor": [ "01269878797503692810" ], "creator": "Guest name changed", "os": [ "All" ], "se_subjectIds": [ "tn_k-12_5_subject_accountancy", "tn_k-12_5_subject_english" ], "se_FWIds": [ "tn_k-12_5" ], "targetFWIds": [ "tn_k-12_5" ], "pkgVersion": 1, "versionKey": "1652859827944", "idealScreenDensity": "hdpi", "framework": "tn_k-12_5", "dialcodes": [ "U5X4C5" ], "depth": 0, "s3Key": "content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "lastSubmittedOn": "2022-05-18T07:43:47.614+0000", "createdBy": "fca2925f-1eee-4654-9177-fece3fd6afc9", "compatibilityLevel": 4, "leafNodesCount": 1, "userConsent": "Yes", "resourceType": "Course" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354031968951500812320", "lastStatusChangedOn": "2022-05-18T10:50:55.849+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "pkgVersion": 1, "versionKey": "1652871055849", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "R8C7G2" ], "depth": 1, "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "compatibilityLevel": 1, "leafNodesCount": 1, "name": "Ch1 ", "attributions": [], "status": "Draft" }, { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "22c05b75-ae2a-6280-c192-2e2a4157d141", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "size": "16926" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar", "size": "8313" } }, "leafNodes": [], "idealScreenSize": "normal", "createdOn": "2022-05-18T10:50:55.854+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354031968955596812322", "copyright": "tn", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "keywords": [ "बतख़!", "വളരെ", "வாக்கியங்கள்", "ಕನ್ನಡ" ], "subject": [ "English" ], "channel": "01269878797503692810", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "organisation": [ "Tamil Nadu", "MPPS HANUMANNAHALLI" ], "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018" } }, "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 1" ], "appIcon": "", "primaryCategory": "Digital Textbook", "contentEncoding": "gzip", "lockKey": "35daaa23-b864-40b6-8709-fedc31de95c0", "generateDIALCodes": "Yes", "totalCompressedSize": 0, "mimeTypesCount": "{\"application/vnd.ekstep.content-collection\":3}", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-17T06:12:28.810+0000", "contentType": "TextBook", "se_gradeLevels": [ "Class 1" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21353947184001843212121", "audience": [ "Student" ], "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "subjectIds": [ "tn_k-12_5_subject_english" ], "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/artifact/do_21353947184001843212121_toc.json", "visibility": "Default", "contentTypesCount": "{\"TextBookUnit\":3}", "author": "Guest name changed", "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "childNodes": [ "do_21353947322769408012133", "do_21353947287477452812131", "do_21353947357250355212135", "do_21353947535707340812137" ], "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "version": 2, "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "size": 4019, "lastPublishedOn": "2022-05-17T06:12:27.957+0000", "name": "ಕನ್ನಡ", "mediumIds": [ "tn_k-12_5_medium_english" ], "status": "Draft", "code": "org.sunbird.zRvSTj", "credentials": { "enabled": "No" }, "prevStatus": "Processing", "description": "ಕನ್ನಡ", "medium": [ "English" ], "idealScreenSize": "normal", "createdOn": "2022-05-17T06:05:58.602+0000", "se_boards": [ "State (Tamil Nadu)" ], "se_mediumIds": [ "tn_k-12_5_medium_english" ], "copyrightYear": 2020, "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "lastUpdatedOn": "2022-05-17T06:13:08.755+0000", "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-17T06:13:08.396+0000", "createdFor": [ "01269878797503692810" ], "creator": "Guest name changed", "os": [ "All" ], "se_subjectIds": [ "tn_k-12_5_subject_english" ], "se_FWIds": [ "tn_k-12_5" ], "pkgVersion": 1, "versionKey": "1652767988755", "idealScreenDensity": "hdpi", "framework": "tn_k-12_5", "depth": 0, "s3Key": "content/do_21353947184001843212121/artifact/do_21353947184001843212121_toc.json", "boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "lastSubmittedOn": "2022-05-17T06:12:01.588+0000", "createdBy": "fca2925f-1eee-4654-9177-fece3fd6afc9", "compatibilityLevel": 1, "leafNodesCount": 0, "userConsent": "Yes", "gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "board": "State (Tamil Nadu)", "resourceType": "Book" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354031968955596812322", "lastStatusChangedOn": "2022-05-18T10:50:55.854+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 2, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "pkgVersion": 1, "versionKey": "1652871055854", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "L5L9A2" ], "depth": 1, "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "compatibilityLevel": 1, "leafNodesCount": 0, "name": "Ch2", "attributions": [], "status": "Draft" }, { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "6b8ccba1-9aa8-19d7-fe3b-35ca4be5d6cf", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "idealScreenSize": "normal", "createdOn": "2022-05-18T11:01:38.790+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354032495648768012324", "copyright": "2013", "se_gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "previewUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/cbse-copy-88.pdf", "keywords": [ "Drop" ], "subject": [ "English" ], "channel": "01345815127107174426", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/aparna-86_1652722733905_do_21353901750772531217809_1.ecar", "language": [ "English" ], "source": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901750772531217809", "mimeType": "application/pdf", "variants": { "full": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/aparna-86_1652722733905_do_21353901750772531217809_1.ecar", "size": "193281" }, "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/aparna-86_1652722733964_do_21353901750772531217809_1_SPINE.ecar", "size": "10073" } }, "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 3" ], "appIcon": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/rose.thumb.jpg", "primaryCategory": "eTextbook", "contentEncoding": "identity", "artifactUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/cbse-copy-88.pdf", "contentType": "eTextBook", "se_gradeLevels": [ "Class 3" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21353901750772531217809", "audience": [ "Student" ], "se_boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "visibility": "Default", "author": "Aparna", "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "56c84dee-7149-47af-902d-0138e080cec0", "version": 2, "pragma": [ "external" ], "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "size": 209850.0, "lastPublishedOn": "2022-05-16T17:38:53.877+0000", "name": "Aparna 86", "topic": [ "Water" ], "mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "attributions": [ "Nadiya Anusha" ], "status": "Live", "topicsIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "code": "3d09fc2e-5705-8f9f-2e1d-4c9c8fd19de2", "interceptionPoints": {}, "credentials": { "enabled": "No" }, "prevStatus": "Processing", "origin": "do_21353901750772531217809", "description": "about water", "streamingUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901750772531217809/artifact/cbse-copy-88.pdf", "medium": [ "English" ], "posterImage": "https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_21353901750772531217809/artifact/rose.jpg", "idealScreenSize": "normal", "createdOn": "2022-05-16T17:38:50.225+0000", "se_boards": [ "CBSE" ], "se_mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "processId": "497335f0-51d2-4cec-a3ba-6eb81309aa87", "contentDisposition": "inline", "lastUpdatedOn": "2022-05-16T17:38:54.005+0000", "originData": { "identifier": "do_21353901750772531217809", "repository": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901750772531217809" }, "se_topicIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-16T17:38:54.005+0000", "createdFor": [ "01345815127107174426" ], "creator": "Aparna", "os": [ "All" ], "se_subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "se_FWIds": [ "ekstep_ncert_k-12" ], "pkgVersion": 1, "versionKey": "1652722732649", "idealScreenDensity": "hdpi", "framework": "ekstep_ncert_k-12", "depth": 2, "boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "lastSubmittedOn": "2022-05-16T17:38:52.212+0000", "createdBy": "c16ccf69-bd42-4dd3-a1e6-435322e78c97", "se_topics": [ "Water" ], "compatibilityLevel": 4, "gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "board": "CBSE", "programId": "9ae297d0-d51a-11ec-907e-e7dd0cd587df" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T11:01:38.790+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354032495648768012324", "lastStatusChangedOn": "2022-05-18T11:01:38.790+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 3, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "versionKey": "1652871698790", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "D1D5H9" ], "depth": 1, "compatibilityLevel": 1, "name": "Ch3", "attributions": [], "status": "Draft" }, { "ownershipType": [ "createdBy" ], "parent": "do_21354027142511820812318", "code": "bf120af0-d50b-b700-baca-1f30214bdb27", "credentials": { "enabled": "No" }, "channel": "01269878797503692810", "language": [ "English" ], "mimeType": "application/vnd.ekstep.content-collection", "idealScreenSize": "normal", "createdOn": "2022-05-18T11:01:38.794+0000", "objectType": "Content", "primaryCategory": "Textbook Unit", "children": [ { "ownershipType": [ "createdBy" ], "parent": "do_21354032495652044812326", "copyright": "2013", "se_gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "previewUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "keywords": [ "Drop" ], "subject": [ "English" ], "channel": "01345815127107174426", "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar", "language": [ "English" ], "source": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149", "mimeType": "application/pdf", "variants": { "full": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar", "size": "193301" }, "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745316_do_21353901951355289618149_1_SPINE.ecar", "size": "10093" } }, "objectType": "Content", "se_mediums": [ "English" ], "gradeLevel": [ "Class 3" ], "appIcon": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/rose.thumb.jpg", "primaryCategory": "eTextbook", "contentEncoding": "identity", "artifactUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "contentType": "eTextBook", "se_gradeLevels": [ "Class 3" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21353901951355289618149", "audience": [ "Student" ], "se_boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "visibility": "Default", "author": "Aparna", "discussionForum": { "enabled": "No" }, "index": 1, "mediaType": "content", "osId": "org.ekstep.quiz.app", "languageCode": [ "en" ], "lastPublishedBy": "56c84dee-7149-47af-902d-0138e080cec0", "version": 2, "pragma": [ "external" ], "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "size": 209850.0, "lastPublishedOn": "2022-05-16T17:39:05.200+0000", "name": "Aparna 87", "topic": [ "Water" ], "mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "attributions": [ "Nadiya Anusha" ], "status": "Live", "topicsIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "code": "2bf23df3-3fa3-c5ad-e1d0-6784a96cb2b8", "interceptionPoints": {}, "credentials": { "enabled": "No" }, "prevStatus": "Processing", "origin": "do_21353901951355289618149", "description": "about water", "streamingUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "medium": [ "English" ], "posterImage": "https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_21353901951355289618149/artifact/rose.jpg", "idealScreenSize": "normal", "createdOn": "2022-05-16T17:39:01.560+0000", "se_boards": [ "CBSE" ], "se_mediumIds": [ "ekstep_ncert_k-12_medium_english" ], "processId": "497335f0-51d2-4cec-a3ba-6eb81309aa87", "contentDisposition": "inline", "lastUpdatedOn": "2022-05-16T17:39:05.366+0000", "originData": { "identifier": "do_21353901951355289618149", "repository": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149" }, "se_topicIds": [ "ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "dialcodeRequired": "No", "lastStatusChangedOn": "2022-05-16T17:39:05.366+0000", "createdFor": [ "01345815127107174426" ], "creator": "27aprilindividual@yopmail.com", "os": [ "All" ], "se_subjectIds": [ "ekstep_ncert_k-12_subject_english" ], "se_FWIds": [ "ekstep_ncert_k-12" ], "pkgVersion": 1, "versionKey": "1652722743949", "idealScreenDensity": "hdpi", "framework": "ekstep_ncert_k-12", "depth": 2, "boardIds": [ "ekstep_ncert_k-12_board_cbse" ], "lastSubmittedOn": "2022-05-16T17:39:03.621+0000", "createdBy": "c6698266-f5d8-4e72-a831-5da9b3205d16", "se_topics": [ "Water" ], "compatibilityLevel": 4, "gradeLevelIds": [ "ekstep_ncert_k-12_gradelevel_class3" ], "board": "CBSE", "programId": "9ae297d0-d51a-11ec-907e-e7dd0cd587df" } ], "contentDisposition": "inline", "lastUpdatedOn": "2022-05-18T11:01:38.794+0000", "contentEncoding": "gzip", "generateDIALCodes": "No", "contentType": "TextBookUnit", "dialcodeRequired": "Yes", "identifier": "do_21354032495652044812326", "lastStatusChangedOn": "2022-05-18T11:01:38.794+0000", "audience": [ "Student" ], "os": [ "All" ], "visibility": "Parent", "discussionForum": { "enabled": "No" }, "index": 4, "mediaType": "content", "osId": "org.ekstep.launcher", "languageCode": [ "en" ], "version": 2, "versionKey": "1652871698794", "license": "CC BY 4.0", "idealScreenDensity": "hdpi", "dialcodes": [ "F6I6S6" ], "depth": 1, "compatibilityLevel": 1, "name": "Ch4", "attributions": [], "status": "Draft" } ], "appId": "staging.sunbird.portal", "contentEncoding": "gzip", "lockKey": "3e222f39-d917-4582-96ee-458f1ab80f30", "generateDIALCodes": "Yes", "totalCompressedSize": 209850, "mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":8,\"\":1}", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T11:02:51.732+0000", "contentType": "TextBook", "se_gradeLevels": [ "Class 1" ], "trackable": { "enabled": "No", "autoBatch": "No" }, "identifier": "do_21354027142511820812318", "audience": [ "Student" ], "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "subjectIds": [ "tn_k-12_5_subject_english" ], "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json", "visibility": "Default", "contentTypesCount": "{\"TextBookUnit\":6,\"eTextBook\":1,\"Course\":1,\"CourseUnit\":1,\"\":1}", "author": "ContentcreatorTN", "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "childNodes": [ "do_21354022673739776012312", "do_21354031968951500812320", "do_21353947184001843212121", "do_21354031968955596812322", "do_21353901750772531217809", "do_21354032495648768012324", "do_21353901951355289618149", "do_21354032495652044812326" ], "discussionForum": { "enabled": "No" }, "mediaType": "content", "osId": "org.ekstep.quiz.app", "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "languageCode": [ "en" ], "version": 2, "se_subjects": [ "English" ], "license": "CC BY 4.0", "prevState": "Review", "qrCodeProcessId": "40c3e296-ccdc-487d-8cda-9f0a88c071d2", "size": 16926, "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "name": "DialCodeHierarchy", "mediumIds": [ "tn_k-12_5_medium_english" ], "status": "Draft", "code": "org.sunbird.D3eL5z", "credentials": { "enabled": "No" }, "prevStatus": "Processing", "description": "Enter description for TextBook", "medium": [ "English" ], "idealScreenSize": "normal", "createdOn": "2022-05-18T09:12:44.203+0000", "reservedDialcodes": { "A3S8L6": 2, "R8C7G2": 1, "D1D5H9": 4, "V2A8N1": 0, "L5L9A2": 5, "F6I6S6": 3 }, "se_boards": [ "State (Tamil Nadu)" ], "se_mediumIds": [ "tn_k-12_5_medium_english" ], "copyrightYear": 2022, "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "lastUpdatedOn": "2022-05-18T10:53:41.611+0000", "dialcodeRequired": "Yes", "lastStatusChangedOn": "2022-05-18T11:01:39.892+0000", "createdFor": [ "01269878797503692810" ], "creator": "ContentcreatorTN", "os": [ "All" ], "se_subjectIds": [ "tn_k-12_5_subject_english" ], "se_FWIds": [ "tn_k-12_5" ], "pkgVersion": 1, "versionKey": "1652871771396", "idealScreenDensity": "hdpi", "framework": "tn_k-12_5", "dialcodes": [ "V2A8N1" ], "depth": 0, "s3Key": "content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json", "boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "lastSubmittedOn": "2022-05-18T10:53:03.017+0000", "createdBy": "4cd4c690-eab6-4938-855a-447c7b1b8ea9", "compatibilityLevel": 1, "leafNodesCount": 1, "userConsent": "Yes", "gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "board": "State (Tamil Nadu)", "resourceType": "Book" }' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy) VALUES ( -'do_21354027142511820812318', -'{"ownershipType": [ "createdBy"],"publish_type": "public","copyright": "tn","se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1"],"keywords": [ "வாக்கியங்கள்", "വളരെ", "बतख़!", "Drop", "മലയാളം"],"subject": [ "English"],"channel": "01269878797503692810","downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar","organisation": [ "Tamil Nadu"],"language": [ "English"],"mimeType": "application/vnd.ekstep.content-collection","variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar\",\"size\":\"16926\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar\",\"size\":\"8313\"}}","body": null,"leafNodes": [ "do_21353901951355289618149"],"objectType": "Content","se_mediums": [ "English"],"gradeLevel": [ "Class 1"],"appIcon": "","children": [ { "lastStatusChangedOn": "2022-05-18T10:50:55.849+0000", "parent": "do_21354027142511820812318", "children": [{ "copyright": "tn", "lastStatusChangedOn": "2022-05-18T07:41:49.160+0000", "publish_type": "public", "parent": "do_21354031968951500812320", "author": "Guest name changed", "se_mediumIds": [ "tn_k-12_5_medium_english" ], "organisation": [ "Tamil Nadu", "MPPS GYARAGONDANAHALLI" ], "children": [ {"lastStatusChangedOn": "2022-05-18T07:43:11.157+0000","parent": "do_21354022673739776012312","children": [ { "copyright": "2013", "lastStatusChangedOn": "2022-05-16T17:39:05.366+0000", "originData": "{\"identifier\":\"do_21353901951355289618149\",\"repository\":\"https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149\"}", "parent": "do_21354022740915814412313", "author": "Aparna", "se_mediumIds": ["ekstep_ncert_k-12_medium_english" ], "mediaType": "content", "name": "Aparna 87", "se_topicIds": ["ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "discussionForum": "{\"enabled\":\"No\"}", "createdOn": "2022-05-16T17:39:01.560+0000", "createdFor": ["01345815127107174426" ], "channel": "01345815127107174426", "source": "https://dockstaging.sunbirded.org/api/content/v1/read/do_21353901951355289618149", "lastUpdatedOn": "2022-05-16T17:39:05.366+0000", "subject": ["English" ], "size": 209850.0, "se_topics": ["Water" ], "streamingUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "identifier": "do_21353901951355289618149", "se_gradeLevelIds": ["ekstep_ncert_k-12_gradelevel_class3" ], "description": "about water", "gradeLevel": ["Class 3" ], "ownershipType": ["createdBy" ], "mediumIds": ["ekstep_ncert_k-12_medium_english" ], "compatibilityLevel": 4, "audience": ["Student" ], "trackable": "{\"enabled\":\"No\",\"autoBatch\":\"No\"}", "se_boards": ["CBSE" ], "os": ["All" ], "primaryCategory": "eTextbook", "appIcon": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/rose.thumb.jpg", "languageCode": ["en" ], "se_mediums": ["English" ], "se_subjectIds": ["ekstep_ncert_k-12_subject_english" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar", "se_subjects": ["English" ], "medium": ["English" ], "attributions": ["Nadiya Anusha" ], "framework": "ekstep_ncert_k-12", "posterImage": "https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_21353901951355289618149/artifact/rose.jpg", "creator": "27aprilindividual@yopmail.com", "versionKey": "1652722743949", "mimeType": "application/pdf", "code": "2bf23df3-3fa3-c5ad-e1d0-6784a96cb2b8", "se_boardIds": ["ekstep_ncert_k-12_board_cbse" ], "license": "CC BY 4.0", "version": 2, "prevStatus": "Processing", "contentType": "eTextBook", "prevState": "Review", "language": ["English" ], "board": "CBSE", "lastPublishedOn": "2022-05-16T17:39:05.200+0000", "objectType": "Content", "origin": "do_21353901951355289618149", "subjectIds": ["ekstep_ncert_k-12_subject_english" ], "status": "Live", "pragma": ["external" ], "programId": "9ae297d0-d51a-11ec-907e-e7dd0cd587df", "createdBy": "c6698266-f5d8-4e72-a831-5da9b3205d16", "dialcodeRequired": "No", "lastSubmittedOn": "2022-05-16T17:39:03.621+0000", "interceptionPoints": "{}", "keywords": ["Drop" ], "idealScreenSize": "normal", "contentEncoding": "identity", "depth": 2, "lastPublishedBy": "56c84dee-7149-47af-902d-0138e080cec0", "topic": ["Water" ], "topicsIds": ["ekstep_ncert_k-12_topic_environmentalstudies_l1con_3" ], "se_gradeLevels": ["Class 3" ], "osId": "org.ekstep.quiz.app", "se_FWIds": ["ekstep_ncert_k-12" ], "contentDisposition": "inline", "previewUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "boardIds": ["ekstep_ncert_k-12_board_cbse" ], "artifactUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/artifact/cbse-copy-89.pdf", "visibility": "Default", "credentials": "{\"enabled\":\"No\"}", "processId": "497335f0-51d2-4cec-a3ba-6eb81309aa87", "variants": "{\"full\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745251_do_21353901951355289618149_1.ecar\",\"size\":\"193301\"},\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353901951355289618149/aparna-87_1652722745316_do_21353901951355289618149_1_SPINE.ecar\",\"size\":\"10093\"}}", "gradeLevelIds": ["ekstep_ncert_k-12_gradelevel_class3" ], "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi" }],"mediaType": "content","name": "Course Unit1","discussionForum": { "enabled": "No"},"createdOn": "2022-05-18T07:43:11.157+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-18T07:43:47.944+0000","identifier": "do_21354022740915814412313","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"trackable": { "enabled": "No", "autoBatch": "No"},"os": [ "All"],"primaryCategory": "Course Unit","languageCode": [ "en"],"downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar","attributions": [],"versionKey": "1652859791157","mimeType": "application/vnd.ekstep.content-collection","code": "8335cfaa-74bc-171d-5c8d-4e895f8fb701","license": "CC BY 4.0","leafNodes": [ "do_21353901951355289618149"],"version": 2,"contentType": "CourseUnit","language": [ "English"],"lastPublishedOn": "2022-05-18T07:44:22.530+0000","objectType": "Content","status": "Live","dialcodeRequired": "No","idealScreenSize": "normal","contentEncoding": "gzip","leafNodesCount": 1,"depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar\",\"size\":\"13797\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862667_do_21354022673739776012312_1_ONLINE.ecar\",\"size\":\"5181\"}}","index": 1,"pkgVersion": 1,"idealScreenDensity": "hdpi" } ], "body": null, "mediaType": "content", "name": "newcert", "toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "batches": [ {"name": "nc","createdFor": [ "01269878797503692810", "01275630040485068839017"],"enrollmentType": "open","endDate": "2022-05-20","enrollmentEndDate": "2022-05-19","status": 1,"batchId": "013540229187338240118","startDate": "2022-05-18" } ], "discussionForum": { "enabled": "Yes" }, "createdOn": "2022-05-18T07:41:49.160+0000", "createdFor": [ "01269878797503692810" ], "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-18T07:43:47.944+0000", "subject": [ "Accountancy" ], "size": 13797, "publishError": null, "targetMediumIds": [ "tn_k-12_5_medium_english" ], "identifier": "do_21354022673739776012312", "se_gradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "description": "Enter description for Course", "resourceType": "Course", "ownershipType": [ "createdBy" ], "compatibilityLevel": 4, "targetBoardIds": [ "tn_k-12_5_board_statetamilnadu" ], "audience": [ "Student" ], "trackable": { "enabled": "Yes", "autoBatch": "No" }, "se_boards": [ "State (Tamil Nadu)" ], "os": [ "All" ], "primaryCategory": "Course", "se_mediums": [ "English" ], "se_subjectIds": [ "tn_k-12_5_subject_accountancy", "tn_k-12_5_subject_english" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar", "se_subjects": [ "Accountancy" ], "lockKey": "3efeaad1-490b-4044-8185-52a421057844", "attributions": [], "framework": "tn_k-12_5", "creator": "Guest name changed", "totalCompressedSize": 209850, "versionKey": "1652859827944", "mimeType": "application/vnd.ekstep.content-collection", "sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T07:44:22.582+0000", "code": "org.sunbird.11SAbB", "se_boardIds": [ "tn_k-12_5_board_statetamilnadu" ], "license": "CC BY 4.0", "leafNodes": [ "do_21353901951355289618149" ], "version": 2, "contentType": "Course", "language": [ "English" ], "lastPublishedOn": "2022-05-18T07:44:22.530+0000", "contentTypesCount": "{\"eTextBook\":1,\"CourseUnit\":1}", "objectType": "Content", "subjectIds": [ "tn_k-12_5_subject_accountancy" ], "status": "Live", "reservedDialcodes": { "U5X4C5": 0 }, "targetFWIds": [ "tn_k-12_5" ], "createdBy": "fca2925f-1eee-4654-9177-fece3fd6afc9", "dialcodeRequired": "No", "lastSubmittedOn": "2022-05-18T07:43:47.614+0000", "keywords": [ "Drop" ], "dialcodes": null, "userConsent": "Yes", "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 1, "depth": 2, "consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada", "lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a", "flagReasons": null, "targetSubjectIds": [ "tn_k-12_5_subject_english" ], "mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":1}", "se_gradeLevels": [ "Class 1" ], "osId": "org.ekstep.quiz.app", "copyrightYear": 2022, "se_FWIds": [ "tn_k-12_5" ], "s3Key": "content/do_21354022673739776012312/artifact/do_21354022673739776012312_toc.json", "contentDisposition": "inline", "additionalCategories": [ "Textbook" ], "childNodes": [ "do_21353901951355289618149", "do_21354022740915814412313" ], "visibility": "Default", "credentials": { "enabled": "Yes" }, "targetGradeLevelIds": [ "tn_k-12_5_gradelevel_class1" ], "variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862583_do_21354022673739776012312_1_SPINE.ecar\",\"size\":\"13797\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354022673739776012312/newcert_1652859862667_do_21354022673739776012312_1_ONLINE.ecar\",\"size\":\"5181\"}}", "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi"} ], "mediaType": "content", "name": "Ch1 ", "discussionForum": {"enabled": "No" }, "createdOn": "2022-05-18T10:50:55.849+0000", "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "identifier": "do_21354031968951500812320", "ownershipType": ["createdBy" ], "compatibilityLevel": 1, "audience": ["Student" ], "os": ["All" ], "primaryCategory": "Textbook Unit", "languageCode": ["en" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "attributions": [], "versionKey": "1652871055849", "mimeType": "application/vnd.ekstep.content-collection", "code": "80986053-eb44-0c35-1c3b-c84806d4775a", "license": "CC BY 4.0", "leafNodes": ["do_21353901951355289618149" ], "version": 2, "contentType": "TextBookUnit", "language": ["English" ], "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "objectType": "Content", "status": "Live", "dialcodeRequired": "Yes", "dialcodes": ["R8C7G2" ], "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 1, "depth": 1, "osId": "org.ekstep.launcher", "contentDisposition": "inline", "visibility": "Parent", "credentials": {"enabled": "No" }, "variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar\",\"size\":\"16926\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar\",\"size\":\"8313\"}}", "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi" }, { "lastStatusChangedOn": "2022-05-18T10:50:55.854+0000", "parent": "do_21354027142511820812318", "children": [{ "parent": "do_21354031968955596812322", "children": [ {"lastStatusChangedOn": "2022-05-17T06:08:04.909+0000","parent": "do_21353947184001843212121","children": [ { "lastStatusChangedOn": "2022-05-17T06:08:47.990+0000", "parent": "do_21353947287477452812131", "mediaType": "content", "name": "വളരെ", "discussionForum": {"enabled": "No" }, "createdOn": "2022-05-17T06:08:47.990+0000", "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-17T06:12:02.265+0000", "identifier": "do_21353947322769408012133", "description": "വളരെ", "ownershipType": ["createdBy" ], "compatibilityLevel": 1, "audience": ["Student" ], "os": ["All" ], "primaryCategory": "Textbook Unit", "languageCode": ["en" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "attributions": [], "versionKey": "1652767727990", "mimeType": "application/vnd.ekstep.content-collection", "code": "f00aabd6-ca25-00de-97c7-44aa0d08b803", "license": "CC BY 4.0", "leafNodes": [], "version": 2, "contentType": "TextBookUnit", "language": ["English" ], "lastPublishedOn": "2022-05-17T06:12:27.957+0000", "objectType": "Content", "status": "Draft", "dialcodeRequired": "No", "keywords": ["വളരെ" ], "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 0, "depth": 2, "osId": "org.ekstep.launcher", "contentDisposition": "inline", "visibility": "Parent", "credentials": {"enabled": "No" }, "variants": {"spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019"},"online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018"} }, "index": 1, "pkgVersion": 1, "idealScreenDensity": "hdpi" }],"mediaType": "content","name": "வாக்கியங்கள்","discussionForum": { "enabled": "No"},"createdOn": "2022-05-17T06:08:04.909+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-17T06:12:02.265+0000","identifier": "do_21353947287477452812131","description": "வாக்கியங்கள்","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"os": [ "All"],"primaryCategory": "Textbook Unit","languageCode": [ "en"],"downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar","attributions": [],"versionKey": "1652767684909","mimeType": "application/vnd.ekstep.content-collection","code": "a1e342e1-a00b-d708-598a-1c52c4fc2cc7","license": "CC BY 4.0","leafNodes": [],"version": 2,"contentType": "TextBookUnit","language": [ "English"],"lastPublishedOn": "2022-05-17T06:12:27.957+0000","objectType": "Content","status": "Draft","dialcodeRequired": "No","keywords": [ "வாக்கியங்கள்"],"idealScreenSize": "normal","contentEncoding": "gzip","leafNodesCount": 0,"depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018" }},"index": 1,"pkgVersion": 1,"idealScreenDensity": "hdpi" }, {"lastStatusChangedOn": "2022-05-17T06:09:30.081+0000","parent": "do_21353947184001843212121","mediaType": "content","name": "बतख़","discussionForum": { "enabled": "No"},"createdOn": "2022-05-17T06:09:30.081+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-17T06:12:02.265+0000","identifier": "do_21353947357250355212135","description": "बतख़","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"os": [ "All"],"primaryCategory": "Textbook Unit","languageCode": [ "en"],"downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar","attributions": [],"versionKey": "1652767770081","mimeType": "application/vnd.ekstep.content-collection","code": "710a43da-9a04-1d67-7d51-17bdce7fc1aa","license": "CC BY 4.0","leafNodes": [],"version": 2,"contentType": "TextBookUnit","language": [ "English"],"lastPublishedOn": "2022-05-17T06:12:27.957+0000","objectType": "Content","status": "Draft","dialcodeRequired": "No","keywords": [ "बतख़!"],"idealScreenSize": "normal","contentEncoding": "gzip","leafNodesCount": 0,"depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"variants": { "spine": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948813_do_21353947184001843212121_1_SPINE.ecar", "size": "4019" }, "online": { "ecarUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21353947184001843212121/knndd_1652767948873_do_21353947184001843212121_1_ONLINE.ecar", "size": "4018" }},"index": 2,"pkgVersion": 1,"idealScreenDensity": "hdpi" }, {"lastStatusChangedOn": "2022-05-17T06:13:07.924+0000","parent": "do_21353947184001843212121","mediaType": "content","name": "മലയാളം","discussionForum": { "enabled": "No"},"createdOn": "2022-05-17T06:13:07.924+0000","channel": "01269878797503692810","generateDIALCodes": "No","lastUpdatedOn": "2022-05-17T06:13:07.924+0000","identifier": "do_21353947535707340812137","description": "മലയാളം","ownershipType": [ "createdBy"],"compatibilityLevel": 1,"audience": [ "Student"],"os": [ "All"],"primaryCategory": "Textbook Unit","languageCode": [ "en"],"attributions": [],"versionKey": "1652767987924","mimeType": "application/vnd.ekstep.content-collection","code": "3e976c16-cefa-f237-f444-d25f2fa8f915","license": "CC BY 4.0","version": 2,"contentType": "TextBookUnit","language": [ "English"],"objectType": "Content","status": "Draft","dialcodeRequired": "No","keywords": [ "മലയാളം"],"idealScreenSize": "normal","contentEncoding": "gzip","depth": 1,"osId": "org.ekstep.launcher","contentDisposition": "inline","visibility": "Parent","credentials": { "enabled": "No"},"index": 3,"idealScreenDensity": "hdpi" } ], "identifier": "do_21353947184001843212121", "objectType": "Content", "depth": 2, "index": 1, "contentType": null, "primaryCategory": null} ], "mediaType": "content", "name": "Ch2", "discussionForum": {"enabled": "No" }, "createdOn": "2022-05-18T10:50:55.854+0000", "channel": "01269878797503692810", "generateDIALCodes": "No", "lastUpdatedOn": "2022-05-18T10:50:56.492+0000", "identifier": "do_21354031968955596812322", "ownershipType": ["createdBy" ], "compatibilityLevel": 1, "audience": ["Student" ], "os": ["All" ], "primaryCategory": "Textbook Unit", "languageCode": ["en" ], "downloadUrl": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar", "attributions": [], "versionKey": "1652871055854", "mimeType": "application/vnd.ekstep.content-collection", "code": "22c05b75-ae2a-6280-c192-2e2a4157d141", "license": "CC BY 4.0", "leafNodes": [], "version": 2, "contentType": "TextBookUnit", "language": ["English" ], "lastPublishedOn": "2022-05-18T10:53:41.260+0000", "objectType": "Content", "status": "Live", "dialcodeRequired": "Yes", "dialcodes": ["A3S8L6" ], "idealScreenSize": "normal", "contentEncoding": "gzip", "leafNodesCount": 0, "depth": 1, "osId": "org.ekstep.launcher", "contentDisposition": "inline", "visibility": "Parent", "credentials": {"enabled": "No" }, "variants": "{\"spine\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221390_do_21354027142511820812318_1_SPINE.ecar\",\"size\":\"16926\"},\"online\":{\"ecarUrl\":\"https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/dialcodehierarchy_1652871221531_do_21354027142511820812318_1_ONLINE.ecar\",\"size\":\"8313\"}}", "index": 2, "pkgVersion": 1, "idealScreenDensity": "hdpi" }],"primaryCategory": "Digital Textbook","appId": "staging.sunbird.portal","contentEncoding": "gzip","lockKey": "3e222f39-d917-4582-96ee-458f1ab80f30","generateDIALCodes": "Yes","totalCompressedSize": 209850,"mimeTypesCount": "{\"application/pdf\":1,\"application/vnd.ekstep.content-collection\":8,\"\":1}","sYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T10:53:41.389+0000","contentType": "TextBook","se_gradeLevels": [ "Class 1"],"trackable": { "enabled": "No", "autoBatch": "No"},"identifier": "do_21354027142511820812318","audience": [ "Student"],"se_boardIds": [ "tn_k-12_5_board_statetamilnadu"],"subjectIds": [ "tn_k-12_5_subject_english"],"toc_url": "https://sunbirdstagingpublic.blob.core.windows.net/sunbird-content-staging/content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json","visibility": "Default","contentTypesCount": "{\"TextBookUnit\":6,\"eTextBook\":1,\"Course\":1,\"CourseUnit\":1,\"\":1}","author": "ContentcreatorTN","consumerId": "cb069f8d-e4e1-46c5-831f-d4a83b323ada","childNodes": [ "do_21354022673739776012312", "do_21354031968951500812320", "do_21353947184001843212121", "do_21354031968955596812322", "do_21353901951355289618149", "do_21354022740915814412313"],"discussionForum": { "enabled": "No"},"mediaType": "content","osId": "org.ekstep.quiz.app","lastPublishedBy": "08631a74-4b94-4cf7-a818-831135248a4a","version": 2,"se_subjects": [ "English"],"license": "CC BY 4.0","qrCodeProcessId": "40c3e296-ccdc-487d-8cda-9f0a88c071d2","size": 16926,"lastPublishedOn": "2022-05-18T10:53:41.260+0000","name": "DialCodeHierarchy","mediumIds": [ "tn_k-12_5_medium_english"],"attributions": [],"status": "Live","code": "org.sunbird.D3eL5z","publishError": null,"credentials": { "enabled": "No"},"description": "Enter description for TextBook","medium": [ "English"],"idealScreenSize": "normal","createdOn": "2022-05-18T09:12:44.203+0000","reservedDialcodes": "{\"A3S8L6\":2,\"R8C7G2\":1,\"D1D5H9\":4,\"V2A8N1\":0,\"L5L9A2\":5,\"F6I6S6\":3}","se_boards": [ "State (Tamil Nadu)"],"se_mediumIds": [ "tn_k-12_5_medium_english"],"copyrightYear": 2022,"contentDisposition": "inline","additionalCategories": [ "Textbook"],"lastUpdatedOn": "2022-05-18T10:50:56.492+0000","SYS_INTERNAL_LAST_UPDATED_ON": "2022-05-18T10:52:07.401+0000","dialcodeRequired": "Yes","lastStatusChangedOn": "2022-05-18T09:12:44.203+0000","createdFor": [ "01269878797503692810"],"creator": "ContentcreatorTN","os": [ "All"],"flagReasons": null,"se_subjectIds": [ "tn_k-12_5_subject_english"],"se_FWIds": [ "tn_k-12_5"],"pkgVersion": 1,"versionKey": "1652871183414","idealScreenDensity": "hdpi","framework": "tn_k-12_5","dialcodes": [ "V2A8N1"],"depth": 0,"s3Key": "content/do_21354027142511820812318/artifact/do_21354027142511820812318_toc.json","boardIds": [ "tn_k-12_5_board_statetamilnadu"],"lastSubmittedOn": "2022-05-18T10:53:03.017+0000","createdBy": "4cd4c690-eab6-4938-855a-447c7b1b8ea9","compatibilityLevel": 1,"leafNodesCount": 1,"userConsent": "Yes","gradeLevelIds": [ "tn_k-12_5_gradelevel_class1"],"board": "State (Tamil Nadu)","resourceType": "Book" }' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy) VALUES ( -'do_1234', -'{"identifier":"do_1234","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_12345', -'{"identifier":"do_12345","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}', -'' -); - -INSERT INTO dev_hierarchy_store.content_hierarchy(identifier, hierarchy, relational_metadata) VALUES ( -'do_123456', -'{"identifier":"do_123456","children":[{"ownershipType":["createdBy"],"parent":"do_123","code":"7cf20ea47763d420865bcc713def7a7b","keywords":["UnitKW1","UnitKW2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.617+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","code":"9e1862f6518a7c87ee693cebb4fec278","keywords":["UnitKW1L2","UnitKW2L2"],"credentials":{"enabled":"No"},"channel":"0126825293972439041","description":"This section describes about various part of the body such as head, hands, legs etc.","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-12-21T19:54:31.660+0530","objectType":"Collection","primaryCategory":"Textbook Unit","children":[{"ownershipType":["createdBy"],"parent":"do_11343567193423872014","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":3,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.840+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11343567193423872014","lastStatusChangedOn":"2021-12-21T19:54:31.661+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671660","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":2,"compatibilityLevel":1,"name":"L2 Folder","attributions":[],"status":"Draft"},{"ownershipType":["createdBy"],"parent":"do_11340511137112064018","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","keywords":["CPPDFContent1","CPPDFContent2","CollectionKW1"],"channel":"0126825293972439041","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","language":["English"],"mimeType":"application/pdf","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar","size":"256918"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar","size":"6378"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","contentType":"Resource","identifier":"do_11340096165525094411","audience":["Student"],"visibility":"Default","discussionForum":{"enabled":"No"},"index":2,"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"","version":2,"pragma":["external"],"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-11-02T19:13:35.589+0530","name":"Collection Publishing PDF Content","status":"Live","code":"c9ce1ce0-b9b4-402e-a9c3-556701070838","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf","idealScreenSize":"normal","createdOn":"2021-11-02T18:56:17.917+0530","copyrightYear":2021,"contentDisposition":"inline","lastUpdatedOn":"2021-11-02T19:13:39.729+0530","dialcodeRequired":"No","lastStatusChangedOn":"2021-11-02T19:13:39.729+0530","createdFor":["01309282781705830427"],"creator":"N131","os":["All"],"se_FWIds":["ekstep_ncert_k-12"],"pkgVersion":1,"versionKey":"1635859577917","idealScreenDensity":"hdpi","framework":"ekstep_ncert_k-12","depth":2,"createdBy":"0b71985d-fcb0-4018-ab14-83f10c3b0426","compatibilityLevel":4,"resourceType":"Learn"}],"contentDisposition":"inline","lastUpdatedOn":"2021-12-21T19:55:11.837+0530","contentEncoding":"gzip","generateDIALCodes":"No","contentType":"TextBookUnit","dialcodeRequired":"No","identifier":"do_11340511137112064018","lastStatusChangedOn":"2021-12-21T19:54:31.617+0530","audience":["Student"],"os":["All"],"visibility":"Parent","discussionForum":{"enabled":"No"},"index":1,"mediaType":"content","osId":"org.ekstep.launcher","languageCode":["en"],"version":2,"versionKey":"1640096671617","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"ncert_k-12","depth":1,"compatibilityLevel":1,"name":"Collection Parent","attributions":[],"status":"Draft"}]}', -'{}' -); diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/domain/EventSpec.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/domain/EventSpec.scala deleted file mode 100644 index 62c25675c..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/domain/EventSpec.scala +++ /dev/null @@ -1,21 +0,0 @@ -package org.sunbird.job.livenodepublisher.domain - -import com.typesafe.config.{Config, ConfigFactory} -import org.scalatest.{FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.livenodepublisher.publish.domain.Event -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.util.JSONUtil - -class EventSpec extends FlatSpec with Matchers with MockitoSugar { - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: LiveNodePublisherConfig = new LiveNodePublisherConfig(config) - - "isValid" should "return true for a valid event" in { - val sunbirdEvent = "{\"eid\":\"BE_JOB_REQUEST\",\"ets\":1619527882745,\"mid\":\"LP.1619527882745.32dc378a-430f-49f6-83b5-bd73b767ad36\",\"actor\":{\"id\":\"collection-publish\",\"type\":\"System\"},\"context\":{\"channel\":\"ORG_001\",\"pdata\":{\"id\":\"org.sunbird.platform\",\"ver\":\"1.0\"},\"env\":\"dev\"},\"object\":{\"id\":\"do_11361948306824396811\",\"ver\":\"1619153418829\"},\"edata\":{\"publish_type\":\"public\",\"metadata\":{\"identifier\":\"do_11361948306824396811\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"objectType\":\"Collection\",\"lastPublishedBy\":\"\",\"pkgVersion\":1},\"action\":\"republish\",\"iteration\":1}}" - val event = new Event(JSONUtil.deserialize[java.util.Map[String, Any]](sunbirdEvent),0,1) - - assert(event.validEvent(jobConfig)) - } -} \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/fixture/EventFixture.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/fixture/EventFixture.scala deleted file mode 100644 index 27ecd7fac..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/fixture/EventFixture.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.sunbird.job.livenodepublisher.fixture - -object EventFixture { - - val PDF_EVENT1: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1619527882745,"mid":"LP.1619527882745.32dc378a-430f-49f6-83b5-bd73b767ad36","actor":{"id":"content-publish","type":"System"},"context":{"channel":"","pdata":{"id":"org.sunbird.platform","ver":"1.0"}},"object":{"id":"do_11329603741667328018","ver":"1619153418829"},"edata":{"publish_type":"public","metadata":{"identifier":"do_11329603741667328018","mimeType":"application/pdf","objectType":"Content","lastPublishedBy":"sample-last-published-by","pkgVersion":1},"action":"publish","iteration":1}} - |""".stripMargin -} diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/ExtractableMimeTypeHelperSpec.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/ExtractableMimeTypeHelperSpec.scala deleted file mode 100644 index 88cfad14f..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/ExtractableMimeTypeHelperSpec.scala +++ /dev/null @@ -1,165 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.livenodepublisher.publish.helpers.ExtractableMimeTypeHelper -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.CloudStorageUtil - -class ExtractableMimeTypeHelperSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val ec = ExecutionContexts.global - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: LiveNodePublisherConfig = new LiveNodePublisherConfig(config) - implicit val cloudStorageUtil = new CloudStorageUtil(jobConfig) - - "processECMLBody with xml " should " throw exception Error! Invalid Media ('id' is required.) in ... if media id is blank and type is other than js and css" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with xml " should " throw exception Error! Invalid Media ('type' is required.) in ... if media type is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with xml " should " throw exception Error! Invalid Media ('src' is required.) in ... if media src is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with xml " should " process the ecml body" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "

A person having less ‘haemoglobin’ is suffering from:

Jaundice

Anaemia

Malaria

Chikungunya

\",\"media\":[],\"responseDeclaration\":{\"responseValue\":{\"cardinality\":\"single\",\"type\":\"integer\",\"correct_response\":{\"value\":\"2\"}}},\"options\":[{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Jaundice

\",\"resvalue\":0,\"resindex\":0}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Anaemia

\",\"resvalue\":1,\"resindex\":1}},{\"answer\":true,\"value\":{\"type\":\"text\",\"body\":\"

Malaria

\",\"resvalue\":2,\"resindex\":2}},{\"answer\":false,\"value\":{\"type\":\"text\",\"body\":\"

Chikungunya

\",\"resvalue\":3,\"resindex\":3}}],\"questionCount\":0}]]>")) - ) - val result: Map[String, AnyRef] = ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - result.contains("artifactUrl") shouldBe true - result.get("artifactUrl") shouldNot be(null) - result.contains("cloudStorageKey") shouldBe true - result.get("cloudStorageKey") shouldNot be(null) - } - - "processECMLBody with json " should " throw exception if no body is available" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]()) - ) - - assertThrows[InvalidInputException] { - val result: Map[String, AnyRef] = ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Invalid Content Body if not a valid body" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "Invalid body")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Controller ('id' is required.)" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"\",\"__cdata\":{}}]}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Controller ('type' is required.)" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"\",\"id\":\"dictionary\",\"__cdata\":{}}]}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Controller ('type' should be either 'items' or 'data')" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"some type\",\"id\":\"dictionary\",\"__cdata\":{}}]}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Media ('id' is required.) if media id is blank and type is other than js and css" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"manifest\":{\"media\":[{\"id\":\"\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png\",\"assetId\":\"QuizImage\",\"type\":\"image\",\"preload\":true}]}}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Media ('type' is required.) if media type is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"manifest\":{\"media\":[{\"id\":\"QuizImage\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png\",\"assetId\":\"QuizImage\",\"type\":\"\",\"preload\":true}]}}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " throw exception Error! Invalid Media ('src' is required.) if media src is blank" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"controller\":[{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"manifest\":{\"media\":[{\"id\":\"QuizImage\",\"src\":\"\",\"assetId\":\"QuizImage\",\"type\":\"image\",\"preload\":true}]}}}")) - ) - - assertThrows[InvalidInputException] { - ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - } - } - - "processECMLBody with json " should " process the ecml body" in { - val obj: ObjectData = new ObjectData("do_113188615625731", - Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Content", "mimeType" -> "application/vnd.ekstep.ecml-archive", "primaryCategory" -> "some category", "name" -> "Some Content", "code" -> "some code"), - Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"manifest\":{\"media\":[{\"id\":253452185,\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/controller/navigation_ctrl.js\",\"type\":\"js\"},{\"id\":\"c264cc74-39dc-40e6-8e6a-04b1b379bd52\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/templates/navigation.html\",\"type\":\"js\"},{\"id\":\"org.ekstep.navigation\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.navigation_manifest\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"org.ekstep.questionunit.renderer.audioicon\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/audio-icon.png\",\"type\":\"image\"},{\"id\":\"org.ekstep.questionunit.renderer.downarrow\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/assets/down_arrow.png\",\"type\":\"image\"},{\"id\":\"489ca96e-de6d-41c2-a346-3e13a5e81cd1\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/js/components.js\",\"type\":\"js\"},{\"id\":\"d3eab6b4-eb5a-4739-b985-ca85caaae11e\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/components/css/components.css\",\"type\":\"css\"},{\"id\":\"4152517e-7698-4813-840c-ab0d4bb1a48a\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.js\",\"type\":\"js\"},{\"id\":\"d9a519d7-e450-4b31-be54-5cec9d21d5ce\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/libs/katex/katex.min.css\",\"type\":\"css\"},{\"id\":\"org.ekstep.questionunit\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionunit_manifest\",\"plugin\":\"org.ekstep.questionunit\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionunit-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"7139ec8e-dc5d-437b-a7c4-62d0be5a28c5\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/style.css\",\"type\":\"css\"},{\"id\":\"bdd3831b-95ca-4398-a6e6-6a61984452a5\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/styles/horizontal_and_vertical.css\",\"type\":\"css\"},{\"id\":\"09bd898a-6132-4ebd-936c-5437a2134229\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/mcq-layouts.js\",\"type\":\"js\"},{\"id\":\"a46352ba-dccb-4c5e-bc1a-84beeb85037c\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/template/template_controller.js\",\"type\":\"js\"},{\"id\":\"38886295-0bac-4654-bded-6d032cb37431\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/tick_icon.png\",\"type\":\"image\"},{\"id\":\"37aabee2-1538-4391-8d1d-5d406722b084\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/audio-icon2.png\",\"type\":\"image\"},{\"id\":\"73d9743a-9202-4e68-8797-ad873b67cdb6\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/assets/music-blue.png\",\"type\":\"image\"},{\"id\":\"org.ekstep.questionunit.mcq\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionunit.mcq_manifest\",\"plugin\":\"org.ekstep.questionunit.mcq\",\"ver\":\"1.1\",\"src\":\"/content-plugins/org.ekstep.questionunit.mcq-1.1/manifest.json\",\"type\":\"json\"},{\"id\":\"org.ekstep.questionset.quiz\",\"plugin\":\"org.ekstep.questionset.quiz\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset.quiz-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionset.quiz_manifest\",\"plugin\":\"org.ekstep.questionset.quiz\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset.quiz-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"org.ekstep.iterator\",\"plugin\":\"org.ekstep.iterator\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.iterator-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.iterator_manifest\",\"plugin\":\"org.ekstep.iterator\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.iterator-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"cb4d4669-9291-4583-906c-685ab9a76994\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/telemetry_logger.js\",\"type\":\"js\"},{\"id\":\"dd12429d-d861-4d54-b80d-12047823b058\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/html_audio_plugin.js\",\"type\":\"js\"},{\"id\":\"56254ade-8e4e-4090-b97b-8b9b7bc12620\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/utils/qs_feedback_popup.js\",\"type\":\"js\"},{\"id\":\"org.ekstep.questionset\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.questionset_manifest\",\"plugin\":\"org.ekstep.questionset\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/manifest.json\",\"type\":\"json\"},{\"id\":\"QuizImage\",\"src\":\"/content-plugins/org.ekstep.questionset-1.0/editor/assets/quizimage.png\",\"assetId\":\"QuizImage\",\"type\":\"image\",\"preload\":true}]},\"id\":\"theme\",\"startStage\":\"scene55138843-9f04-487d-bf33-1920769f2c96\",\"ver\":0.2,\"controller\":[{\"name\":\"assessment_185d692a-1841-4a35-96c1-67cebcde42a2\",\"type\":\"items\",\"id\":\"assessment_185d692a-1841-4a35-96c1-67cebcde42a2\",\"__cdata\":{\"total_items\":1,\"subject\":\"domain\",\"code\":\"ItemSet_921ee047-c676-4e0b-9a71-42f371f7b843\",\"type\":\"materialised\",\"lastUpdatedOn\":\"2016-06-15T09:42:19.585+0000\",\"showImmediateFeedback\":true,\"SET_TYPE\":\"MATERIALISED_SET\",\"createdOn\":\"2016-06-15T09:42:19.575+0000\",\"title\":\"Assessment Title\",\"items\":{\"domain_11824\":[{\"model\":{\"keys\":\"र,रा,रि,री,रु,रू,रे,रै,रो,रौ,रं,रः,ट,टा,टि,टी,टु,टू,टे,टै,टो,टौ,टं,टः\"},\"question_audio\":\"domain_10561\",\"subject\":\"domain\",\"template_id\":\"domain_4171\",\"type\":\"ftb\",\"feedback\":\"\",\"qlevel\":\"EASY\",\"title\":\"Story5Q1\",\"question_image\":\"domain_10624\",\"name\":\"Story5Q1\",\"domain\":\"literacy\",\"max_score\":1,\"question\":\"\",\"template\":\"org.ekstep.ftb.barakhadi\",\"answer\":{\"ans1\":{\"value\":\"रोटी\",\"score\":1}},\"code\":\"org.ekstep.assessmentitem.domain_4533\",\"lastUpdatedOn\":\"2016-06-15T09:40:54.289+0000\",\"concepts\":[{\"identifier\":\"LO1\",\"name\":\"Receptive Vocabulary\",\"objectType\":\"Concept\",\"relation\":\"associatedTo\",\"description\":null,\"index\":null}],\"createdOn\":\"2016-05-23T10:09:03.976+0000\",\"lastUpdatedBy\":\"323\",\"used_for\":\"worksheet\",\"owner\":\"323\",\"gradeLevel\":[\"Other\"],\"language\":[\"Hindi\"],\"identifier\":\"domain_4533\",\"media\":[{\"id\":\"domain_10624\",\"type\":\"image\",\"src\":\"https://ekstep-public.s3-ap-southeast-1.amazonaws.com/content/roti_323_1465971255_1465971255545.jpg\",\"asset_id\":\"domain_10624\",\"preload\":true},{\"id\":\"domain_10561\",\"type\":\"audio\",\"src\":\"https://ekstep-public.s3-ap-southeast-1.amazonaws.com/content/story11q4a1_141_1465970351_1465970351272.mp3\",\"asset_id\":\"domain_10561\",\"preload\":true}]}]},\"SET_OBJECT_TYPE_KEY\":\"AssessmentItem\",\"shuffle\":false,\"lastUpdatedBy\":\"323\",\"owner\":\"Ilimi\",\"used_for\":\"assessment\",\"max_score\":1,\"gradeLevel\":[\"Grade 1\"],\"item_sets\":[{\"id\":\"domain_11824\",\"count\":1}],\"language\":[\"English\"],\"identifier\":\"domain_11824\"}},{\"name\":\"dictionary\",\"type\":\"data\",\"id\":\"dictionary\",\"__cdata\":{}}],\"template\":[{\"text\":[{\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":70,\"model\":\"item.title\",\"w\":80,\"x\":10,\"y\":6,\"z-index\":101},{\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":100,\"h\":15,\"id\":\"newText\",\"model\":\"item.ans1\",\"valign\":\"middle\",\"w\":35,\"x\":58,\"y\":68,\"z-index\":100}],\"shape\":[{\"event\":{\"type\":\"click\"},\"h\":15,\"hitArea\":true,\"opacity\":1,\"w\":80,\"x\":10,\"y\":6,\"z-index\":99},{\"event\":{\"action\":{\"asset\":\"bKeyboard\",\"command\":\"custom\",\"id\":\"newText\",\"invoke\":\"switchTarget\",\"type\":\"command\"},\"type\":\"click\"},\"h\":15,\"hitArea\":true,\"stroke\":\"black\",\"stroke-width\":5,\"w\":35,\"x\":58,\"y\":67,\"z-index\":99},{\"event\":{\"action\":{\"asset_model\":\"item.question_audio\",\"command\":\"play\",\"type\":\"command\"},\"type\":\"click\"},\"h\":40,\"hitArea\":true,\"stroke-width\":5,\"w\":20,\"x\":10,\"y\":58,\"z-index\":99}],\"keyboard\":{\"id\":\"bKeyboard\",\"keys\":\"item.keys\",\"limit\":10,\"target\":\"newText\",\"type\":\"custom\",\"x\":5,\"y\":15},\"g\":{\"image\":{\"h\":75,\"model\":\"item.question_image\",\"w\":100,\"x\":0,\"y\":0,\"z-index\":102},\"text\":{\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":\"2em\",\"h\":25,\"model\":\"item.question\",\"valign\":\"middle\",\"w\":100,\"x\":0,\"y\":75,\"z-index\":103},\"h\":40,\"w\":20,\"x\":10,\"y\":58},\"id\":\"org.ekstep.ftb.barakhadi\"},{\"text\":{\"event\":{\"action\":[{\"command\":\"stop\",\"sound\":true,\"type\":\"command\"},{\"asset_model\":\"item.question_audio\",\"command\":\"play\",\"type\":\"command\"}],\"type\":\"click\"},\"align\":\"center\",\"color\":\"black\",\"font\":\"Verdana\",\"fontsize\":\"2em\",\"lineHeight\":2,\"model\":\"item.question\",\"valign\":\"middle\",\"w\":100,\"x\":0,\"y\":15},\"shape\":{\"event\":{\"action\":[{\"command\":\"stop\",\"sound\":true,\"type\":\"command\"},{\"asset_model\":\"item.question_audio\",\"command\":\"play\",\"type\":\"command\"}],\"type\":\"click\"},\"h\":10,\"hitArea\":true,\"type\":\"rect\",\"w\":100,\"x\":0,\"y\":15},\"mcq\":{\"options\":{\"shape\":{\"h\":100,\"stroke\":\"black\",\"type\":\"roundrect\",\"w\":100,\"x\":0,\"y\":0},\"text\":{\"align\":\"center\",\"font\":\"Verdana\",\"fontsize\":\"1.6em\",\"h\":100,\"model\":\"option.value.text\",\"valign\":\"middle\",\"w\":100,\"x\":0,\"y\":0},\"event\":{\"action\":[{\"command\":\"stop\",\"sound\":true,\"type\":\"command\"},{\"asset_model\":\"option.value.audio\",\"command\":\"play\",\"type\":\"command\"}],\"type\":\"click\"},\"cols\":4,\"h\":30,\"highlight\":\"yellow\",\"layout\":\"table\",\"marginX\":5,\"marginY\":5,\"options\":\"options\",\"w\":97,\"x\":1.5,\"y\":50},\"model\":\"item\",\"multi_select\":false,\"shadow\":\"#56B1F7\"},\"id\":\"org.ekstep.mcq.ta\"},{\"text\":{\"model\":\"item.title\",\"x\":9,\"y\":7,\"w\":86,\"h\":4,\"font\":\"Georgia\",\"fontsize\":42},\"mcq\":{\"options\":{\"layout\":\"table\",\"x\":20,\"y\":15,\"w\":70,\"h\":85,\"cols\":2,\"marginX\":10,\"marginY\":5,\"options\":\"options\"},\"multi_select\":false,\"model\":\"item\"},\"g\":{\"image\":{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"stop\",\"asset_model\":\"item.hints[0].asset\"},{\"type\":\"command\",\"command\":\"toggleShow\",\"asset\":\"hint\"}],\"type\":\"click\"},\"asset\":\"speech_bubble\",\"x\":0,\"y\":0,\"w\":100,\"h\":100},\"text\":[{\"x\":10,\"y\":20,\"w\":80,\"h\":80,\"font\":\"Georgia\",\"weight\":\"bold\",\"fontsize\":150,\"__text\":\"Hint\"},{\"x\":10,\"y\":40,\"w\":80,\"h\":80,\"font\":\"Georgia\",\"fontsize\":120,\"model\":\"item.hints[1].asset\"}],\"x\":9,\"y\":17,\"w\":20,\"h\":20,\"id\":\"hint\",\"visible\":false},\"image\":{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"togglePlay\",\"asset_model\":\"item.hints[0].asset\"},{\"type\":\"command\",\"command\":\"toggleShow\",\"asset\":\"hint\"}],\"type\":\"click\"},\"asset\":\"icon_hint\",\"x\":5,\"y\":35},\"id\":\"mcq_template_1\"},{\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"show\",\"asset\":\"retryDialog\"},\"type\":\"click\"},\"asset\":\"popupTint\",\"x\":-100,\"y\":-150,\"w\":550,\"h\":600,\"visible\":true,\"id\":\"popup-Tint\"},{\"asset\":\"retryBg\",\"x\":0,\"y\":0,\"w\":150,\"h\":150,\"visible\":true,\"id\":\"right\"}],\"shape\":[{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"hide\",\"asset\":\"retryDialog\"},{\"type\":\"command\",\"command\":\"SHOWHTMLELEMENTS\",\"asset\":\"retry\"}],\"type\":\"click\"},\"type\":\"roundrect\",\"x\":72,\"y\":25,\"w\":50,\"h\":65,\"visible\":true,\"id\":\"retry\",\"hitArea\":true},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"type\":\"roundrect\",\"x\":110,\"y\":100,\"w\":25,\"h\":35,\"visible\":true,\"id\":\"continue\",\"hitArea\":true}],\"id\":\"retry\"},{\"g\":{\"image\":[{\"asset\":\"popupTint\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"visible\":true,\"id\":\"popup-Tint\"}],\"text\":[{\"x\":25,\"y\":25,\"w\":50,\"h\":9,\"visible\":true,\"editable\":true,\"model\":\"word.lemma\",\"weight\":\"normal\",\"font\":\"helvetica\",\"color\":\"rgb(0,0,0)\",\"fontstyle\":\"\",\"fontsize\":75,\"align\":\"left\",\"z-index\":1,\"id\":\"lemma\"},{\"x\":25,\"y\":35,\"w\":50,\"h\":40,\"visible\":true,\"editable\":true,\"model\":\"word.gloss\",\"weight\":\"normal\",\"font\":\"helvetica\",\"color\":\"rgb(0,0,0)\",\"fontstyle\":\"\",\"fontsize\":43,\"align\":\"left\",\"z-index\":2,\"id\":\"gloss\"}],\"shape\":[{\"x\":20,\"y\":20,\"w\":60,\"h\":60,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":10,\"opacity\":1,\"fill\":\"#45b3a5\",\"stroke-width\":1,\"z-index\":0,\"id\":\"textBg\"}],\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"SHOWHTMLELEMENTS\",\"asset\":\"textBg\"},{\"type\":\"command\",\"command\":\"hide\",\"parent\":true}],\"type\":\"click\"}},\"id\":\"infoTemplate\"},{\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"show\",\"asset\":\"\"},\"type\":\"click\"},\"asset\":\"popupTint\",\"x\":-100,\"y\":-150,\"w\":550,\"h\":600,\"visible\":true,\"id\":\"popup-Tint\"},{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500}],\"type\":\"click\"},\"asset\":\"goodjobBg\",\"x\":0,\"y\":0,\"w\":150,\"h\":150,\"visible\":true,\"id\":\"continue\"}],\"id\":\"goodjob\"}],\"stage\":[{\"id\":\"recordAudio7\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"previousScreen\",\"value\":\"recordAudio6\"},{\"name\":\"nextScreen\",\"value\":\"recordAudio8\"},{\"name\":\"next\",\"value\":\"scene7b2e5492-b134-4945-9cde-803a094fb430\"},{\"name\":\"previous\",\"value\":\"recordAudio6\"}],\"events\":{\"event\":[{\"action\":[{\"asset\":\"startrec\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"stoprec\",\"command\":\"show\",\"type\":\"command\"},{\"asset\":\"mover\",\"command\":\"show\",\"type\":\"command\"},{\"tween\":{\"to\":[{\"duration\":0,\"ease\":\"linear\",\"__cdata\":{\"x\":47,\"y\":45}},{\"duration\":1000,\"ease\":\"sineInOut\",\"__cdata\":{\"x\":62,\"y\":45}},{\"duration\":1000,\"ease\":\"sineInOut\",\"__cdata\":{\"x\":47,\"y\":45}}],\"id\":\"mover\",\"loop\":true},\"asset\":\"mover\",\"type\":\"animation\"}],\"type\":\"rec_started\"},{\"action\":[{\"asset\":\"stoprec\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"mover\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"playback\",\"command\":\"show\",\"type\":\"command\"}],\"type\":\"rec_stopped\"},{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_4002\",\"loop\":1},\"type\":\"enter\"},{\"action\":{\"type\":\"command\",\"command\":\"stop\",\"asset\":\"domain_4002\",\"loop\":1},\"type\":\"exit\"}]},\"image\":[{\"x\":16.38888888888889,\"y\":20.22222222222222,\"w\":69.44444444444444,\"h\":77.77777777777779,\"visible\":true,\"editable\":true,\"asset\":\"domain_3126\",\"z-index\":0},{\"x\":45,\"y\":80,\"w\":10,\"h\":18,\"visible\":true,\"editable\":true,\"asset\":\"mic\",\"event\":{\"action\":{\"asset\":\"recorder\",\"command\":\"toggleShow\",\"type\":\"command\"},\"type\":\"click\"},\"z-index\":6},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500},\"type\":\"click\"},\"asset\":\"next\",\"x\":93,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"next\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100}],\"text\":[{\"x\":25.27777777777778,\"y\":6.222222222222222,\"w\":71.11111111111111,\"h\":9.322222222222223,\"visible\":true,\"editable\":true,\"__text\":\"बिल्लियाँ देखती ही रह गई |\",\"weight\":\"normal\",\"font\":\"Helvetica\",\"color\":\"rgb(0,0,0)\",\"fontstyle\":\"\",\"fontsize\":85,\"lineHeight\":1.3,\"align\":\"left\",\"z-index\":7}],\"shape\":[],\"hotspot\":[{\"x\":25.208333333333332,\"y\":4.888888888888889,\"w\":14.722222222222223,\"h\":8.444444444444445,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7637\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":1},{\"x\":40.97222222222222,\"y\":5.111111111111112,\"w\":10.277777777777777,\"h\":8,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7638\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":2},{\"x\":52.361111111111114,\"y\":5.111111111111112,\"w\":3.75,\"h\":8.666666666666668,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7639\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":3},{\"x\":57.36111111111111,\"y\":7.333333333333333,\"w\":4.861111111111112,\"h\":6.888888888888889,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7640\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":4},{\"x\":63.47222222222222,\"y\":4.888888888888889,\"w\":5.555555555555555,\"h\":9.777777777777779,\"visible\":true,\"editable\":true,\"type\":\"roundrect\",\"radius\":1,\"fill\":\"red\",\"stroke-width\":1,\"keyword\":\"\",\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_7641\"},\"type\":\"click\"}],\"hitArea\":true,\"z-index\":5}],\"embed\":[],\"div\":[],\"audio\":[{\"asset\":\"domain_4002\"},{\"asset\":\"domain_7637\"},{\"asset\":\"domain_7638\"},{\"asset\":\"domain_7639\"},{\"asset\":\"domain_7640\"},{\"asset\":\"domain_7641\"}],\"scribble\":[],\"g\":[{\"g\":[{\"image\":{\"event\":{\"action\":{\"asset\":\"recorder\",\"command\":\"hide\",\"type\":\"command\"},\"type\":\"click\"},\"asset\":\"blacktint\",\"h\":100,\"w\":100,\"x\":0,\"y\":0},\"h\":100,\"w\":100,\"x\":0,\"y\":0},{\"shape\":[{\"fill\":\"gray\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":36,\"y\":37},{\"event\":{\"action\":{\"asset\":\"startrec\",\"command\":\"startRecord\",\"failure\":\"rec_start_fail\",\"success\":\"rec_started\",\"type\":\"command\"},\"type\":\"click\"},\"fill\":\"#34e941\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":35,\"y\":35}],\"image\":{\"asset\":\"record\",\"h\":10,\"w\":6,\"x\":38,\"y\":40},\"text\":{\"color\":\"darkgreen\",\"font\":\"sans-serif\",\"fontsize\":50,\"h\":5,\"w\":20,\"x\":47,\"y\":43,\"__text\":\"Start\"},\"h\":100,\"id\":\"startrec\",\"w\":100,\"x\":0,\"y\":0},{\"shape\":[{\"fill\":\"gray\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":36,\"y\":37},{\"event\":{\"action\":{\"asset\":\"stoprec\",\"command\":\"stopRecord\",\"failure\":\"rec_stop_failed\",\"success\":\"rec_stopped\",\"type\":\"command\"},\"type\":\"click\"},\"fill\":\"#e93441\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":35,\"y\":35},{\"fill\":\"#e98888\",\"h\":2,\"id\":\"mover\",\"type\":\"circle\",\"visible\":false,\"w\":2,\"x\":47,\"y\":45}],\"image\":{\"asset\":\"stop_record\",\"h\":10,\"w\":6,\"x\":38,\"y\":40},\"h\":100,\"id\":\"stoprec\",\"visible\":false,\"w\":100,\"x\":0,\"y\":0},{\"shape\":[{\"fill\":\"gray\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":36,\"y\":37},{\"event\":{\"action\":[{\"asset\":\"current_rec\",\"command\":\"TOGGLEPLAY\",\"type\":\"command\"},{\"asset\":\"recorder\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"playback\",\"command\":\"hide\",\"type\":\"command\"},{\"asset\":\"startrec\",\"command\":\"show\",\"type\":\"command\"}],\"type\":\"click\"},\"fill\":\"#34e941\",\"h\":20,\"type\":\"roundrect\",\"w\":30,\"x\":35,\"y\":35}],\"image\":{\"asset\":\"icon_sound\",\"h\":10,\"id\":\"story_audio_button\",\"w\":6,\"x\":38,\"y\":40},\"text\":{\"color\":\"darkgreen\",\"font\":\"sans-serif\",\"fontsize\":50,\"h\":5,\"w\":20,\"x\":47,\"y\":43,\"__text\":\"Play\"},\"h\":100,\"id\":\"playback\",\"visible\":false,\"w\":100,\"x\":0,\"y\":0}],\"h\":100,\"id\":\"recorder\",\"z-index\":200,\"visible\":false,\"w\":100,\"x\":0,\"y\":0}],\"appEvents\":{\"list\":\"rec_started,rec_start_fail,rec_stopped,rec_stop_failed\"}},{\"id\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"next\",\"value\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\"},{\"name\":\"previous\",\"value\":\"scene5de719ed-207c-4f01-a7e8-e1eb4abf77ed\"}],\"events\":{\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_11825\",\"loop\":1},\"type\":\"enter\"},{\"action\":{\"type\":\"command\",\"command\":\"stop\",\"asset\":\"domain_11825\",\"loop\":1},\"type\":\"exit\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"retry_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"retryDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut \",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"retryShowAnim\"},\"type\":\"animation\",\"asset\":\"retryDialog\"}],\"type\":\"wrong_response\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"goodjob_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"goodjobDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut\",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"gjShowAnim\"},\"type\":\"animation\",\"asset\":\"goodjobDialog\"}],\"type\":\"correct_response\"}]},\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500},\"type\":\"click\"},\"asset\":\"next\",\"x\":93,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"next\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"eval\",\"asset\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\",\"success\":\"correct_response\",\"failure\":\"wrong_response\"},{\"type\":\"command\",\"command\":\"HIDEHTMLELEMENTS\",\"asset\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\"}],\"type\":\"click\"},\"asset\":\"validate\",\"x\":91,\"y\":15,\"h\":15}],\"text\":[],\"shape\":[],\"hotspot\":[],\"embed\":[],\"div\":[],\"audio\":[{\"asset\":\"domain_11825\"}],\"scribble\":[],\"g\":[{\"embed\":{\"template\":\"item\",\"var-item\":\"item\"},\"x\":10,\"y\":0,\"w\":80,\"h\":90},{\"embed\":{\"template-name\":\"retry\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"retryDialog\",\"visible\":false},{\"embed\":{\"template-name\":\"goodjob\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"goodjobDialog\",\"visible\":false}],\"iterate\":\"assessment_185d692a-1841-4a35-96c1-67cebcde42a2\",\"preload\":true,\"var\":\"item\",\"appEvents\":{\"list\":\"next_item,correct_response,wrong_response\"}},{\"id\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"next\",\"value\":\"endScreen\"},{\"name\":\"previous\",\"value\":\"scene185d692a-1841-4a35-96c1-67cebcde42a2\"}],\"events\":{\"event\":[{\"action\":{\"type\":\"command\",\"command\":\"play\",\"asset\":\"domain_8313\",\"loop\":1},\"type\":\"enter\"},{\"action\":{\"type\":\"command\",\"command\":\"stop\",\"asset\":\"domain_8313\",\"loop\":1},\"type\":\"exit\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"retry_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"retryDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut \",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"retryShowAnim\"},\"type\":\"animation\",\"asset\":\"retryDialog\"}],\"type\":\"wrong_response\"},{\"action\":[{\"type\":\"command\",\"command\":\"play\",\"asset\":\"goodjob_audio\"},{\"type\":\"command\",\"command\":\"show\",\"asset\":\"goodjobDialog\"},{\"tween\":{\"to\":{\"ease\":\"bounceOut\",\"duration\":1500,\"__cdata\":{\"x\":20,\"y\":20,\"w\":40,\"h\":30}},\"id\":\"gjShowAnim\"},\"type\":\"animation\",\"asset\":\"goodjobDialog\"}],\"type\":\"correct_response\"}]},\"image\":[{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"next\",\"effect\":\"fadein\",\"direction\":\"left\",\"ease\":\"linear\",\"duration\":500},\"type\":\"click\"},\"asset\":\"next\",\"x\":93,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"next\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100},{\"event\":{\"action\":[{\"type\":\"command\",\"command\":\"eval\",\"asset\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\",\"success\":\"correct_response\",\"failure\":\"wrong_response\"},{\"type\":\"command\",\"command\":\"HIDEHTMLELEMENTS\",\"asset\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\"}],\"type\":\"click\"},\"asset\":\"validate\",\"x\":91,\"y\":15,\"h\":15}],\"text\":[],\"shape\":[],\"hotspot\":[],\"embed\":[],\"div\":[],\"audio\":[{\"asset\":\"domain_8313\"}],\"scribble\":[],\"g\":[{\"embed\":{\"template\":\"item\",\"var-item\":\"item\"},\"x\":10,\"y\":0,\"w\":80,\"h\":90},{\"embed\":{\"template-name\":\"retry\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"retryDialog\",\"visible\":false},{\"embed\":{\"template-name\":\"goodjob\"},\"x\":30,\"y\":-40,\"w\":40,\"h\":48,\"id\":\"goodjobDialog\",\"visible\":false}],\"iterate\":\"assessment_a3cc8a52-833a-4a9d-b6e6-3be6b266b70f\",\"preload\":true,\"var\":\"item\",\"appEvents\":{\"list\":\"next_item,correct_response,wrong_response\"}},{\"id\":\"endScreen\",\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"param\":[{\"name\":\"previousScreen\",\"value\":\"recordAudio9\"},{\"name\":\"nextScreen\",\"value\":\"splash\"},{\"name\":\"previous\",\"value\":\"scenea3cc8a52-833a-4a9d-b6e6-3be6b266b70f\"}],\"events\":{\"event\":[]},\"image\":[{\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"visible\":true,\"editable\":true,\"asset\":\"endScreen\",\"z-index\":0},{\"x\":40,\"y\":50,\"w\":8,\"h\":13,\"visible\":false,\"editable\":true,\"asset\":\"previous\",\"event\":{\"action\":{\"asset\":\"theme\",\"command\":\"transitionTo\",\"direction\":\"right\",\"duration\":600,\"ease\":\"linear\",\"effect\":\"scroll\",\"param\":\"previousScreen\",\"type\":\"command\"},\"type\":\"click\"},\"z-index\":1},{\"x\":50,\"y\":50,\"w\":8,\"h\":13,\"visible\":false,\"editable\":true,\"asset\":\"next\",\"event\":{\"action\":{\"asset\":\"theme\",\"command\":\"transitionTo\",\"direction\":\"right\",\"duration\":600,\"ease\":\"linear\",\"effect\":\"scroll\",\"param\":\"nextScreen\",\"type\":\"command\"},\"type\":\"click\"},\"z-index\":2},{\"event\":{\"action\":{\"type\":\"command\",\"command\":\"transitionTo\",\"asset\":\"theme\",\"param\":\"previous\",\"effect\":\"fadein\",\"direction\":\"right\",\"ease\":\"linear\",\"duration\":100},\"type\":\"click\"},\"asset\":\"previous\",\"x\":2,\"y\":3,\"w\":5,\"h\":8.3,\"id\":\"previous\",\"visible\":true,\"editable\":true,\"z-index\":100}],\"text\":[],\"shape\":[],\"hotspot\":[],\"embed\":[],\"div\":[],\"audio\":[],\"scribble\":[],\"g\":[]}]}}")) - ) - val result: Map[String, AnyRef] = ExtractableMimeTypeHelper.processECMLBody(obj, jobConfig)(ec, cloudStorageUtil) - result.contains("artifactUrl") shouldBe true - result.get("artifactUrl") shouldNot be(null) - result.contains("cloudStorageKey") shouldBe true - result.get("cloudStorageKey") shouldNot be(null) - } - -} diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveCollectionPublisherSpec.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveCollectionPublisherSpec.scala deleted file mode 100644 index 1f419ee4d..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveCollectionPublisherSpec.scala +++ /dev/null @@ -1,226 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.commons.lang3.StringUtils -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.{doNothing, when} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.livenodepublisher.publish.helpers.LiveCollectionPublisher -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util._ - -import java.text.SimpleDateFormat -import java.util -import java.util.Date -import scala.concurrent.ExecutionContextExecutor - -class LiveCollectionPublisherSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: LiveNodePublisherConfig = new LiveNodePublisherConfig(config) - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val ec: ExecutionContextExecutor = ExecutionContexts.global - implicit val defCache: DefinitionCache = new DefinitionCache() - implicit val defConfig: DefinitionConfig = DefinitionConfig(jobConfig.schemaSupportVersionMap, jobConfig.definitionBasePath) - implicit val publishConfig: PublishConfig = jobConfig.asInstanceOf[PublishConfig] - implicit val httpUtil: HttpUtil = new HttpUtil - val mockElasticUtil: ElasticSearchUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - var definitionCache = new DefinitionCache() - implicit val definition: ObjectDefinition = definitionCache.getDefinition("Collection", jobConfig.schemaSupportVersionMap.getOrElse("collection", "1.0").asInstanceOf[String], jobConfig.definitionBasePath) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.hierarchyKeyspaceName, jobConfig.hierarchyTableName, definition.getExternalPrimaryKey, definition.getExternalProps) - - def getTimeStamp: String = { - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - sdf.format(new Date()) - } - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => - } - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } - - "enrichObjectMetadata" should "enrich the Content pkgVersion metadata" in { - val data = new ObjectData("do_2133950809948078081503", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_2133950809948078081503", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.content-collection", "keywords" -> Array[String]("test keyword"))) - val result: ObjectData = new TestCollectionPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - // "validateMetadata with invalid external data" should "return exception messages" in { - // val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef]), Some(Map[String, AnyRef]("artifactUrl" -> "artifactUrl"))) - // val result: List[String] = new TestCollectionPublisher().validateMetadata(data, data.identifier) - // result.size should be(1) - // } - - "saveExternalData " should "save external data to cassandra table" in { - val data = new ObjectData("do_1234", Map[String, AnyRef](), Some(Map[String, AnyRef]("body" -> "body", "answer" -> "answer"))) - new TestCollectionPublisher().saveExternalData(data, readerConfig) - } - - "getHierarchy " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestCollectionPublisher().getHierarchy(identifier, readerConfig) - } - - "getExtDatas " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestCollectionPublisher().getExtDatas(List(identifier), readerConfig) - } - - "getHierarchies " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestCollectionPublisher().getHierarchies(List(identifier), readerConfig) - } - - "getDataForEcar" should "return one element in list" in { - val data = new ObjectData("do_123", Map("objectType" -> "Content"), Some(Map("responseDeclaration" -> "test")), Some(Map())) - val result: Option[List[Map[String, AnyRef]]] = new TestCollectionPublisher().getDataForEcar(data) - result.size should be(1) - } - - "getObjectWithEcar" should "return object with ecar url" in { - val unpublishedChildrenObj: List[Map[String, AnyRef]] = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](unpublishedChildrenData) - val data = new ObjectData("do_123", Map("objectType" -> "Collection", "identifier" -> "do_123", "name" -> "Test Collection", "lastPublishedOn" -> getTimeStamp, "lastUpdatedOn" -> getTimeStamp, "status" -> "Draft", "downloadUrl" -> "downloadUrl", "variants" -> Map.empty[String,AnyRef]), Some(Map()), Some(Map("children" -> unpublishedChildrenObj))) - val result = new TestCollectionPublisher().getObjectWithEcar(data, List(EcarPackageType.SPINE, EcarPackageType.ONLINE))(ec, mockNeo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, jobConfig, defCache, defConfig, httpUtil) - StringUtils.isNotBlank(result.metadata.getOrElse("downloadUrl", "").asInstanceOf[String]) - } - - "syncNodes" should "sync Child Nodes into ElasticSearch" in { - Mockito.reset(mockElasticUtil) - doNothing().when(mockElasticUtil).addDocument(anyString(), anyString()) - doNothing().when(mockElasticUtil).bulkIndexWithIndexId(anyString(), anyString(), any()) - - val do_113405023736512512114Json = """{"identifier":"do_113405023736512512114","graph_id":"domain","node_id":0,"collections":["do_11340502373639782416", "do_113405023736479744112"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_113405023736512512114")).thenReturn(do_113405023736512512114Json) - - val do_11340502373639782416Json = """{"identifier":"do_11340502373639782416","graph_id":"domain","node_id":0,"collections":["do_11340502373642240018", "do_11340502373608652812"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373639782416")).thenReturn(do_11340502373639782416Json) - - val do_11340502373642240018Json = """{"identifier":"do_11340502373642240018","graph_id":"domain","node_id":0,"collections":["do_11336831941257625611"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373642240018")).thenReturn(do_11340502373642240018Json) - - val do_11340502373608652812Json = """{"identifier":"do_11340502373608652812","graph_id":"domain","node_id":0,"collections":["do_11340096165525094411"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373608652812")).thenReturn(do_11340502373608652812Json) - - val do_113405023736479744112Json = """{"identifier":"do_113405023736479744112","graph_id":"domain","node_id":0,"collections":["do_11340502373638144014"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_113405023736479744112")).thenReturn(do_113405023736479744112Json) - - val do_11340502373638144014Json = """{"identifier":"do_11340502373638144014","graph_id":"domain","node_id":0,"collections":["do_113405023736455168110"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_11340502373638144014")).thenReturn(do_11340502373638144014Json) - - val do_113405023736455168110Json = """{"identifier":"do_113405023736455168110","graph_id":"domain","node_id":0,"collections":["do_11340096293585715212"],"objectType":"Collection","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString("do_113405023736455168110")).thenReturn(do_113405023736455168110Json) - - val publishedCollectionNodeMetadataObj: Map[String,AnyRef] = ScalaJsonUtil.deserialize[Map[String,AnyRef]](publishedCollectionNodeMetadata) - val data = new ObjectData("do_123", publishedCollectionNodeMetadataObj, Some(Map.empty[String, AnyRef])) - val syncChildrenData = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](publishedChildrenData) - val messages: Map[String, Map[String, AnyRef]] = new TestCollectionPublisher().syncNodes(data, syncChildrenData, List.empty)(mockElasticUtil, mockNeo4JUtil, cassandraUtil, readerConfig, definition, jobConfig) - - assert(messages != null && messages.size > 0) - } - - "updateHierarchyMetadata" should "update child nodes with published object metadata" in { - val unpublishedChildrenObj: List[Map[String, AnyRef]] = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](unpublishedChildrenData) - val publishedCollectionNodeMetadataObj: Map[String,AnyRef] = ScalaJsonUtil.deserialize[Map[String,AnyRef]](publishedCollectionNodeMetadata) - - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_123", readerConfig)(cassandraUtil).get - // Collection - update and publish children - line 418 in PublishFinalizer - val updatedChildren: List[Map[String, AnyRef]] = new TestCollectionPublisher().updateHierarchyMetadata(unpublishedChildrenObj, publishedCollectionNodeMetadataObj, collRelationalMetadata)(jobConfig) - - assert(updatedChildren.nonEmpty) - - updatedChildren.map(child => { - assert(child.contains("pkgVersion")) - assert(child("pkgVersion").toString.equalsIgnoreCase("1")) - }) - } - - "getRelationalMetadata" should "return empty Map when there is no entry in relational_metadata column" in { - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_1234", readerConfig)(cassandraUtil).get - assert(collRelationalMetadata != null && collRelationalMetadata.isEmpty) - } - - "getRelationalMetadata" should "return empty Map when there is empty entry in relational_metadata column" in { - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_12345", readerConfig)(cassandraUtil).get - assert(collRelationalMetadata != null && collRelationalMetadata.isEmpty) - } - - "getRelationalMetadata" should "return empty Map when there is empty object entry in relational_metadata column" in { - val collRelationalMetadata = new TestCollectionPublisher().getRelationalMetadata("do_123456", readerConfig)(cassandraUtil).get - assert(collRelationalMetadata != null && collRelationalMetadata.isEmpty) - } - - "publishHierarchy " should "save hierarchy data to cassandra table" in { - val publishChildrenData = ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](publishedChildrenData) - val publishedCollectionNodeMetadataObj: Map[String,AnyRef] = ScalaJsonUtil.deserialize[Map[String,AnyRef]](publishedCollectionNodeMetadata) - val data = new ObjectData("do_123", publishedCollectionNodeMetadataObj, Some(Map.empty[String, AnyRef])) - val result = new TestCollectionPublisher().publishHierarchy(publishChildrenData, data, readerConfig, jobConfig) - assert(result) - } - - "getUnitsFromLiveContent" should "return object hierarchy" in { - val data = new ObjectData("do_2133950809948078081503", Map("identifier" -> "do_2133950809948078081503"), Some(Map.empty[String, AnyRef])) - val fetchedChildren = new TestCollectionPublisher().getUnitsFromLiveContent(data)(cassandraUtil,readerConfig,jobConfig) - assert(fetchedChildren.nonEmpty) - } - - "updateOriginPkgVersion" should "return origin node Data" in { - val metaData = new java.util.HashMap[String, AnyRef]() { - { - put("IL_UNIQUE_ID", "do_11300581751853056018") - put("identifier", "do_11300581751853056018") - put("name", "Origin Content") - put("createdBy", "874ed8a5-782e-4f6c-8f36-e0288455901e") - put("channel", "b00bc992ef25f1a9a8d63291e20efc8d") - put("trackable", "{\"enabled\":\"Yes\",\"autoBatch\":\"Yes\"}") - put("createdFor", util.Arrays.asList("ORG_001")) - put("pkgVersion", 3.asInstanceOf[AnyRef]) - } - } - val collectionObj = new ObjectData("do_123", Map("objectType" -> "Collection", "identifier" -> "do_123", "name" -> "Test Collection","origin" -> "do_456", "originData" -> Map("name" -> "Contemporary India I", "copyType" -> "deep", "license" -> "CC BY 4.0", "organisation" -> Array("NCERT"))), Some(Map()), Some(Map())) - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(metaData) - val originObj = new TestCollectionPublisher().updateOriginPkgVersion(collectionObj) - assert(originObj.metadata("originData").asInstanceOf[Map[String,AnyRef]]("pkgVersion") == 3) - } - - val collRelationalMetadataStr = "{\"do_123\":{\"name\":\"Collection Publish T21\",\"children\":[\"do_11340511137112064018\",\"do_11340511137080934412\"],\"root\":true},\"do_11340511137112064018\":{\"name\":\"Collection Parent\",\"children\":[\"do_11340096165525094411\"],\"root\":false,\"relationalMetadata\":{\"do_11340096165525094411\":{\"name\":\"Test Name RM L1 - R1\",\"keywords\":[\"Overwriting content KW1\"]}}},\"do_11340511137080934412\":{\"name\":\"Collection Parent\",\"children\":[\"do_11340096165525094411\"],\"root\":false,\"relationalMetadata\":{\"do_11340096165525094411\":{\"name\":\"Test Name RM L1 - R1\",\"keywords\":[\"Overwriting content KW1\"]}}},\"do_11340096165525094411\":{\"name\":\"PDF Content\",\"children\":[],\"root\":false},\"do_113405111371145216110\":{\"name\":\"test\",\"children\":[], \"root\":false}}" - val publishedChildrenData = "[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.586+0530\",\"parent\":\"do_11340502356035174411\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.572+0530\",\"parent\":\"do_113405023736512512114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.575+0530\",\"parent\":\"do_11340502373639782416\",\"children\":[{\"copyright\":\"J H S BHARKHOKHA, Tamil Nadu\",\"lastStatusChangedOn\":\"2021-09-17T16:22:50.404+0530\",\"parent\":\"do_11340502373642240018\",\"licenseterms\":\"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\"organisation\":[\"J H S BHARKHOKHA\",\"Tamil Nadu\"],\"mediaType\":\"content\",\"name\":\"jaga Aug 25th more than 200mp mp4 update 1\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-09-17T16:05:29.019+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-09-17T16:22:50.404+0530\",\"size\":363062652,\"identifier\":\"do_11336831941257625611\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"eTextbook\",\"appIcon\":\"https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_2134462034258575361402/artifact/rhinocerous.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"\",\"framework\":\"tn_k-12_5\",\"creator\":\"सामग्री निर्माता TN\",\"versionKey\":\"1631875539805\",\"mimeType\":\"video/mp4\",\"code\":\"3bd7411e-c03c-4997-a247-4d43a5cc820b\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Live\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-09-17T16:22:15.047+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"fca2925f-1eee-4654-9177-fece3fd6afc9\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"consumerId\":\"2eaff3db-cdd1-42e5-a611-bebbf906e6cf\",\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"tn_k-12_5\"],\"contentDisposition\":\"online-only\",\"previewUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"artifactUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11336831941257625611/jaga-aug-25th-more-than-200mp-mp4-update-1_1631875935857_do_11336831941257625611_3_SPINE.ecar\",\"size\":\"2172\"}},\"index\":1,\"pkgVersion\":3,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.1 Key parts in the head\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.575+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373642240018\",\"description\":\"xyz\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436575\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"7991af5c2e51e4d3d7b83167aaac8829\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11336831941257625611\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.534+0530\",\"parent\":\"do_11340502373639782416\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:13:39.729+0530\",\"parent\":\"do_11340502373608652812\",\"mediaType\":\"content\",\"name\":\"Collection Publishing PDF Content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:56:17.917+0530\",\"createdFor\":[\"01309282781705830427\"],\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:13:39.729+0530\",\"streamingUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"identifier\":\"do_11340096165525094411\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":4,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"creator\":\"N131\",\"versionKey\":\"1635859577917\",\"mimeType\":\"application/pdf\",\"code\":\"c9ce1ce0-b9b4-402e-a9c3-556701070838\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Processing\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:13:35.589+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"pragma\":[\"external\"],\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPPDFContent1\",\"CPPDFContent2\",\"CollectionKW1\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"size\":\"256918\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar\",\"size\":\"6378\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.2 Other parts\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.534+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373608652812\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436534\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"3bf70f06d3e8dba010d8806fd94259b1\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":2,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1 Parts of Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.572+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373639782416\",\"description\":\"This section describes about various part of the body such as head, hands, legs etc.\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436572\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"20cc1f31e62f924c6e47bf04c994376b\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11336831941257625611\",\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":2,\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.582+0530\",\"parent\":\"do_113405023736512512114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.570+0530\",\"parent\":\"do_113405023736479744112\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T12:40:36.579+0530\",\"parent\":\"do_11340502373638144014\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:16:10.667+0530\",\"parent\":\"do_113405023736455168110\",\"mediaType\":\"content\",\"name\":\"Collection Publish MP4 content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:58:53.445+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:16:10.667+0530\",\"identifier\":\"do_11340096293585715212\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1134009488766730241130/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"versionKey\":\"1635859733445\",\"mimeType\":\"video/mp4\",\"code\":\"e0b58864-3dc5-484a-b194-38c3eddcbce1\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Draft\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:16:08.789+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPMP4ContentKW1\",\"CPMP4ContentKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":5,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"size\":\"2692101\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860770277_do_11340096293585715212_1_SPINE.ecar\",\"size\":\"6275\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"dsffgdg\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.579+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_113405023736455168110\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436579\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"9cf84ff2fb08f9af4c23eb09df9b2520\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":4,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2.1 Respiratory System\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.570+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_11340502373638144014\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"attributions\":[],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436570\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"b186b1bbcc9c58db865f75e34345179e\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"keywords\":[\"UnitKW1\",\"UnitKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2 Organ Systems\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.582+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_113405023736479744112\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436582\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"40a1ed37e0fad94eca76b2a96fe086ab\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":1,\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":2,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5. Human Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T12:40:36.586+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T12:46:52.715+0530\",\"identifier\":\"do_113405023736512512114\",\"description\":\"This chapter describes about human body\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\",\"framework\":\"ncert_k-12\",\"versionKey\":\"1636355436586\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"76abafa2a0c2cfef90b52db1ef41fb82\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\",\"do_11336831941257625611\",\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T12:53:09.398+0530\",\"objectType\":\"Collection\",\"status\":\"Live\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":3,\"depth\":1,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202227_do_11340502356035174411_1_SPINE.ecar\\\",\\\"size\\\":\\\"12044\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340502356035174411/collection-publish-t20_1636356202412_do_11340502356035174411_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5074\\\"}}\",\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}]" - val unpublishedChildrenData = "[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.180+0530\",\"parent\":\"do_11340511118032076811\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.166+0530\",\"parent\":\"do_113405111371202560114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.170+0530\",\"parent\":\"do_11340511137108787216\",\"children\":[{\"copyright\":\"J H S BHARKHOKHA, Tamil Nadu\",\"lastStatusChangedOn\":\"2021-09-17T16:22:50.404+0530\",\"parent\":\"do_11340511137112064018\",\"licenseterms\":\"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\"organisation\":[\"J H S BHARKHOKHA\",\"Tamil Nadu\"],\"mediaType\":\"content\",\"name\":\"jaga Aug 25th more than 200mp mp4 update 1\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-09-17T16:05:29.019+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-09-17T16:22:50.404+0530\",\"size\":363062652,\"identifier\":\"do_11336831941257625611\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"eTextbook\",\"appIcon\":\"https://stagingdock.blob.core.windows.net/sunbird-content-dock/content/do_2134462034258575361402/artifact/rhinocerous.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"\",\"framework\":\"tn_k-12_5\",\"creator\":\"सामग्री निर्माता TN\",\"versionKey\":\"1631875539805\",\"mimeType\":\"video/mp4\",\"code\":\"3bd7411e-c03c-4997-a247-4d43a5cc820b\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Live\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-09-17T16:22:15.047+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"fca2925f-1eee-4654-9177-fece3fd6afc9\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"consumerId\":\"2eaff3db-cdd1-42e5-a611-bebbf906e6cf\",\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"tn_k-12_5\"],\"contentDisposition\":\"online-only\",\"previewUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"artifactUrl\":\"https://preprodall.blob.core.windows.net/ntp-content-preprod/content/assets/do_2133520666218741761984/como-kids-tv-_-the-story-of-comos-family-_-30min-_-cartoon-video-for-kids.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11336831941257625611/jaga-aug-25th-more-than-200mp-mp4-update-1_1631875935857_do_11336831941257625611_3_SPINE.ecar\",\"size\":\"2172\"}},\"index\":1,\"pkgVersion\":3,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.1 Key parts in the head\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.170+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.170+0530\",\"identifier\":\"do_11340511137112064018\",\"description\":\"xyz\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134170\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"7991af5c2e51e4d3d7b83167aaac8829\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.133+0530\",\"parent\":\"do_11340511137108787216\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:13:39.729+0530\",\"parent\":\"do_11340511137080934412\",\"mediaType\":\"content\",\"name\":\"Collection Publishing PDF Content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:56:17.917+0530\",\"createdFor\":[\"01309282781705830427\"],\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:13:39.729+0530\",\"streamingUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"identifier\":\"do_11340096165525094411\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":4,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340094790233292811/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"creator\":\"N131\",\"versionKey\":\"1635859577917\",\"mimeType\":\"application/pdf\",\"code\":\"c9ce1ce0-b9b4-402e-a9c3-556701070838\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Processing\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:13:35.589+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"pragma\":[\"external\"],\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPPDFContent1\",\"CPPDFContent2\",\"CollectionKW1\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":4,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009478823116801129/chapter_1.pdf\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860615969_do_11340096165525094411_1.ecar\",\"size\":\"256918\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096165525094411/collection-publishing-pdf-content_1635860619148_do_11340096165525094411_1_SPINE.ecar\",\"size\":\"6378\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1.2 Other parts\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.133+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.132+0530\",\"identifier\":\"do_11340511137080934412\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134133\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"3bf70f06d3e8dba010d8806fd94259b1\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":2,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.1 Parts of Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.166+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.165+0530\",\"identifier\":\"do_11340511137108787216\",\"description\":\"This section describes about various part of the body such as head, hands, legs etc.\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134166\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"20cc1f31e62f924c6e47bf04c994376b\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"},{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.177+0530\",\"parent\":\"do_113405111371202560114\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.162+0530\",\"parent\":\"do_113405111371169792112\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-08T15:38:54.173+0530\",\"parent\":\"do_11340511137105510414\",\"children\":[{\"lastStatusChangedOn\":\"2021-11-02T19:16:10.667+0530\",\"parent\":\"do_113405111371145216110\",\"mediaType\":\"content\",\"name\":\"Collection Publish MP4 content\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-02T18:58:53.445+0530\",\"channel\":\"0126825293972439041\",\"lastUpdatedOn\":\"2021-11-02T19:16:10.667+0530\",\"identifier\":\"do_11340096293585715212\",\"resourceType\":\"Learn\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Explanation Content\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1134009488766730241130/artifact/033019_sz_reviews_feat_1564126718632.thumb.jpg\",\"languageCode\":[\"en\"],\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"framework\":\"ekstep_ncert_k-12\",\"versionKey\":\"1635859733445\",\"mimeType\":\"video/mp4\",\"code\":\"e0b58864-3dc5-484a-b194-38c3eddcbce1\",\"license\":\"CC BY 4.0\",\"version\":2,\"prevStatus\":\"Draft\",\"contentType\":\"Resource\",\"prevState\":\"Draft\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-02T19:16:08.789+0530\",\"objectType\":\"Content\",\"status\":\"Live\",\"createdBy\":\"0b71985d-fcb0-4018-ab14-83f10c3b0426\",\"dialcodeRequired\":\"No\",\"interceptionPoints\":{},\"keywords\":[\"CPMP4ContentKW1\",\"CPMP4ContentKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"identity\",\"depth\":5,\"lastPublishedBy\":\"\",\"osId\":\"org.ekstep.quiz.app\",\"se_FWIds\":[\"ekstep_ncert_k-12\"],\"contentDisposition\":\"inline\",\"previewUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"artifactUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1134009488766730241130/amoeba-eat.mp4\",\"visibility\":\"Default\",\"credentials\":{\"enabled\":\"No\"},\"variants\":{\"full\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860769119_do_11340096293585715212_1.ecar\",\"size\":\"2692101\"},\"spine\":{\"ecarUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340096293585715212/collection-publish-mp4-content_1635860770277_do_11340096293585715212_1_SPINE.ecar\",\"size\":\"6275\"}},\"index\":1,\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"dsffgdg\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.173+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.173+0530\",\"identifier\":\"do_113405111371145216110\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134173\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"9cf84ff2fb08f9af4c23eb09df9b2520\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":4,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2.1 Respiratory System\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.162+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:53:32.894+0530\",\"identifier\":\"do_11340511137105510414\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"attributions\":[],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134162\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"b186b1bbcc9c58db865f75e34345179e\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"keywords\":[\"UnitKW1\",\"UnitKW2\"],\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":3,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5.2 Organ Systems\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.176+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.176+0530\",\"identifier\":\"do_113405111371169792112\",\"description\":\"\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134176\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"40a1ed37e0fad94eca76b2a96fe086ab\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":2,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":2,\"idealScreenDensity\":\"hdpi\"}],\"mediaType\":\"content\",\"name\":\"5. Human Body\",\"discussionForum\":{\"enabled\":\"No\"},\"createdOn\":\"2021-11-08T15:38:54.180+0530\",\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:38:54.179+0530\",\"identifier\":\"do_113405111371202560114\",\"description\":\"This chapter describes about human body\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"os\":[\"All\"],\"primaryCategory\":\"Textbook Unit\",\"languageCode\":[\"en\"],\"framework\":\"ncert_k-12\",\"versionKey\":\"1636366134180\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"76abafa2a0c2cfef90b52db1ef41fb82\",\"license\":\"CC BY 4.0\",\"version\":2,\"contentType\":\"TextBookUnit\",\"language\":[\"English\"],\"objectType\":\"Collection\",\"status\":\"Draft\",\"dialcodeRequired\":\"No\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"depth\":1,\"osId\":\"org.ekstep.launcher\",\"contentDisposition\":\"inline\",\"visibility\":\"Parent\",\"credentials\":{\"enabled\":\"No\"},\"index\":1,\"idealScreenDensity\":\"hdpi\"}]" - val publishedCollectionNodeMetadata = "{\"copyright\":\"tn\",\"lastStatusChangedOn\":\"2021-11-08T15:38:31.391+0530\",\"publish_type\":\"public\",\"author\":\"ContentcreatorTN\",\"children\":[{\"name\":\"5. Human Body\",\"identifier\":\"do_113405111371202560114\",\"description\":\"This chapter describes about human body\",\"objectType\":\"Collection\",\"index\":1}],\"body\":null,\"mediaType\":\"content\",\"name\":\"Collection Publish T20\",\"toc_url\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11340511118032076811/artifact/do_11340511118032076811_toc.json\",\"discussionForum\":\"{\\\"enabled\\\":\\\"No\\\"}\",\"createdOn\":\"2021-11-08T15:38:31.391+0530\",\"createdFor\":[\"0125196274181898243\"],\"channel\":\"0126825293972439041\",\"generateDIALCodes\":\"No\",\"lastUpdatedOn\":\"2021-11-08T15:53:33.587+0530\",\"size\":12048,\"publishError\":null,\"identifier\":\"do_11340511118032076811\",\"description\":\"Collection Publish\",\"resourceType\":\"Book\",\"ownershipType\":[\"createdBy\"],\"compatibilityLevel\":1,\"audience\":[\"Student\"],\"trackable\":\"{\\\"enabled\\\":\\\"No\\\",\\\"autoBatch\\\":\\\"No\\\"}\",\"os\":[\"All\"],\"primaryCategory\":\"Digital Textbook\",\"appIcon\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/artifact/16_bvnvrokht6hn97eqcklwk2fs6ppx2z.thumb.png\",\"downloadUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/collection-publish-t20_1636367178646_do_11340511118032076811_1_SPINE.ecar\",\"attributions\":[],\"framework\":\"ncert_k-12\",\"posterImage\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/assets/do_31321903538537267212904/16_bvnvrokht6hn97eqcklwk2fs6ppx2z.png\",\"creator\":\"NCERT\",\"totalCompressedSize\":363062652,\"versionKey\":\"1636367013587\",\"mimeType\":\"application/vnd.ekstep.content-collection\",\"code\":\"0125196274181898243\",\"license\":\"CC BY 4.0\",\"leafNodes\":[\"do_11340096293585715212\",\"do_11336831941257625611\",\"do_11340096165525094411\"],\"version\":2,\"contentType\":\"TextBook\",\"language\":[\"English\"],\"lastPublishedOn\":\"2021-11-08T15:56:15.604+0530\",\"contentTypesCount\":\"{\\\"TextBookUnit\\\":7,\\\"Resource\\\":3}\",\"objectType\":\"Collection\",\"status\":\"Live\",\"createdBy\":\"220d4745-6764-498d-ad37-5e49b8cce716\",\"dialcodeRequired\":\"No\",\"keywords\":[\"CPPDFContent1\",\"UnitKW2\",\"CollectionKW1\",\"CPPDFContent2\",\"UnitKW1\",\"CPMP4ContentKW1\",\"CPMP4ContentKW2\"],\"userConsent\":\"Yes\",\"idealScreenSize\":\"normal\",\"contentEncoding\":\"gzip\",\"leafNodesCount\":3,\"depth\":0,\"flagReasons\":null,\"mimeTypesCount\":\"{\\\"application/pdf\\\":1,\\\"video/mp4\\\":2,\\\"application/vnd.ekstep.content-collection\\\":7}\",\"osId\":\"org.ekstep.quiz.app\",\"copyrightYear\":2021,\"se_FWIds\":[\"ncert_k-12\"],\"contentDisposition\":\"inline\",\"additionalCategories\":[],\"childNodes\":[\"do_11336831941257625611\",\"do_11340511137112064018\",\"do_11340511137108787216\",\"do_113405111371202560114\",\"do_11340096165525094411\",\"do_11340511137080934412\",\"do_11340096293585715212\",\"do_113405111371145216110\",\"do_11340511137105510414\",\"do_113405111371169792112\"],\"visibility\":\"Default\",\"credentials\":\"{\\\"enabled\\\":\\\"No\\\"}\",\"variants\":\"{\\\"spine\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/collection-publish-t20_1636367178646_do_11340511118032076811_1_SPINE.ecar\\\",\\\"size\\\":\\\"12048\\\"},\\\"online\\\":{\\\"ecarUrl\\\":\\\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/collection/do_11340511118032076811/collection-publish-t20_1636367178792_do_11340511118032076811_1_ONLINE.ecar\\\",\\\"size\\\":\\\"5081\\\"}}\",\"pkgVersion\":1,\"idealScreenDensity\":\"hdpi\"}" -} - - -class TestCollectionPublisher extends LiveCollectionPublisher {} \ No newline at end of file diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveContentPublisherSpec.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveContentPublisherSpec.scala deleted file mode 100644 index 1b61077eb..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveContentPublisherSpec.scala +++ /dev/null @@ -1,243 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.commons.lang3.StringUtils -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.livenodepublisher.publish.helpers.LiveContentPublisher -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} - -import scala.concurrent.ExecutionContextExecutor - -class LiveContentPublisherSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: LiveNodePublisherConfig = new LiveNodePublisherConfig(config) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.contentKeyspaceName, jobConfig.contentTableName) - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val ec: ExecutionContextExecutor = ExecutionContexts.global - implicit val defCache: DefinitionCache = new DefinitionCache() - implicit val defConfig: DefinitionConfig = DefinitionConfig(jobConfig.schemaSupportVersionMap, jobConfig.definitionBasePath) - implicit val publishConfig: PublishConfig = jobConfig.asInstanceOf[PublishConfig] - implicit val httpUtil: HttpUtil = new HttpUtil - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => - } - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } - - "enrichObjectMetadata" should "enrich the Content pkgVersion metadata" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf")) - val result: ObjectData = new TestLiveContentPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - ignore should "enrich the Content metadata for application/vnd.ekstep.html-archive should through exception in artifactUrl is not available" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.html-archive")) - val result: ObjectData = new TestLiveContentPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - "enrichObjectMetadata" should "enrich the Content metadata for application/vnd.ekstep.html-archive" in { - val data = new ObjectData("do_1132167819505500161297", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_1132167819505500161297", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.html-archive", "artifactUrl" -> "artifactUrl.zip")) - val result: ObjectData = new TestLiveContentPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - - "validateMetadata with invalid external data" should "return exception messages" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef]), Some(Map[String, AnyRef]("artifactUrl" -> "artifactUrl"))) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " return exception messages if extData is set as None" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Either 'body' or 'artifactUrl' are required for processing of ECML content for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " return exception messages if is having body=\"\"" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), Some(Map[String, AnyRef]("body" -> ""))) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Either 'body' or 'artifactUrl' are required for processing of ECML content for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " return exception messages if is having body=null" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), Some(Map[String, AnyRef]("body" -> null))) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Either 'body' or 'artifactUrl' are required for processing of ECML content for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " not return exception messages if is having body=null but artifactUrl is available" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive", "artifactUrl" -> "sampleUrl"), Some(Map[String, AnyRef]("body" -> null))) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/vnd.ekstep.ecml-archive " should " not return exception messages if is having valid body" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/vnd.ekstep.ecml-archive"), Some(Map[String, AnyRef]("body" -> "{\"theme\":{\"id\":\"theme\",\"version\":\"1.0\",\"startStage\":\"8ab605cc-b26d-4d0d-a827-2112b0330c3a\",\"stage\":[{\"x\":0,\"y\":0,\"w\":100,\"h\":100,\"id\":\"8ab605cc-b26d-4d0d-a827-2112b0330c3a\",\"rotate\":null,\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true,\\\"color\\\":\\\"#FFFFFF\\\",\\\"genieControls\\\":false,\\\"instructions\\\":\\\"\\\"}\"},\"manifest\":{\"media\":[{\"assetId\":\"do_113303238321799168110\"},{\"assetId\":\"do_113303315360907264114\"}]},\"image\":[{\"asset\":\"do_113303238321799168110\",\"x\":20,\"y\":20,\"w\":49.51,\"h\":14.29,\"rotate\":0,\"z-index\":0,\"id\":\"a62c82d4-e497-424a-b9a5-45b585572e20\",\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true}\"}},{\"asset\":\"do_113303315360907264114\",\"x\":20,\"y\":20,\"w\":49.51,\"h\":14.29,\"rotate\":0,\"z-index\":1,\"id\":\"dcc3d9e7-e4e0-409c-b099-434523556c10\",\"config\":{\"__cdata\":\"{\\\"opacity\\\":100,\\\"strokeWidth\\\":1,\\\"stroke\\\":\\\"rgba(255, 255, 255, 0)\\\",\\\"autoplay\\\":false,\\\"visible\\\":true}\"}}]}],\"manifest\":{\"media\":[{\"id\":\"1b34ae41-4c56-4c4d-a0e4-be56169cb7e6\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/controller/navigation_ctrl.js\",\"type\":\"js\"},{\"id\":\"e65308d4-8420-4533-b74d-87c1909e9b12\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/templates/navigation.html\",\"type\":\"js\"},{\"id\":\"org.ekstep.navigation\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/renderer/plugin.js\",\"type\":\"plugin\"},{\"id\":\"org.ekstep.navigation_manifest\",\"plugin\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"src\":\"/content-plugins/org.ekstep.navigation-1.0/manifest.json\",\"type\":\"json\"},{\"name\":\"Screenshot 2021-06-17 at 5 17 12 PM\",\"id\":\"do_113303238321799168110\",\"src\":\"/assets/public/content/do_113303238321799168110/artifact/do_113303238321799168110_1623930459486_screenshot-2021-06-17-at-5.17.12-pm.png\",\"type\":\"image\"},{\"name\":\"Screenshot 2021-06-17 at 5 17 12 PM\",\"id\":\"do_113303315360907264114\",\"src\":\"/assets/public/content/do_113303315360907264114/artifact/do_113303315360907264114_1623939863955_screenshot-2021-06-17-at-5.17.12-pm.png\",\"type\":\"image\"}]},\"plugin-manifest\":{\"plugin\":[{\"id\":\"org.ekstep.navigation\",\"ver\":\"1.0\",\"type\":\"plugin\",\"depends\":\"\"}]},\"compatibilityVersion\":2}}"))) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " return exception messages if content is having invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://www.youtube.com/"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Invalid youtube Url = https://www.youtube.com/ for : do_123") shouldBe true - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " not return exception messages if content is having valid artifactUrl = https://www.youtube.com/embed/watch?" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://www.youtube.com/embed/watch?"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " not return exception messages if content is having valid artifactUrl = https://www.youtube.com/watch?v=6Js8tBCfbWk" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://www.youtube.com/watch?v=6Js8tBCfbWk"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType video/x-youtube or video/youtube " should " not return exception messages if content is having valid artifactUrl = https://youtu.be/6Js8tBCfbWk" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "video/x-youtube", "artifactUrl" -> "https://youtu.be/6Js8tBCfbWk"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/pdf " should " throw InvalidInputException invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf", "artifactUrl" -> "https://www.youtube.com/"), None) - assertThrows[InvalidInputException] { - new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - } - } - - "validateMetadata with mimeType application/pdf " should " exception messages if content is having other fileType in artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130958930694553601102/artifact/index.epub"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Error! Invalid File Extension. Uploaded file https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130958930694553601102/artifact/index.epub is not a pdf file for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/pdf " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/pdf", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/epub " should " return exception messages if content is having invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/epub", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Error! Invalid File Extension. Uploaded file https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf is not a epub file for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/epub " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/epub", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_1130958930694553601102/artifact/index.epub"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/msword " should " return exception messages if content is having invalid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/msword", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(1) - result.contains("Error! Invalid File Extension. | Uploaded file https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11329603741667328018/artifact/do_11329603741667328018_1623058698775_intellijidea_referencecard.pdf should be among the Allowed_file_extensions for mimeType doc [doc, docx, ppt, pptx, key, odp, pps, odt, wpd, wps, wks] for : do_123") shouldBe true - } - - "validateMetadata with mimeType application/msword and .pptx " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/msword", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112216616320983040129/artifact/performance_out_1491286194831.pptx"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "validateMetadata with mimeType application/msword and .docx " should " not return exception messages if content is having valid artifactUrl" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "mimeType" -> "application/msword", "artifactUrl" -> "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112216615190192128128/artifact/prdassetstagging-2_1491286084107.docx"), None) - val result: List[String] = new TestLiveContentPublisher().validateMetadata(data, data.identifier, jobConfig) - result.size should be(0) - } - - "saveExternalData " should "save external data to cassandra table" in { - val data = new ObjectData("do_123", Map[String, AnyRef](), Some(Map[String, AnyRef]("body" -> "body", "answer" -> "answer"))) - new TestLiveContentPublisher().saveExternalData(data, readerConfig) - } - - "getExtData " should " get content body for application/vnd.ekstep.ecml-archive mimeType " in { - val identifier = "do_11321328578759884811663" - val result: Option[ObjectExtData] = new TestLiveContentPublisher().getExtData(identifier, "application/vnd.ekstep.ecml-archive", readerConfig) - result.getOrElse(new ObjectExtData).data.getOrElse(Map()).contains("body") shouldBe true - } - - "getExtData " should " not get content body for other than application/pdf mimeType " in { - val identifier = "do_11321328578759884811663" - val result: Option[ObjectExtData] = new TestLiveContentPublisher().getExtData(identifier, "application/pdf", readerConfig) - result.getOrElse(new ObjectExtData).data.getOrElse(Map()).contains("body") shouldBe false - } - - "getHierarchy " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestLiveContentPublisher().getHierarchy(identifier, readerConfig) - } - - "getExtDatas " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestLiveContentPublisher().getExtDatas(List(identifier), readerConfig) - } - - "getHierarchies " should "do nothing " in { - val identifier = "do_11329603741667328018" - new TestLiveContentPublisher().getHierarchies(List(identifier), readerConfig) - } - - "getDataForEcar" should "return one element in list" in { - val data = new ObjectData("do_123", Map("objectType" -> "Content"), Some(Map("responseDeclaration" -> "test")), Some(Map())) - val result: Option[List[Map[String, AnyRef]]] = new TestLiveContentPublisher().getDataForEcar(data) - result.size should be(1) - } - - "getObjectWithEcar" should "return object with ecar url" in { - val data = new ObjectData("do_123", Map("objectType" -> "Content", "identifier" -> "do_123", "name" -> "Test PDF Content"), Some(Map("responseDeclaration" -> "test", "media" -> "[{\"id\":\"do_1127129497561497601326\",\"type\":\"image\",\"src\":\"/content/do_1127129497561497601326.img/artifact/sunbird_1551961194254.jpeg\",\"baseUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev\"}]")), Some(Map())) - val result = new TestLiveContentPublisher().getObjectWithEcar(data, List(EcarPackageType.FULL.toString, EcarPackageType.ONLINE))(ec, mockNeo4JUtil, cloudStorageUtil, jobConfig, defCache, defConfig, httpUtil) - StringUtils.isNotBlank(result.metadata.getOrElse("downloadUrl", "").asInstanceOf[String]) - } -} - -class TestLiveContentPublisher extends LiveContentPublisher {} diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveObjectReaderTestSpec.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveObjectReaderTestSpec.scala deleted file mode 100644 index 67248f9cc..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveObjectReaderTestSpec.scala +++ /dev/null @@ -1,53 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.livenodepublisher.publish.helpers.LiveObjectReader -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{ExtDataConfig, ObjectExtData} -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} - -import scala.collection.JavaConverters._ - -class LiveObjectReaderTestSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val mockCassandraUtil: CassandraUtil = mock[CassandraUtil](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val jobConfig: LiveNodePublisherConfig = new LiveNodePublisherConfig(config) - - "Object Reader " should " read the metadata " in { - when(mockNeo4JUtil.getNodeProperties("do_123.img")).thenReturn(Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123.img", "IL_UNIQUE_ID" -> "do_123.img", "pkgVersion" -> 2.0.asInstanceOf[AnyRef]).asJava) - val objectReader = new TestObjectReader() - val readerConfig = ExtDataConfig("test", "test") - val obj = objectReader.getObject("do_123", 2, "", "Public", readerConfig) - val metadata = obj.metadata.asJava - metadata.isEmpty should be(false) - obj.extData should be(None) - obj.hierarchy should be(None) - } -} - -class TestObjectReader extends LiveObjectReader { - - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def getExtData(identifier: String, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, jobConfig: PublishConfig): Option[ObjectExtData] = None - - override def getHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, jobConfig: PublishConfig): Option[Map[String, AnyRef]] = None -} diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveObjectUpdaterSpec.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveObjectUpdaterSpec.scala deleted file mode 100644 index fb1cd4dbc..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/publish/helpers/spec/LiveObjectUpdaterSpec.scala +++ /dev/null @@ -1,80 +0,0 @@ -package org.sunbird.job.livenodepublisher.publish.helpers.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.livenodepublisher.publish.helpers.LiveObjectUpdater -import org.sunbird.job.livenodepublisher.task.LiveNodePublisherConfig -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} - -import java.util - -class LiveObjectUpdaterSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val mockCassandraUtil: CassandraUtil = mock[CassandraUtil](Mockito.withSettings().serializable()) - implicit val readerConfig = ExtDataConfig("test", "test") - implicit lazy val defCache: DefinitionCache = new DefinitionCache() - implicit val definitionConfig: DefinitionConfig = DefinitionConfig(Map("itemset" -> "2.0"), "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local") - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val jobConfig: LiveNodePublisherConfig = new LiveNodePublisherConfig(config) - - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "ObjectUpdater saveOnSuccess" should " update the status for successfully published data " in { - - when(mockNeo4JUtil.executeQuery(anyString())).thenReturn(any()); - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("objectType" -> "QuestionSet", "identifier" -> "do_123","publish_type" -> "Public", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - - val obj = new TestObjectUpdater() - obj.saveOnSuccess(objData) - } - - "ObjectUpdater saveOnFailure" should " update the status for failed published data " in { - - when(mockNeo4JUtil.executeQuery(anyString())).thenReturn(any()) - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("objectType" -> "QuestionSet","identifier" -> "do_123","publish_type" -> "Public", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - - val obj = new TestObjectUpdater() - obj.saveOnFailure(objData, List("Testing Save on Publish Failure"), 1) - } - - "ObjectUpdater metaDataQuery" should " give the update query " in { - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val variants = new util.HashMap[String, String] {{ - put("spine","https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_1132380439842324481319/hindi-questionset-17_1615972827743_do_1132380439842324481319_1_SPINE.ecar") - put("online","https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_1132380439842324481319/hindi-questionset-17_1615972829357_do_1132380439842324481319_1_ONLINE.ecar") - }} - val outcomeProcessing = Map("name" -> "abc") - val metadata: Map[String, AnyRef] = Map("description" -> "Hello \"World\"", "keywords" -> List("anusha"),"channel" -> "01309282781705830427","mimeType" -> "application/vnd.sunbird.questionset","showHints" -> "No","objectType" -> "QuestionSet","primaryCategory" -> "Practice Question Set","contentEncoding" -> "gzip","showSolutions" -> "No","identifier" -> "do_1132421849134858241686","visibility" -> "Default","showTimer" -> "Yes","author" -> "anusha","childNodes" -> util.Arrays.asList("do_1132421859856875521687"),"consumerId" -> "273f3b18-5dda-4a27-984a-060c7cd398d3","version" -> 1.asInstanceOf[AnyRef],"prevState" -> "Draft","IL_FUNC_OBJECT_TYPE" -> "QuestionSet","name" -> "Timer","timeLimits" -> "{\"maxTime\":\"3541\",\"warningTime\":\"2401\"}","IL_UNIQUE_ID" -> "do_1132421849134858241686","board" -> "CBSE", "variants" -> variants, "outcomeProcessing" -> outcomeProcessing) - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - val obj = new TestObjectUpdater() - println(obj.metaDataQuery(objData)(defCache, definitionConfig)) - } -} - -class TestObjectUpdater extends LiveObjectUpdater { - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None -} - diff --git a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/spec/LiveNodePublisherStreamTaskSpec.scala b/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/spec/LiveNodePublisherStreamTaskSpec.scala deleted file mode 100644 index 11c5540aa..000000000 --- a/publish-pipeline/live-node-publisher/src/test/scala/org/sunbird/job/livenodepublisher/spec/LiveNodePublisherStreamTaskSpec.scala +++ /dev/null @@ -1,108 +0,0 @@ -package org.sunbird.job.livenodepublisher.spec - -import com.google.gson.Gson -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.ArgumentMatchers.anyString -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.livenodepublisher.fixture.EventFixture -import org.sunbird.job.livenodepublisher.publish.domain.Event -import org.sunbird.job.livenodepublisher.task.{LiveNodePublisherConfig, LiveNodePublisherStreamTask} -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.ExtDataConfig -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.text.SimpleDateFormat -import java.util -import java.util.Date - -class LiveNodePublisherStreamTaskSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val jobConfig: LiveNodePublisherConfig = new LiveNodePublisherConfig(config) - var definitionCache = new DefinitionCache() - implicit val definition: ObjectDefinition = definitionCache.getDefinition("Collection", jobConfig.schemaSupportVersionMap.getOrElse("collection", "1.0").asInstanceOf[String], jobConfig.definitionBasePath) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.hierarchyKeyspaceName, jobConfig.hierarchyTableName, definition.getExternalPrimaryKey, definition.getExternalProps) - - val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - val publishConfig: PublishConfig = new PublishConfig(config, "") - val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - flinkCluster.after() - } - - def initialize(): Unit = { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new ContentPublishEventSource) - } - - def getTimeStamp: String = { - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - sdf.format(new Date()) - } - - ignore should " publish the content " in { - when(mockNeo4JUtil.getNodeProperties(anyString())).thenReturn(new util.HashMap[String, AnyRef]) - initialize - new LiveNodePublisherStreamTask(jobConfig, mockKafkaUtil, mockHttpUtil).process() - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.contentPublishEventCount}").getValue() should be(1) - } -} - -private class ContentPublishEventSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(jsonToEvent(EventFixture.PDF_EVENT1)) - } - - override def cancel() = {} - - def jsonToEvent(json: String): Event = { - val gson = new Gson() - val data = gson.fromJson(json, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]] - val metadataMap = data.get("edata").asInstanceOf[util.Map[String, Any]].get("metadata").asInstanceOf[util.Map[String, Any]] - metadataMap.put("pkgVersion", metadataMap.get("pkgVersion").asInstanceOf[Double].toInt) - new Event(data, 0, 10) - } -} diff --git a/publish-pipeline/pom.xml b/publish-pipeline/pom.xml deleted file mode 100644 index 8a0824a30..000000000 --- a/publish-pipeline/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - knowledge-platform-jobs - org.sunbird - 1.0 - - 4.0.0 - - publish-pipeline - pom - publish-pipeline - - - publish-core - - content-publish - live-node-publisher - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - org.sunbird.incredible - - - - - - - \ No newline at end of file diff --git a/publish-pipeline/publish-core/pom.xml b/publish-pipeline/publish-core/pom.xml deleted file mode 100644 index 0c8270be9..000000000 --- a/publish-pipeline/publish-core/pom.xml +++ /dev/null @@ -1,148 +0,0 @@ - - - 4.0.0 - - org.sunbird - publish-pipeline - 1.0 - - - publish-core - 1.0.0 - jar - publish-core - - - UTF-8 - 1.4.0 - - - - - org.sunbird - jobs-core - 1.0.0 - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - junit - junit - 4.13.1 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.apache.velocity - velocity-engine-core - 2.1 - - - org.imgscalr - imgscalr-lib - 4.2 - jar - - - - - src/main/scala - src/test/scala - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - kp-publish-core-testsuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.2.0 - - - - test-jar - - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - - - \ No newline at end of file diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/config/PublishConfig.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/config/PublishConfig.scala deleted file mode 100644 index b6b87752e..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/config/PublishConfig.scala +++ /dev/null @@ -1,10 +0,0 @@ -package org.sunbird.job.publish.config - -import com.typesafe.config.Config -import org.sunbird.job.BaseJobConfig - -class PublishConfig(override val config: Config, override val jobName: String) extends BaseJobConfig(config, jobName) { - - def getConfig() = config - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/core/Models.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/core/Models.scala deleted file mode 100644 index f78eb3abe..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/core/Models.scala +++ /dev/null @@ -1,21 +0,0 @@ -package org.sunbird.job.publish.core - -class ObjectData(val identifier: String, val metadata: Map[String, AnyRef], val extData: Option[Map[String, AnyRef]] = None, val hierarchy: Option[Map[String, AnyRef]] = None) { - - val dbId: String = metadata.getOrElse("identifier", identifier).asInstanceOf[String] - - val dbObjType: String = metadata.getOrElse("objectType", "").asInstanceOf[String] - - val pkgVersion: Int = metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number].intValue() - - val mimeType: String = metadata.getOrElse("mimeType", "").asInstanceOf[String] - - def getString(key: String, defaultVal: String): String = metadata.getOrElse(key, defaultVal).asInstanceOf[String] - -} - -case class ExtDataConfig(keyspace: String, table: String, primaryKey: List[String] = List(), propsMapping: Map[String, AnyRef] = Map()) - -case class DefinitionConfig(supportedVersion: Map[String, AnyRef], basePath: String) - -case class ObjectExtData(data: Option[Map[String, AnyRef]] = None, hierarchy: Option[Map[String, AnyRef]] = None) diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/handler/QuestionHandlerFactory.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/handler/QuestionHandlerFactory.scala deleted file mode 100644 index 42e2bade8..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/handler/QuestionHandlerFactory.scala +++ /dev/null @@ -1,59 +0,0 @@ -package org.sunbird.job.publish.handler - -import com.google.gson.Gson - -import java.util -import scala.collection.JavaConverters._ - -object QuestionHandlerFactory { - lazy private val gson = new Gson() - - - private class MCQHandler extends QuestionTypeHandler { - override def getQuestion(extData: Option[Map[String, AnyRef]]): String = { - extData.getOrElse(Map()).getOrElse("body", "").asInstanceOf[String] - } - - override def getAnswers(extData: Option[Map[String, AnyRef]]): List[String] = { - val responseDeclaration = gson.fromJson(extData.getOrElse(Map()).getOrElse("responseDeclaration", "").asInstanceOf[String], - classOf[java.util.Map[String, AnyRef]]).asScala - val valueOption: Option[Any] = if(null != responseDeclaration) responseDeclaration.getOrElse("response1", new util.HashMap()).asInstanceOf[java.util.Map[String, AnyRef]].asScala - .getOrElse("correctResponse", new util.HashMap()).asInstanceOf[java.util.Map[String, AnyRef]].asScala - .get("value") else Some(AnyRef) - val answersValues: List[Double] = valueOption match { - case Some(element: Double) => List[Double](element) - case Some(element: util.ArrayList[Double]) => element.asScala.toList - case _ => List() - } - val interactions = gson.fromJson(extData.getOrElse(Map()).getOrElse("interactions", "").asInstanceOf[String], - classOf[java.util.Map[String, AnyRef]]).asScala - - val answers = if(null != interactions) interactions.getOrElse("response1", new util.HashMap()).asInstanceOf[java.util.Map[String, AnyRef]].asScala - .getOrElse("options", new util.ArrayList[util.Map[String, AnyRef]]()).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]].asScala - .filter(element => answersValues.contains(element.asScala.getOrElse("value", -1).asInstanceOf[Double])) - .toList - .map(element => element.asScala.getOrElse("label", "").asInstanceOf[String]) else List() - answers - } - } - - private class SubjectiveHandler extends QuestionTypeHandler { - override def getQuestion(extData: Option[Map[String, AnyRef]]): String = { - extData.getOrElse(Map()).getOrElse("body", "").asInstanceOf[String] - - } - - override def getAnswers(extData: Option[Map[String, AnyRef]]): List[String] = { - List(extData.getOrElse(Map()).getOrElse("answer", "").asInstanceOf[String]) - } - } - - - def apply(questionType: Option[String]): Option[QuestionTypeHandler] = questionType match { - case Some("Multiple Choice Question") => Some(new MCQHandler) - case Some("Subjective Question") => Some(new SubjectiveHandler) - case Some("FTB Question") => None - case _ => None - } - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/handler/QuestionTypeHandler.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/handler/QuestionTypeHandler.scala deleted file mode 100644 index 69c9646ea..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/handler/QuestionTypeHandler.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.sunbird.job.publish.handler - -trait QuestionTypeHandler { - - def getQuestion(extData: Option[Map[String, AnyRef]]): String - - def getAnswers(extData: Option[Map[String, AnyRef]]): List[String] - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/EcarGenerator.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/EcarGenerator.scala deleted file mode 100644 index 368d4ec1d..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/EcarGenerator.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ObjectData} -import org.sunbird.job.util.{CloudStorageUtil, Neo4JUtil} - -import java.io.File -import scala.concurrent.ExecutionContext - -trait EcarGenerator extends ObjectBundle { - - private[this] val logger = LoggerFactory.getLogger(classOf[EcarGenerator]) - - def generateEcar(obj: ObjectData, pkgType: List[String])(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig): Map[String, String] = { - logger.info("Generating Ecar For : " + obj.identifier) - val enObjects: List[Map[String, AnyRef]] = getDataForEcar(obj).getOrElse(List()) - pkgType.flatMap(pkg => Map(pkg -> generateEcar(obj, enObjects, pkg))).toMap - } - - def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] - - // this method returns only cloud url for given pkg - def generateEcar(obj: ObjectData, objList: List[Map[String, AnyRef]], pkgType: String)(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig): String = { - logger.info(s"Generating ${pkgType} Ecar For : " + obj.identifier) - val bundle: File = getObjectBundle(obj, objList, pkgType) - uploadFile(Some(bundle), obj.identifier, obj.dbObjType.replaceAll("Image", "")).getOrElse("") - } - - private def uploadFile(fileOption: Option[File], identifier: String, objectType: String)(implicit cloudStorageUtil: CloudStorageUtil): Option[String] = { - fileOption match { - case Some(file: File) => { - logger.info("bundle file path ::: "+file.getAbsolutePath) - val folder = objectType.toLowerCase + File.separator + identifier - val urlArray: Array[String] = cloudStorageUtil.uploadFile(folder, file, Some(false)) - logger.info(s"EcarGenerator ::: uploadFile ::: ecar url for $identifier is : ${urlArray(1)}") - Some(urlArray(1)) - } - case _ => None - } - } -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/EcarPackageType.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/EcarPackageType.scala deleted file mode 100644 index f1d476064..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/EcarPackageType.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.sunbird.job.publish.helpers - -object EcarPackageType extends Enumeration { - - val FULL: String = Value("FULL").toString - val SPINE: String = Value("SPINE").toString - val OPTIMIZED: String = Value("OPTIMIZED").toString - val ONLINE: String = Value("ONLINE").toString -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/FrameworkDataEnrichment.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/FrameworkDataEnrichment.scala deleted file mode 100644 index 0caf3bd6e..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/FrameworkDataEnrichment.scala +++ /dev/null @@ -1,117 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.Neo4JUtil - -import java.util -import org.apache.commons.collections.CollectionUtils -import org.sunbird.job.cache.local.FrameworkMasterCategoryMap -import org.sunbird.job.publish.config.PublishConfig - -import scala.collection.JavaConverters._ -import scala.collection.immutable -import scala.collection.immutable.HashMap - -trait FrameworkDataEnrichment { - - private[this] val logger = LoggerFactory.getLogger(classOf[FrameworkDataEnrichment]) - - //private val fwMetaFields = List("boardIds", "subjectIds", "mediumIds", "topicsIds", "gradeLevelIds", "targetBoardIds", "targetSubjectIds", "targetMediumIds", "targetTopicIds", "targetGradeLevelIds") - //private val fwMetaMap = Map(("se_boardIds", "se_boards") -> List("boardIds", "targetBoardIds"), ("se_subjectIds", "se_subjects") -> List("subjectIds", "targetSubjectIds"), ("se_mediumIds", "se_mediums") -> List("mediumIds", "targetMediumIds"), ("se_topicIds", "se_topics") -> List("topicsIds", "targetTopicIds"), ("se_gradeLevelIds", "se_gradeLevels") -> List("gradeLevelIds", "targetGradeLevelIds")) - - private val frameworkCategorySearchMetadataMapping: HashMap[String, String] = HashMap[String, String]("se_boards" -> "board", "se_subjects"-> "subject", "se_mediums" -> "medium", "se_topics"-> "topic", "se_gradeLevels"-> "gradeLevel") - - def enrichFrameworkData(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, config: PublishConfig): ObjectData = { - val (fwMetaFields, fwMetaMap) : (List[String], Map[(String, String), List[String]]) = - if(StringUtils.equalsIgnoreCase(config.getString("master.category.validation.enabled", "Yes"), "Yes")) - getFrameworkCategoryMetadata("domain", "Category") - else (List(), Map()) - val enMetadata = enrichFwData(obj, fwMetaFields, fwMetaMap) - logger.info("Enriched Framework Metadata for " + obj.identifier + " are : " + enMetadata) - val reMetadata = revalidateFrameworkCategoryMetadata(obj, enMetadata) - logger.info("Revalidated Framework Metadata for " + obj.identifier + " are : " + reMetadata) - val finalMeta = if(reMetadata.nonEmpty) obj.metadata ++ reMetadata else obj.metadata - new ObjectData(obj.identifier, finalMeta, obj.extData, obj.hierarchy) - } - - private def enrichFwData(obj: ObjectData, fwMetaFields: List[String], fwMetaMap: Map[(String, String), List[String]])(implicit neo4JUtil: Neo4JUtil): Map[String, AnyRef] = { - val mFwId: List[String] = getList(obj.metadata.getOrElse("framework", "")) ::: getList(obj.metadata.getOrElse("targetFWIds", List())) - if (mFwId.isEmpty) Map() else { - val labels: Map[String, List[String]] = getLabels(obj.identifier, obj.metadata, fwMetaFields) - val metaMap = fwMetaMap.flatMap(entry => Map(entry._1._1 -> (getList(obj.metadata.getOrElse(entry._2(0), List())) ::: getList(obj.metadata.getOrElse(entry._2(1), List()))).distinct, entry._1._2 -> (labels.getOrElse(entry._2(0), List()) ::: labels.getOrElse(entry._2(1), List())).distinct)) - (metaMap ++ Map("se_FWIds" -> mFwId.distinct)).filter(entry => entry._2.nonEmpty) - } - } - - private def revalidateFrameworkCategoryMetadata(obj: ObjectData, enMetadata: Map[String, AnyRef]) : Map[String, AnyRef] = { - val updatedFwData: immutable.Iterable[(String, AnyRef)] = frameworkCategorySearchMetadataMapping.map(category => { - val data: AnyRef = obj.metadata.getOrElse(category._2, null) - if(data != null) Map(category._1 -> getList(data)) else Map.empty - }).filter(rec => rec.nonEmpty).flatten - - enMetadata ++ updatedFwData - } - - private def getLabels(identifier: String, metadata: Map[String, AnyRef], fwMetaFields: List[String])(implicit neo4JUtil: Neo4JUtil): Map[String, List[String]] = { - val fwMetaIds = fwMetaFields.flatMap(meta => getList(metadata.getOrElse(meta, List()))) - if (fwMetaIds.isEmpty) { - logger.info("No framework categories are present for identifier : " + identifier) - Map() - } else { - val nameMap: Map[String, String] = neo4JUtil.getNodesName(fwMetaIds) - if (nameMap.isEmpty) Map() else { - fwMetaFields.flatMap(meta => { - val metaNames: List[String] = getList(metadata.getOrElse(meta, List())).map(id => nameMap.getOrElse(id, "")) - Map(meta -> metaNames) - }).toMap - } - } - } - - private def getList(obj: AnyRef): List[String] = { - (obj match { - case obj: List[String] => obj.distinct - case obj: String => List(obj).distinct - case obj: util.List[String] => obj.asScala.toList.distinct - case _ => List.empty - }).filter((x: String) => StringUtils.isNotBlank(x) && !StringUtils.equals(" ", x)) - } - - def getFrameworkCategoryMetadata(graphId: String, objectType: String)(implicit neo4JUtil: Neo4JUtil): (List[String], Map[(String, String), List[String]]) ={ - val masterCategories: List[Map[String, AnyRef]] = getMasterCategory(graphId, objectType) - val fwMetaFields: List[String] = masterCategories.flatMap(category => - List(category.getOrElse("orgIdFieldName", "").asInstanceOf[String], - category.getOrElse("targetIdFieldName", "").asInstanceOf[String])) - - val fwMetaMap: Map[(String, String), List[String]] = masterCategories.map(category => - (category.getOrElse("searchIdFieldName", "").asInstanceOf[String], category.getOrElse("searchLabelFieldName", "").asInstanceOf[String]) -> - List(category.getOrElse("orgIdFieldName", "").asInstanceOf[String], category.getOrElse("targetIdFieldName", "").asInstanceOf[String])).toMap - (fwMetaFields, fwMetaMap) - } - - - def getMasterCategory(graphId: String, objectType: String)(implicit neo4JUtil: Neo4JUtil): List[Map[String, AnyRef]] = { - if (FrameworkMasterCategoryMap.containsKey("masterCategories") && null != FrameworkMasterCategoryMap.get("masterCategories")) { - val masterCategories: Map[String, AnyRef] = FrameworkMasterCategoryMap.get("masterCategories") - masterCategories.map(obj => obj._2.asInstanceOf[Map[String, AnyRef]]).toList - }else{ - val nodes: util.List[util.Map[String, AnyRef]] = neo4JUtil.getNodePropertiesWithObjectType(objectType) - if(CollectionUtils.isEmpty(nodes)){ - logger.info("No Framework Master Category found.") - List() - }else{ - val masterCategories: Map[String, AnyRef] = nodes.asScala.map(node => node.getOrDefault("code", "").asInstanceOf[String] -> - Map("code" -> node.getOrDefault("code", "").asInstanceOf[String], - "orgIdFieldName" -> node.getOrDefault("orgIdFieldName", "").asInstanceOf[String], - "targetIdFieldName" -> node.getOrDefault("targetIdFieldName", "").asInstanceOf[String], - "searchIdFieldName" -> node.getOrDefault("searchIdFieldName", "").asInstanceOf[String], - "searchLabelFieldName" -> node.getOrDefault("searchLabelFieldName", "").asInstanceOf[String]) - ).toMap - FrameworkMasterCategoryMap.put("masterCategories", masterCategories) - masterCategories.map(obj => obj._2.asInstanceOf[Map[String, AnyRef]]).toList - } - } - } -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/MimeType.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/MimeType.scala deleted file mode 100644 index 27a22247a..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/MimeType.scala +++ /dev/null @@ -1,61 +0,0 @@ -package org.sunbird.job.publish.helpers - -object MimeType extends Enumeration { - - val PDF: String = Value("application/pdf").toString - val EPUB: String = Value("application/epub").toString - val MSWORD: String = Value("application/msword").toString - val H5P_Archive: String = Value("application/vnd.ekstep.h5p-archive").toString - - val X_Youtube: String = Value("video/x-youtube").toString - val Youtube: String = Value("video/youtube").toString - val X_URL: String = Value("text/x-url").toString - - val ECML_Archive: String = Value("application/vnd.ekstep.ecml-archive").toString - val HTML_Archive: String = Value("application/vnd.ekstep.html-archive").toString - val Android_Package: String = Value("application/vnd.android.package-archive").toString - val Content_Archive: String = Value("application/vnd.ekstep.content-archive").toString - - val ASSETS: String = Value("assets").toString - val Plugin_Archive: String = Value("application/vnd.ekstep.plugin-archive").toString - val Collection: String = Value("application/vnd.ekstep.content-collection").toString - - val Content_Package: String = Value("application/octet-stream").toString - val JSON_FILE: String = Value("application/json").toString - val JS_FILE: String = Value("application/javascript").toString - val XML_FILE: String = Value("application/xml").toString - val TEXT_FILE: String = Value("text/plain").toString - val HTML_FILE: String = Value("text/html").toString - val TEXT_JS_FILE: String = Value("text/javascript").toString - val TEXT_XML_FILE: String = Value("text/xml").toString - val CSS_FILE: String = Value("text/css").toString - val JPEG_Image: String = Value("image/jpeg").toString - val JPG_Image: String = Value("image/jpg").toString - val PNG_Image: String = Value("image/png").toString - val TIFF_Image: String = Value("image/tiff").toString - val BMP_Image: String = Value("image/bmp").toString - val GIF_Image: String = Value("image/gif").toString - val SVG_XML_Image: String = Value("image/svg+xml").toString - val QUICK_TIME_Image: String = Value("image/x-quicktime").toString - val AVI_Video: String = Value("video/avi").toString - val AVI_MS_VIDEO_Video: String = Value("video/msvideo").toString - val AVI_X_MS_VIDEO_Video: String = Value("video/x-msvideo").toString - val QUICK_TIME_Video: String = Value("video/quicktime").toString - val X_QUICK_TIME_Video: String = Value("video/x-qtc").toString - val III_gpp_Video: String = Value("video/3gpp").toString - val MP4_Video: String = Value("video/mp4").toString - val OGG_Video: String = Value("video/ogg").toString - val WEBM_Video: String = Value("video/webm").toString - val MP3_MPEG_Audio: String = Value("video/mpeg").toString - val MP3_X_MPEG_Audio: String = Value("video/x-mpeg").toString - val MP3_Audio: String = Value("audio/mp3").toString - val MP3_MPEG3_Audio: String = Value("audio/mpeg3").toString - val MP3_X_MPEG3_Audio: String = Value("audio/x-mpeg-3").toString - val MP4_Audio: String = Value("audio/mp4").toString - val MPEG_Audio: String = Value("audio/mpeg").toString - val OGG_Audio: String = Value("audio/ogg").toString - val OGG_VORBIS_Audio: String = Value("audio/vorbis").toString - val WEBM_Audio: String = Value("audio/webm").toString - val X_WAV_Audio: String = Value("audio/x-wav").toString - val X_FONT_TTF: String = Value("application/x-font-ttf").toString -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectBundle.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectBundle.scala deleted file mode 100644 index 6fe1773e2..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectBundle.scala +++ /dev/null @@ -1,341 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.apache.commons.io.{FilenameUtils, IOUtils} -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ObjectData} -import org.sunbird.job.util.{FileUtils, JSONUtil, Neo4JUtil, ScalaJsonUtil, Slug} - -import java.io._ -import java.net.URL -import java.text.SimpleDateFormat -import java.util -import java.util.zip.{ZipEntry, ZipOutputStream} -import java.util.{Date, Optional} -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext, Future} -import scala.collection.JavaConverters._ - -trait ObjectBundle { - - private[this] val logger = LoggerFactory.getLogger(classOf[ObjectBundle]) - private val onlineMimeTypes = List("video/youtube", "video/x-youtube", "text/x-url") - private val bundleLocation: String = "/tmp" - private val defaultManifestVersion = "1.2" - private val manifestFileName = "manifest.json" - private val hierarchyFileName = "hierarchy.json" - private val hierarchyVersion = "1.0" - val excludeBundleMeta = List("screenshots", "posterImage", "index", "depth") - - def getBundleFileName(identifier: String, metadata: Map[String, AnyRef], pkgType: String)(implicit config: PublishConfig): String = { - val maxAllowedContentName = config.getInt("max_allowed_content_name", 120) - val contentName = if (metadata.getOrElse("name", "").asInstanceOf[String].length > maxAllowedContentName) metadata.getOrElse("name", "").asInstanceOf[String].substring(0, maxAllowedContentName) else metadata.getOrElse("name", "").asInstanceOf[String] - Slug.makeSlug(contentName, isTransliterate = true) + "_" + System.currentTimeMillis() + "_" + identifier + "_" + metadata.getOrElse("pkgVersion", "") + (if (StringUtils.equals(EcarPackageType.FULL, pkgType)) ".ecar" else "_" + pkgType + ".ecar") - } - - def getManifestData(objIdentifier: String, rootObjectType: String, pkgType: String, objList: List[Map[String, AnyRef]])(implicit defCache: DefinitionCache, neo4JUtil: Neo4JUtil, defConfig: DefinitionConfig, config: PublishConfig): (List[Map[String, AnyRef]], List[Map[AnyRef, String]]) = { - objList.map(data => { - val identifier = data.getOrElse("identifier", "").asInstanceOf[String].replaceAll(".img", "") - val mimeType = data.getOrElse("mimeType", "").asInstanceOf[String] - val objectType: String = if(!data.contains("objectType") || data.getOrElse("objectType", "").asInstanceOf[String].isBlank || data.getOrElse("objectType", "").asInstanceOf[String].isEmpty) { - val metaData = Option(neo4JUtil.getNodeProperties(identifier)).getOrElse(neo4JUtil.getNodeProperties(identifier)).asScala.toMap - logger.info("ObjectBundle:: getManifestData:: if objectType does not exist identifier:: " + identifier) - if (metaData == null || metaData.isEmpty) rootObjectType else metaData.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String] - } else data.getOrElse("objectType", "").asInstanceOf[String] .replaceAll("Image", "") - val contentDisposition = data.getOrElse("contentDisposition", "").asInstanceOf[String] - logger.info("ObjectBundle:: getManifestData:: identifier:: " + identifier + " || objectType:: " + objectType) - val dUrlMap: Map[AnyRef, String] = getDownloadUrls(identifier, pkgType, isOnline(mimeType, contentDisposition), data) - val updatedObj: Map[String, AnyRef] = data.map(entry => - if (dUrlMap.contains(entry._2)) { - (entry._1, dUrlMap.getOrElse(entry._2.asInstanceOf[String], "").asInstanceOf[AnyRef]) - } else if (StringUtils.equalsIgnoreCase(EcarPackageType.FULL, pkgType) && StringUtils.equalsIgnoreCase(entry._1, "media")) { - val media: List[Map[String, AnyRef]] = Optional.ofNullable(ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](entry._2.asInstanceOf[String])).orElse(List[Map[String, AnyRef]]()) - val newMedia = media.map(m => { - m.map(entry => { - entry._1 match { - case "baseUrl" => (entry._1, "") - case "src" => (entry._1, getRelativePath(identifier, entry._2.asInstanceOf[String])) - case _ => entry - } - }) - }) - (entry._1, ScalaJsonUtil.serialize(newMedia)) - } else { - if (entry._1.equalsIgnoreCase("identifier")) - entry._1 -> entry._2.asInstanceOf[String].replaceAll(".img", "") - else if (entry._1.equalsIgnoreCase("objectType")) - entry._1 -> entry._2.asInstanceOf[String].replaceAll("Image", "") - else entry - } - ) - val downloadUrl: String = updatedObj.getOrElse("downloadUrl", "").asInstanceOf[String] - val dUrl: String = if (StringUtils.isNotBlank(downloadUrl)) downloadUrl else updatedObj.getOrElse("artifactUrl", "").asInstanceOf[String] - val dMap = if (StringUtils.equalsIgnoreCase(contentDisposition, "online-only")) Map("downloadUrl" -> null) else Map("downloadUrl" -> dUrl) - val downloadUrls: Map[AnyRef, String] = dUrlMap.keys.flatMap(key => Map(key -> identifier)).toMap - - // TODO: Addressing visibility "Parent" issue for collection children as expected by Mobile - ContentBundle.java line120 - start - val mergedMeta = if(!identifier.equalsIgnoreCase(objIdentifier) && (objectType.equalsIgnoreCase("Content") - || objectType.equalsIgnoreCase("Collection") || objectType.equalsIgnoreCase("QuestionSet"))) { - updatedObj + ("visibility" -> "Parent") ++ dMap - } else updatedObj ++ dMap - // TODO: Addressing visibility "Parent" issue for collection children as expected by Mobile - ContentBundle.java line120 - end - - val definition: ObjectDefinition = defCache.getDefinition(objectType, defConfig.supportedVersion.getOrElse(objectType.toLowerCase, "1.0").asInstanceOf[String], defConfig.basePath) - val enMeta = mergedMeta.filter(x => null != x._2).map(element => (element._1, convertJsonProperties(element, definition.getJsonProps()))) - (enMeta, downloadUrls) - }).unzip - } - - def getObjectBundle(obj: ObjectData, objList: List[Map[String, AnyRef]], pkgType: String)(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig): File = { - val bundleFileName = bundleLocation + File.separator + getBundleFileName(obj.identifier, obj.metadata, pkgType) - val bundlePath = bundleLocation + File.separator + System.currentTimeMillis + "_temp" - val objType = if(obj.getString("objectType", "").replaceAll("Image", "").equalsIgnoreCase("collection")) "content" else obj.getString("objectType", "").replaceAll("Image", "") - logger.info("ObjectBundle ::: getObjectBundle ::: input objList :::: " + objList) - // create manifest data - val (updatedObjList, dUrls) = getManifestData(obj.identifier, objType, pkgType, objList) - logger.info("ObjectBundle ::: getObjectBundle ::: updatedObjList :::: " + updatedObjList) - val downloadUrls: Map[AnyRef, List[String]] = dUrls.flatten.groupBy(_._1).map { case (k, v) => k -> v.map(_._2) } - logger.info("ObjectBundle ::: getObjectBundle ::: downloadUrls :::: " + downloadUrls) - val duration: String = config.getString("media_download_duration", "300 seconds") - val downloadedMedias: List[File] = Await.result(downloadFiles(obj.identifier, downloadUrls, bundlePath), Duration.apply(duration)) - if (downloadUrls.nonEmpty && downloadedMedias.isEmpty) - throw new InvalidInputException("Error Occurred While Downloading Bundle Media Files For : " + obj.identifier) - val manifestFile: File = getManifestFile(obj.identifier, objType, bundlePath, updatedObjList) - val hierarchyFile: File = getHierarchyFile(obj, bundlePath).getOrElse(new File(bundlePath)) - val fList = if (obj.hierarchy.getOrElse(Map()).nonEmpty) List(manifestFile, hierarchyFile) else List(manifestFile) - createBundle(obj.identifier, bundleFileName, bundlePath, pkgType, downloadedMedias ::: fList) - } - - //TODO: Enhance this method of .ecar & .zip extension - def downloadFiles(identifier: String, files: Map[AnyRef, List[String]], bundlePath: String)(implicit ec: ExecutionContext): Future[List[File]] = { - val futures = files.map { - case (k, v) => - v.map { - id => { - Future { - val destPath = s"""$bundlePath${File.separator}${StringUtils.replace(id, ".img", "")}""" - logger.info(s"ObjectBundle ::: downloadFiles ::: Processing file: $k for : " + identifier) - k match { - case _: File => - val file = k.asInstanceOf[File] - val newFile = new File(s"""${destPath}${File.separator}${file.getName}""") - FileUtils.copyFile(file, newFile) - newFile - case _ => - val url = k.asInstanceOf[String] - // UnknownHostException | FileNotFoundException - try { - FileUtils.downloadFile(url, destPath) - } catch { - case e: Exception => throw new InvalidInputException(s"Error while downloading file $url", e) - } - } - } - } - } - }.flatten.toList - Future.sequence(futures) - } - - def createBundle(identifier: String, bundleFileName: String, bundlePath: String, pkgType: String, downloadedFiles: List[File]): File = { - try { - val stream = new FileOutputStream(bundleFileName) - stream.write(getByteStream(identifier, downloadedFiles)) - stream.flush() - stream.close() - new File(bundleFileName) - } catch { - case ex: Exception => - ex.printStackTrace() - throw new Exception(s"Error While Generating $pkgType ECAR Bundle For : " + identifier, ex) - } finally { - FileUtils.deleteDirectory(new File(bundlePath)) - } - } - - def getByteStream(identifier: String, files: List[File]): Array[Byte] = { - val byteArrayOutputStream = new ByteArrayOutputStream - val bufferedOutputStream = new BufferedOutputStream(byteArrayOutputStream) - val zipOutputStream = new ZipOutputStream(bufferedOutputStream) - try { - files.foreach(file => { - val fileName = getFileName(file) - try { - zipOutputStream.putNextEntry(new ZipEntry(fileName)) - val fileInputStream = new FileInputStream(file) - IOUtils.copy(fileInputStream, zipOutputStream) - } catch {case ze:java.util.zip.ZipException => logger.info("ObjectBundle:: getByteStream:: ", ze.getMessage) } - zipOutputStream.closeEntry() - }) - - if (zipOutputStream != null) { - zipOutputStream.finish() - zipOutputStream.flush() - IOUtils.closeQuietly(zipOutputStream) - } - IOUtils.closeQuietly(bufferedOutputStream) - IOUtils.closeQuietly(byteArrayOutputStream) - byteArrayOutputStream.toByteArray - } catch { - case ex: Exception => throw new Exception("Error While Generating Byte Stream Of Bundle For : " + identifier, ex) - } - } - - def getFileName(file: File): String = { - if (file.getName().toLowerCase().endsWith("manifest.json") || file.getName().endsWith("hierarchy.json")) file.getName else - file.getParent().substring(file.getParent().lastIndexOf(File.separator) + 1) + File.separator + file.getName() - } - - def getDownloadUrls(identifier: String, pkgType: String, isOnlineObj: Boolean, data: Map[String, AnyRef])(implicit config: PublishConfig): Map[AnyRef, String] = { - val urlFields = pkgType match { - case "ONLINE" => List.empty - case "SPINE" => - val spineDownloadFiles: util.List[String] = if (config.getConfig().hasPath("content.downloadFiles.spine")) config.getConfig().getStringList("content.downloadFiles.spine") else util.Arrays.asList[String]("appIcon") - spineDownloadFiles.asScala.toList - case _ => - val fullDownloadFiles: util.List[String] = if (config.getConfig().hasPath("content.downloadFiles.full")) config.getConfig().getStringList("content.downloadFiles.full") else util.Arrays.asList[String]("appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media") - fullDownloadFiles.asScala.toList - } - data.filter(en => urlFields.contains(en._1) && null != en._2).flatMap(entry => { - isOnlineObj match { - case true => { - if (!StringUtils.equalsIgnoreCase("artifactUrl", entry._1) && validUrl(entry._2.asInstanceOf[String])) { - getUrlMap(identifier, pkgType, entry._1, entry._2) - } else Map[AnyRef, String]() - } - case false => { - if (StringUtils.equalsIgnoreCase(entry._1, "media")) { - logger.info("MEDIA:: " + entry._2.asInstanceOf[String]) - val media: List[Map[String, AnyRef]] = if (null != entry._2) ScalaJsonUtil.deserialize[List[Map[String, AnyRef]]](entry._2.asInstanceOf[String]) else List() - logger.info("MEDIA:::: " + media) - getMediaUrl(media, identifier, pkgType) - } else { - entry._2 match { - case _: File => - getUrlMap(identifier, pkgType, entry._1, entry._2) - case str: String if validUrl(str) => - getUrlMap(identifier, pkgType, entry._1, entry._2) - case _ => Map[AnyRef, String]() - } - } - } - } - }) - } - - def getMediaUrl(media: List[Map[String, AnyRef]], identifier: String, pkgType: String): Map[AnyRef, String] = { - media.map(entry => { - val url = entry.getOrElse("baseUrl", "").asInstanceOf[String] + entry.getOrElse("src", "").asInstanceOf[String] - if (url.isInstanceOf[String] && validUrl(url.asInstanceOf[String])) { - Map[AnyRef, String](url -> (identifier.trim + entry.getOrElse("src", "").asInstanceOf[String])) - } else Map[AnyRef, String]() - }).flatten.toMap - } - - def getUrlMap(identifier: String, pkgType: String, key: String, value: AnyRef): Map[AnyRef, String] = { - val pkgKeys = List("artifactUrl", "downloadUrl") - if (!pkgKeys.contains(key) || StringUtils.equalsIgnoreCase(EcarPackageType.FULL, pkgType)) { - val fileName = value match { - case file: File => file.getName - case _ => value.asInstanceOf[String] - } - Map[AnyRef, String](value -> getRelativePath(identifier, fileName.asInstanceOf[String])) - } else Map[AnyRef, String]() - } - - def getRelativePath(identifier: String, value: String): String = { - val fileName = FilenameUtils.getName(value) - val suffix = if (fileName.endsWith(".ecar")) identifier.trim + ".zip" else Slug.makeSlug(fileName, isTransliterate = true) - val filePath = identifier.trim + File.separator + suffix - filePath - } - - @throws[Exception] - def getManifestFile(identifier: String, objType: String, bundlePath: String, objList: List[Map[String, AnyRef]]): File = { - try { - val file: File = new File(bundlePath + File.separator + manifestFileName) - val header: String = s"""{"id": "sunbird.${objType.toLowerCase()}.archive", "ver": "$defaultManifestVersion" ,"ts":"$getTimeStamp", "params":{"resmsgid": "$getUUID"}, "archive":{ "count": ${objList.size}, "ttl":24, "items": """ - val mJson = header + ScalaJsonUtil.serialize(objList) + "}}" - FileUtils.writeStringToFile(file, mJson) - file - } catch { - case e: Exception => throw new Exception("Exception occurred while writing manifest file for : " + identifier, e) - } - } - - @throws[Exception] - def getHierarchyFile(obj: ObjectData, bundlePath: String): Option[File] = { - try { - if (obj.hierarchy.getOrElse(Map()).nonEmpty) { - val file: File = new File(bundlePath + File.separator + hierarchyFileName) - val objType: String = obj.getString("objectType", "") - val metadata = obj.metadata - ("IL_UNIQUE_ID", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE") - val children = obj.hierarchy.get.getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - val hMap: Map[String, AnyRef] = metadata ++ Map("identifier" -> obj.identifier.replace(".img", ""), "objectType" -> objType, "children" -> children) - val hJson = getHierarchyHeader(objType.toLowerCase()) + ScalaJsonUtil.serialize(hMap) + "}" - FileUtils.writeStringToFile(file, hJson) - Option(file) - } else None - } catch { - case e: Exception => throw new Exception("Exception occurred while writing hierarchy file for : " + obj.identifier, e) - } - } - - def isOnline(mimeType: String, contentDisposition: String): Boolean = onlineMimeTypes.contains(mimeType) || StringUtils.equalsIgnoreCase(contentDisposition, "online-only") - - def validUrl(str: String): Boolean = { - if (str.isEmpty) false else { - try { - new URL(str).toURI - true - } catch { - case _ => false - } - } - } - - def getHierarchyHeader(objType: String): String = { - s"""{"id": "sunbird.$objType.hierarchy", "ver": "$hierarchyVersion" ,"ts":"$getTimeStamp", "params":{"resmsgid": "$getUUID"}, "$objType": """ - } - - def getTimeStamp: String = { - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - sdf.format(new Date()) - } - - def getUUID: String = util.UUID.randomUUID().toString - - def convertJsonProperties(entry: (String, AnyRef), jsonProps: scala.List[String]): AnyRef = { - if (jsonProps.contains(entry._1)) { - try { - JSONUtil.deserialize[Object](entry._2.asInstanceOf[String]) - } - catch { - case e: Exception => entry._2 - } - } - else entry._2 - } - - def getFlatStructure(children: List[Map[String, AnyRef]], childrenList: List[Map[String, AnyRef]]): List[Map[String, AnyRef]] = { - children.flatMap(child => { - val innerChildren = getInnerChildren(child) - val updatedChild: Map[String, AnyRef] = if (innerChildren.nonEmpty) child ++ Map("children" -> innerChildren) else child - val finalChild = updatedChild.filter(p => !excludeBundleMeta.contains(p._1.asInstanceOf[String])) - val updatedChildren: List[Map[String, AnyRef]] = finalChild :: childrenList - val result = getFlatStructure(child.getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]], updatedChildren) - finalChild :: result - }).distinct - } - - def getInnerChildren(child: Map[String, AnyRef]): List[Map[String, AnyRef]] = { - val metaList: List[String] = List("identifier", "name", "objectType", "description", "index", "depth") - child.getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - .map(ch => ch.filterKeys(key => metaList.contains(key))) - } - - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectEnrichment.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectEnrichment.scala deleted file mode 100644 index a0f913466..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectEnrichment.scala +++ /dev/null @@ -1,18 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, Neo4JUtil} - -trait ObjectEnrichment extends FrameworkDataEnrichment with ThumbnailGenerator { - - def enrichObject(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig): ObjectData = { - val newObj = enrichFrameworkData(obj) - val enObj = enrichObjectMetadata(newObj).getOrElse(newObj) - generateThumbnail(enObj).getOrElse(enObj) - } - - def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig): Option[ObjectData] - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectReader.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectReader.scala deleted file mode 100644 index 760360f58..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectReader.scala +++ /dev/null @@ -1,47 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} - -import scala.collection.JavaConverters._ - -trait ObjectReader { - - private[this] val logger = LoggerFactory.getLogger(classOf[ObjectReader]) - - def getObject(identifier: String, pkgVersion: Double, mimeType: String, publishType: String, readerConfig: ExtDataConfig)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, config: PublishConfig): ObjectData = { - logger.info("Reading editable object data for: " + identifier + " with pkgVersion: " + pkgVersion) - val metadata = getMetadata(identifier, mimeType, publishType, pkgVersion) - logger.info("Reading metadata for: " + identifier + " with metadata: " + metadata) - val extData = getExtData(identifier, pkgVersion, mimeType, readerConfig) - logger.info("Reading extData for: " + identifier + " with extData: " + extData) - new ObjectData(identifier, metadata, extData.getOrElse(ObjectExtData()).data, extData.getOrElse(ObjectExtData()).hierarchy) - } - - private def getMetadata(identifier: String, mimeType: String, publishType: String, pkgVersion: Double)(implicit neo4JUtil: Neo4JUtil): Map[String, AnyRef] = { - val nodeId = getEditableObjId(identifier, pkgVersion) - val metaData = Option(neo4JUtil.getNodeProperties(nodeId)).getOrElse(neo4JUtil.getNodeProperties(identifier)).asScala.toMap - val id = metaData.getOrElse("IL_UNIQUE_ID", identifier).asInstanceOf[String] - val objType = metaData.getOrElse("IL_FUNC_OBJECT_TYPE", "").asInstanceOf[String] - logger.info("ObjectReader:: getMetadata:: identifier: " + identifier + " with objType: " + objType) - if(mimeType.contains("application/vnd.sunbird.question") || objType.equalsIgnoreCase("Question") || objType.equalsIgnoreCase("QuestionSet")) - metaData ++ Map[String, AnyRef]("identifier" -> id, "objectType" -> objType) - ("IL_UNIQUE_ID", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE") - else - metaData ++ Map[String, AnyRef]("identifier" -> id, "objectType" -> objType, "publish_type" -> publishType) - ("IL_UNIQUE_ID", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE") - } - - def getExtData(identifier: String, pkgVersion: Double, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] - - def getHierarchy(identifier: String, pkgVersion: Double, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] - - def getEditableObjId(identifier: String, pkgVersion: Double): String = { - if (pkgVersion > 0) identifier + ".img" else identifier - } - - def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] - - def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectTemplateGenerator.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectTemplateGenerator.scala deleted file mode 100644 index b1ecc4e1d..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectTemplateGenerator.scala +++ /dev/null @@ -1,29 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.apache.velocity.VelocityContext -import org.apache.velocity.app.Velocity - -import java.io.StringWriter -import java.util.Properties - -trait ObjectTemplateGenerator { - - - def handleHtmlTemplate(templateName: String, context: Map[String,AnyRef]): String = { - initVelocityEngine(templateName) - val veContext: VelocityContext = new VelocityContext() - context.foreach(entry => veContext.put(entry._1, entry._2)) - val writer:StringWriter = new StringWriter() - Velocity.mergeTemplate(templateName, "UTF-8", veContext, writer) - writer.toString - } - - private def initVelocityEngine(templateName: String): Unit = { - val properties = new Properties() - if (!templateName.startsWith("http") && !templateName.startsWith("/")) { - properties.setProperty("resource.loader", "class") - properties.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader") - } - Velocity.init(properties) - } -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectUpdater.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectUpdater.scala deleted file mode 100644 index f5f99eb6e..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectUpdater.scala +++ /dev/null @@ -1,164 +0,0 @@ -package org.sunbird.job.publish.helpers - -import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} -import org.apache.commons.lang3.StringUtils -import org.neo4j.driver.v1.StatementResult -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.util.{CSPMetaUtil, CassandraUtil, JSONUtil, Neo4JUtil, ScalaJsonUtil} - -import java.text.SimpleDateFormat -import java.util -import java.util.Date - -trait ObjectUpdater { - - private[this] val logger = LoggerFactory.getLogger(classOf[ObjectUpdater]) - - @throws[Exception] - def saveOnSuccess(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig, config: PublishConfig): Unit = { - val publishType = obj.getString("publish_type", "Public") - val status = if (StringUtils.equalsIgnoreCase("Unlisted", publishType)) "Unlisted" else "Live" - val editId = obj.dbId - val identifier = obj.identifier - val metadataUpdateQuery = metaDataQuery(obj)(definitionCache, definitionConfig) - val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"$identifier"}) SET n.status="$status",n.pkgVersion=${obj.pkgVersion},n.prevStatus="Processing",$metadataUpdateQuery,$auditPropsUpdateQuery;""" - logger.info("ObjectUpdater:: saveOnSuccess:: Query: " + query) - logger.info(s"ObjectUpdater:: saveOnSuccess:: DB ID for ${obj.identifier} is : ${obj.dbId} | pkgVersion : ${obj.pkgVersion}" ) - - if (obj.mimeType.equalsIgnoreCase("application/vnd.ekstep.ecml-archive")) { - val ecmlBody = getContentBody(identifier, readerConfig) - updateContentBody(identifier,ecmlBody,readerConfig) - } - - if (!StringUtils.equalsIgnoreCase(editId, identifier)) { - val imgNodeDelQuery = s"""MATCH (n:domain{IL_UNIQUE_ID:"$editId"}) DETACH DELETE n;""" - neo4JUtil.executeQuery(imgNodeDelQuery) - deleteExternalData(obj, readerConfig) - logger.info(s"Image Node Data Is Deleted Successfully For ${editId}") - } - val result: StatementResult = neo4JUtil.executeQuery(query) - if (null != result && result.hasNext) - logger.info(s"ObjectUpdater:: saveOnSuccess:: statement result : ${result.next().asMap()}") - saveExternalData(obj, readerConfig) - } - - @throws[Exception] - def updateProcessingNode(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definitionCache: DefinitionCache, config: DefinitionConfig): Unit = { - val status = "Processing" - val prevState = obj.getString("status", "Draft") - val identifier = obj.dbId - val metadataUpdateQuery = metaDataQuery(obj)(definitionCache, config) - val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"$identifier"}) SET n.status="$status",n.prevState="$prevState",$metadataUpdateQuery,$auditPropsUpdateQuery;""" - logger.info("ObjectUpdater:: updateProcessingNode:: Query: " + query) - val result: StatementResult = neo4JUtil.executeQuery(query) - if (null != result && result.hasNext) - logger.info(s"ObjectUpdater:: updateProcessingNode:: statement result : ${result.next().asMap()}") - } - - def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit - - def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit - - @throws[Exception] - def saveOnFailure(obj: ObjectData, messages: List[String], pkgVersion: Double)(implicit neo4JUtil: Neo4JUtil): Unit = { - val upPkgVersion = pkgVersion + 1 - val errorMessages = messages.mkString("; ") - val nodeId = obj.dbId - val query = s"""MATCH (n:domain{IL_UNIQUE_ID:"$nodeId"}) SET n.status="Failed", n.pkgVersion=$upPkgVersion, n.publishError="$errorMessages", $auditPropsUpdateQuery;""" - logger.info("ObjectUpdater:: saveOnFailure:: Query: " + query) - neo4JUtil.executeQuery(query) - } - - def metaDataQuery(obj: ObjectData)(definitionCache: DefinitionCache, config: DefinitionConfig): String = { - val version = config.supportedVersion.getOrElse(obj.dbObjType.toLowerCase(), "1.0").asInstanceOf[String] - val definition = definitionCache.getDefinition(obj.dbObjType, version, config.basePath) - val metadata = obj.metadata - ("IL_UNIQUE_ID", "identifier", "IL_FUNC_OBJECT_TYPE", "IL_SYS_NODE_TYPE", "pkgVersion", "lastStatusChangedOn", "lastUpdatedOn", "status", "objectType", "publish_type") - metadata.map(prop => { - if (null == prop._2) s"n.${prop._1}=${prop._2}" - else if (definition.objectTypeProperties.contains(prop._1)) { - prop._2 match { - case _: Map[String, AnyRef] => - val strValue = JSONUtil.serialize(ScalaJsonUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: util.Map[String, AnyRef] => - val strValue = JSONUtil.serialize(JSONUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _ => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - } - } else { - prop._2 match { - case _: Map[String, AnyRef] => - val strValue = JSONUtil.serialize(ScalaJsonUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: util.Map[String, AnyRef] => - val strValue = JSONUtil.serialize(JSONUtil.serialize(prop._2)) - s"""n.${prop._1}=$strValue""" - case _: List[String] => - val strValue = ScalaJsonUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - case _: util.List[String] => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - case _ => - val strValue = JSONUtil.serialize(prop._2) - s"""n.${prop._1}=$strValue""" - } - } - }).mkString(",") - } - - private def auditPropsUpdateQuery(): String = { - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ") - val updatedOn = sdf.format(new Date()) - s"""n.lastUpdatedOn="$updatedOn",n.lastStatusChangedOn="$updatedOn"""" - } - - def getContentBody(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): String = { - // fetch content body from cassandra - val select = QueryBuilder.select() - select.fcall("blobAsText", QueryBuilder.column("body")).as("body") - val selectWhere: Select.Where = select.from(readerConfig.keyspace, readerConfig.table).where().and(QueryBuilder.eq("content_id", identifier + ".img")) - logger.info("ObjectUpdater:: getContentBody:: Cassandra Fetch Query for image:: " + selectWhere.toString) - val row = cassandraUtil.findOne(selectWhere.toString) - if (null != row) { - val body = row.getString("body") - val updatedBody = if (isrRelativePathEnabled(config)) CSPMetaUtil.updateAbsolutePath(body) else body - updatedBody - } else { - val selectId = QueryBuilder.select() - selectId.fcall("blobAsText", QueryBuilder.column("body")).as("body") - val selectWhereId: Select.Where = selectId.from(readerConfig.keyspace, readerConfig.table).where().and(QueryBuilder.eq("content_id", identifier)) - logger.info("ObjectUpdater:: getContentBody:: Cassandra Fetch Query :: " + selectWhereId.toString) - val rowId = cassandraUtil.findOne(selectWhereId.toString) - if (null != rowId) { - val body = rowId.getString("body") - val updatedBody = if (isrRelativePathEnabled(config)) CSPMetaUtil.updateAbsolutePath(body) else body - updatedBody - } else "" - } - } - - private def isrRelativePathEnabled(config: PublishConfig): Boolean = { - config.getBoolean("cloudstorage.metadata.replace_absolute_path", false) - } - - def updateContentBody(identifier: String, ecmlBody: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val updateQuery = QueryBuilder.update(readerConfig.keyspace, readerConfig.table) - .where(QueryBuilder.eq("content_id", identifier)) - .`with`(QueryBuilder.set("body", QueryBuilder.fcall("textAsBlob", ecmlBody))) - logger.info(s"ObjectUpdater:: updateContentBody:: Updating Content Body in Cassandra For $identifier : ${updateQuery.toString}") - val result = cassandraUtil.upsert(updateQuery.toString) - if (result) logger.info(s"ObjectUpdater:: updateContentBody:: Content Body Updated Successfully For $identifier") - else { - logger.error(s"ObjectUpdater:: updateContentBody:: Content Body Update Failed For $identifier") - throw new InvalidInputException(s"Content Body Update Failed For $identifier") - } - } - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectValidator.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectValidator.scala deleted file mode 100644 index dc93315b6..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ObjectValidator.scala +++ /dev/null @@ -1,31 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.ObjectData - -import scala.collection.mutable.ListBuffer - -trait ObjectValidator { - - private[this] val logger = LoggerFactory.getLogger(classOf[ObjectValidator]) - - def validate(obj: ObjectData, identifier: String, customFn: (ObjectData, String) => List[String]): List[String] = { - validate(obj, identifier) ++ customFn(obj, identifier) - } - - def validate(obj: ObjectData, identifier: String, config: PublishConfig, customFn: (ObjectData, String, PublishConfig) => List[String]): List[String] = { - validate(obj, identifier) ++ customFn(obj, identifier, config) - } - - def validate(obj: ObjectData, identifier: String): List[String] = { - logger.info("Validating object with id: " + obj.identifier) - val messages = ListBuffer[String]() - if (obj.metadata.isEmpty) messages += s"""There is no metadata available for : $identifier""" - if (obj.metadata.get("mimeType").isEmpty) messages += s"""There is no mimeType defined for : $identifier""" - if (obj.metadata.get("primaryCategory").isEmpty) messages += s"""There is no primaryCategory defined for : $identifier""" - if (obj.metadata.get("name").isEmpty) messages += s"""There is no name defined for : $identifier""" - if (obj.metadata.get("code").isEmpty) messages += s"""There is no code defined for : $identifier""" - messages.toList - } -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/QuestionPdfGenerator.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/QuestionPdfGenerator.scala deleted file mode 100644 index 6e384feb6..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/QuestionPdfGenerator.scala +++ /dev/null @@ -1,159 +0,0 @@ -package org.sunbird.job.publish.helpers - -import com.google.gson.Gson -import org.slf4j.LoggerFactory -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.publish.handler.{QuestionHandlerFactory, QuestionTypeHandler} -import org.sunbird.job.util.{CloudStorageUtil, FileUtils, HttpUtil, Slug} - -import java.io.{BufferedWriter, File, FileWriter} - -trait QuestionPdfGenerator extends ObjectTemplateGenerator { - private[this] val logger = LoggerFactory.getLogger(classOf[QuestionPdfGenerator]) - lazy private val gson = new Gson() - - def getPdfFileUrl(objList: List[ObjectData], obj: ObjectData, templateName: String, baseUrl: String, fileNameSuffix: String)(implicit httpUtil: HttpUtil, cloudStorageUtil: CloudStorageUtil): (Option[String], Option[String]) = { - val previewUrl: Option[String] = getPreviewFileUrl(objList, obj, templateName, fileNameSuffix) - logger.info(s"QuestionPdfGenerator ::: preview url (Html File Url) for ${obj.identifier} is : ${previewUrl.getOrElse("")}") - val pdfFileUrl = convertFileToPdfUrl(previewUrl, baseUrl) - logger.info(s"QuestionPdfGenerator ::: pdf file local path for ${obj.identifier} is : ${pdfFileUrl.getOrElse("")}") - pdfFileUrl match { - case Some(url: String) => (uploadFileString(url, obj, fileNameSuffix), previewUrl) - case _ => (None, previewUrl) - } - } - - def getPreviewFileUrl(objList: List[ObjectData], obj: ObjectData, templateName: String, fileNameSuffix: String)(implicit cloudStorageUtil: CloudStorageUtil): Option[String] = { - val fileContent: String = getFileString(objList, obj.getString("name", ""), templateName).getOrElse("") - val fileName: String = s"/tmp/${obj.identifier}_${getHtmlFileSuffix(fileNameSuffix)}" - val file: Option[File] = writeFile(fileName, fileContent) - uploadFile(file, obj) - } - - def getFileString(objList: List[ObjectData], fileTitle: String, templateName: String, - customGenerator: (List[ObjectData], String, String) => Option[String] = getHtmlString): Option[String] = { - logger.info("Generating File string for objects") - customGenerator(objList, fileTitle, templateName) - } - - private def getHtmlFileSuffix(fileNameSuffix: String): String = "html_" + fileNameSuffix + ".html" - - def writeFile(filename: String, content: String): Option[File] = { - try { - logger.info(s"Writing to file with name $filename") - val file = new File(filename) - val bw = new BufferedWriter(new FileWriter(file)) - bw.write(content) - bw.close() - Some(file) - } catch { - case e: Exception => - logger.error("Error occurred while writing to file", e) - None - } - } - - def convertFileToPdfUrl(fileString: Option[String], baseUrl: String, - customConverter: (String, HttpUtil, String) => Option[String] = convertHtmlToPDF)(implicit httpUtil: HttpUtil): Option[String] = { - fileString match { - case Some(content: String) => customConverter(content, httpUtil, baseUrl) - case _ => - logger.error("Error occurred while converting file, File cannot be empty") - None - } - } - - def getHtmlString(questions: List[ObjectData], title: String, templateName: String): Option[String] = { - val questionsDataMap = populateQuestionsData(questions) - generateHtmlString(questionsDataMap, title, templateName) match { - case "" => None - case x: String => Some(x) - } - } - - //Will require index (Need to be got from question set) - private def populateQuestionsData(questions: List[ObjectData]): Map[String, AnyRef] = { - questions.map(question => question.dbId -> { - val handlerOption = QuestionHandlerFactory.apply(question.metadata.get("primaryCategory").asInstanceOf[Option[String]]) - handlerOption match { - case Some(handler: QuestionTypeHandler) => - Map("question" -> handler.getQuestion(question.extData), - "answer" -> handler.getAnswers(question.extData), - "index" -> question.metadata.getOrElse("index", 0.asInstanceOf[AnyRef])) - case _ => Map() - } - }).toMap - } - - private def generateHtmlString(questionsMap: Map[String, AnyRef], title: String, templateName: String): String = { - val filteredQMap = questionsMap.filter(entry => entry._2.asInstanceOf[Map[String, AnyRef]].nonEmpty) - if(filteredQMap.isEmpty) return "" - val questionString: String = questionsMap.map(entry => - s""" - |
- |
- |${entry._2.asInstanceOf[Map[String, AnyRef]].getOrElse("index", 0.asInstanceOf[AnyRef])}.  - |
- |${entry._2.asInstanceOf[Map[String, AnyRef]].getOrElse("question", "")} - |
- """.stripMargin - ).reduce(_ + _) - val answerString: String = questionsMap.map(entry => - s""" - |
- |
- |${entry._2.asInstanceOf[Map[String, AnyRef]].getOrElse("index", 0.asInstanceOf[AnyRef])}.  - |
- |${entry._2.asInstanceOf[Map[String, AnyRef]].getOrElse("question", "")} - |
- |
- |${entry._2.asInstanceOf[Map[String, AnyRef]].getOrElse("answer", List()).asInstanceOf[List[String]].mkString(", ")} - |
- """.stripMargin - ).reduce(_ + _) - val velocityContext: Map[String, AnyRef] = Map("questions" -> questionString, "answers" -> answerString, "title" -> title) - handleHtmlTemplate(templateName, velocityContext) - } - - //TODO: Remove hardcoded print-service url - private def convertHtmlToPDF(htmlFileUrl: String, httpUtil: HttpUtil, baseUrl: String): Option[String] = { - val response = httpUtil.post(s"$baseUrl/v1/print/preview/generate?fileUrl=$htmlFileUrl", "") - if (response.status == 200) { - val responseBody = gson.fromJson(response.body, classOf[java.util.Map[String, AnyRef]]) - val result = responseBody.getOrDefault("result", new java.util.HashMap[String, AnyRef]()).asInstanceOf[java.util.Map[String, AnyRef]] - val pdfUrl = result.getOrDefault("pdfUrl", "").asInstanceOf[String] - logger.info("QuestionPdfGenerator ::: pdf file url generated by print service : " + pdfUrl) - if (pdfUrl.isEmpty) None else Some(pdfUrl) - } else if (response.status == 400) { - logger.error("Client Error during Generate Question Set previewUrl: " + response.status) - None - } else { - logger.error("Server Error during Generate Question Set previewUrl: " + response.status) - None - } - } - - private def uploadFile(fileOption: Option[File], obj: ObjectData)(implicit cloudStorageUtil: CloudStorageUtil): Option[String] = { - fileOption match { - case Some(file: File) => - val folder = "questionset" + File.separator + obj.identifier - val urlArray: Array[String] = cloudStorageUtil.uploadFile(folder, file, Some(true)) - Some(urlArray(1)) - case _ => None - } - } - - private def uploadFileString(fileUrl: String, obj: ObjectData, fileNameSuffix: String)(implicit cloudStorageUtil: CloudStorageUtil): Option[String] = { - //Todo: Rename Status? - val fileName = s"${obj.identifier}_pdf_$fileNameSuffix.pdf" - FileUtils.copyURLToFile(obj.identifier, fileUrl, fileName) match { - case Some(file: File) => - val folder = "questionset" + File.separator + Slug.makeSlug(obj.identifier, isTransliterate = true) - val urlArray: Array[String] = cloudStorageUtil.uploadFile(folder, file, Some(true)) - Some(urlArray(1)) - case _ => logger.error("ERR_INVALID_FILE_URL", "Please Provide Valid File Url!") - None - } - } - -} diff --git a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ThumbnailGenerator.scala b/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ThumbnailGenerator.scala deleted file mode 100644 index 876fefb36..000000000 --- a/publish-pipeline/publish-core/src/main/scala/org/sunbird/job/publish/helpers/ThumbnailGenerator.scala +++ /dev/null @@ -1,104 +0,0 @@ -package org.sunbird.job.publish.helpers - -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils -import org.imgscalr.Scalr -import org.slf4j.LoggerFactory -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.util.{CloudStorageUtil, FileUtils, Slug} - -import java.awt.image.BufferedImage -import java.io.File -import java.net.{HttpURLConnection, URL} -import javax.imageio.ImageIO - -trait ThumbnailGenerator { - - private[this] val logger = LoggerFactory.getLogger(classOf[ThumbnailGenerator]) - private val THUMBNAIL_SIZE = 56 - private val ARTIFACT_FOLDER = "artifact" - - def generateThumbnail(obj: ObjectData)(implicit cloudStorageUtil: CloudStorageUtil): Option[ObjectData] = { - val appIcon: String = obj.metadata.getOrElse("appIcon", "").asInstanceOf[String] - if (StringUtils.isBlank(appIcon)) return None - val fileName = getFileName(appIcon) - val suffix = if (StringUtils.isNotBlank(fileName)) Slug.makeSlug(fileName, true) else Slug.makeSlug(FilenameUtils.getName(appIcon), false) - try { - FileUtils.copyURLToFile(obj.identifier, appIcon, suffix) match { - case Some(file: File) => { - logger.info("downloaded file path ::: " + file.getAbsolutePath) - val outFile: Option[File] = generateOutFile(file) - outFile match { - case Some(file: File) => { - val urlArray: Array[String] = cloudStorageUtil.uploadFile(getUploadFolderName(obj.identifier, ARTIFACT_FOLDER, obj.dbObjType.toLowerCase.replaceAll("image", "")), file, Some(true)) - Some(new ObjectData(obj.identifier, obj.metadata ++ Map("appIcon" -> urlArray(1), "posterImage" -> appIcon), obj.extData, obj.hierarchy)) - } - case _ => { - logger.error("Thubnail Could Not Be Generated For :" + obj.identifier) - None - } - } - } - case _ => logger.error("ERR_INVALID_FILE_URL", "Please Provide Valid File Url!") - None - } - } finally { - org.apache.commons.io.FileUtils.deleteDirectory(new File(s"/tmp/${obj.identifier}")) - } - } - - def generateOutFile(inFile: File): Option[File] = { - if (inFile != null) { - try { - val srcImage = ImageIO.read(inFile) - if ((srcImage.getHeight > THUMBNAIL_SIZE) || (srcImage.getWidth > THUMBNAIL_SIZE)) { - val scaledImage: BufferedImage = Scalr.resize(srcImage, THUMBNAIL_SIZE) - val thumbFile = getThumbnailFileName(inFile) - val outFile = new File(thumbFile) - ImageIO.write(scaledImage, "png", outFile) - Some(outFile) - } else { - None - } - } catch { - case ex: Exception => - logger.error("ERR_INVALID_FILE_URL", "Please Provide Valid File Url!") - None - } - } else { - None - } - } - - def getThumbnailFileName(input: File): String = { - val outputFileName = input.getName.replaceAll("\\.", "\\.thumb\\.") - val outputFolder = input.getParent - outputFolder + "/" + outputFileName - } - - protected def getUploadFolderName(identifier: String, folder: String, objectType: String): String = { - objectType.toLowerCase() + File.separator + Slug.makeSlug(identifier, true) + "/" + folder - } - - def getFileName(fileUrl: String): String = { - try { - val url: URL = new URL(fileUrl) - val httpConn: HttpURLConnection = url.openConnection.asInstanceOf[HttpURLConnection] - if (null != httpConn && httpConn.getResponseCode == HttpURLConnection.HTTP_OK) { - val disposition = httpConn.getHeaderField("Content-Disposition") - if (null != httpConn) - httpConn.disconnect() - if (StringUtils.isNotBlank(disposition)) { - val index = disposition.indexOf("filename=") - if (index > 0) disposition.substring(index + 10, disposition.indexOf("\"", index + 10)) else "" - } else "" - } else "" - } catch { - case e: Exception => { - e.printStackTrace() - "" - } - } - } - -} diff --git a/publish-pipeline/publish-core/src/test/resources/questionSetTemplate.vm b/publish-pipeline/publish-core/src/test/resources/questionSetTemplate.vm deleted file mode 100644 index baf0b5d37..000000000 --- a/publish-pipeline/publish-core/src/test/resources/questionSetTemplate.vm +++ /dev/null @@ -1,76 +0,0 @@ -
- -
- -
-
-

$title

-
$questions
-
-
-

Answers

-
$answers
-
- -
\ No newline at end of file diff --git a/publish-pipeline/publish-core/src/test/resources/test.conf b/publish-pipeline/publish-core/src/test/resources/test.conf deleted file mode 100644 index d8264900f..000000000 --- a/publish-pipeline/publish-core/src/test/resources/test.conf +++ /dev/null @@ -1,40 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.learning.job.request" - post_publish.topic = "sunbirddev.content.postpublish.request" - groupId = "local-questionset-publish-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - router.parallelism = 1 -} - -question { - keyspace = "dev_question_store" - table = "question_data" -} - -questionset { - keyspace = "dev_hierarchy_store" - table = "questionset_hierarchy" -} - -neo4j { - routePath = "bolt://localhost:7687" - graph = "domain" -} - -//cloud_storage_type="azure" -//azure_storage_key="dev" -//azure_storage_container="dev" -//azure_storage_container="dummy_secret" -//aws_storage_key="dev" -//aws_storage_secret="dummy_secret" -//aws_storage_container="dev" \ No newline at end of file diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala deleted file mode 100644 index e62da72e5..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala +++ /dev/null @@ -1,55 +0,0 @@ -package org.sunbird.job.publish.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.mockito.Mockito -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar.mock -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.util.{CloudStorageUtil, Neo4JUtil, ScalaJsonUtil} -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarGenerator - -import scala.collection.JavaConverters._ -import scala.concurrent.ExecutionContext.Implicits.global - -class EcarGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - val definitionBasePath: String = if (config.hasPath("schema.basePath")) config.getString("schema.basePath") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - val schemaSupportVersionMap = if (config.hasPath("schema.supportedVersion")) config.getObject("schema.supportedVersion").unwrapped().asScala.toMap else Map[String, AnyRef]() - implicit val defCache = new DefinitionCache() - implicit val defConfig = DefinitionConfig(schemaSupportVersionMap, definitionBasePath) - - "Object Ecar Generator generateEcar" should "return a Map containing Packaging Type and its url after uploading it to cloud" in { - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question"))) - val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - val obj = new TestEcarGenerator() - val result = obj.generateEcar(objData,List("SPINE")) - result.isEmpty should be(false) - } -} - -class TestEcarGenerator extends EcarGenerator { - val media = Map( - "id" -> "do_1127129497561497601326", - "type" -> "image", - "src" -> "somepath/sunbird_1551961194254.jpeg", - "baseUrl" -> "some_base_url" - ) - val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")) - override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = Some(testObj) -} diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/FrameworkDataEnrichmentTestSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/FrameworkDataEnrichmentTestSpec.scala deleted file mode 100644 index f5476e8dc..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/FrameworkDataEnrichmentTestSpec.scala +++ /dev/null @@ -1,140 +0,0 @@ -package org.sunbird.job.publish.spec - -import java.util - -import org.mockito.Mockito.when -import org.mockito.{ArgumentMatchers, Mockito} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.cache.local.FrameworkMasterCategoryMap -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.publish.helpers.FrameworkDataEnrichment -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} - -import scala.collection.JavaConverters._ - -class FrameworkDataEnrichmentTestSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val mockCassandraUtil: CassandraUtil = mock[CassandraUtil](Mockito.withSettings().serializable()) - implicit val mockConfig: PublishConfig = mock[PublishConfig](Mockito.withSettings().serializable()) - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "enrichFrameworkData" should "enrich the framework metadata" in { - enrichFrameworkMasterCategoryMap() - - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "IL_UNIQUE_ID" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "framework" -> "NCF", "targetFWIds" -> List("TPD", "NCFCOPY","NCF").asJava, "mediumIds" -> List("ncf_medium_telugu").asJava, "targetMediumIds" -> List("ncf_medium_english").asJava, "boardIds" -> List("ncf_board_cbse").asJava, "targetBoardIds" -> List("ncfcopy_board_ncert").asJava)) - when(mockNeo4JUtil.getNodesName(ArgumentMatchers.anyObject())).thenReturn(Map[String, String]("ncf_medium_telugu" -> "Telugu", "ncf_medium_english" -> "English", "ncf_board_cbse" -> "CBSE", "ncfcopy_board_ncert" -> "NCERT")) - when(mockConfig.getString("master.category.validation.enabled", "Yes")).thenReturn("Yes") - val obj = new TestFrameworkDataEnrichment() - val result = obj.enrichFrameworkData(data) - result.metadata.getOrElse("se_mediumIds", List()).asInstanceOf[List[String]] should have length(2) - result.metadata.getOrElse("se_mediumIds", List()).asInstanceOf[List[String]].contains("ncf_medium_telugu") should be (true) - result.metadata.getOrElse("se_boardIds", List()).asInstanceOf[List[String]] should have length(2) - result.metadata.getOrElse("se_boardIds", List()).asInstanceOf[List[String]].contains("ncf_board_cbse") should be (true) - result.metadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]] should have length(3) - result.metadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]].contains("TPD") should be (true) - result.metadata.getOrElse("se_mediums", List()).asInstanceOf[List[String]] should have length(2) - result.metadata.getOrElse("se_mediums", List()).asInstanceOf[List[String]].contains("English") should be (true) - result.metadata.getOrElse("se_boards", List()).asInstanceOf[List[String]] should have length(2) - result.metadata.getOrElse("se_boards", List()).asInstanceOf[List[String]].contains("CBSE") should be (true) - FrameworkMasterCategoryMap.put("masterCategories", null) - } - - "enrichFrameworkData with only targetFramework" should "enrich only se_FWIds" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "IL_UNIQUE_ID" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "framework" -> "NCF", "targetFWIds" -> List("TPD", "NCFCOPY").asJava)) - when(mockNeo4JUtil.getNodesName(ArgumentMatchers.anyObject())).thenReturn(Map[String, String]("ncf_medium_telugu" -> "Telugu", "ncf_medium_english" -> "English", "ncf_board_cbse" -> "CBSE", "ncfcopy_board_ncert" -> "NCERT")) - val obj = new TestFrameworkDataEnrichment() - val result = obj.enrichFrameworkData(data) - result.metadata.contains("se_mediumIds") should be (false) - result.metadata.contains("se_boardIds") should be (false) - result.metadata.contains("se_FWIds") should be (true) - result.metadata.contains("se_mediums") should be (false) - result.metadata.contains("se_boards") should be (false) - result.metadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]] should have length(3) - result.metadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]].contains("TPD") should be (true) - } - - "enrichFrameworkData with board, medium, gradeLevel and subject " should "enrich se_boards, se_mediums, se_gradeLevels and se_subjects" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "IL_UNIQUE_ID" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "framework" -> "NCF", "board" -> "some board", "medium" -> List("some medium 1", "some_medium_2").asJava)) - when(mockNeo4JUtil.getNodesName(ArgumentMatchers.anyObject())).thenReturn(Map[String, String]("ncf_medium_telugu" -> "Telugu", "ncf_medium_english" -> "English", "ncf_board_cbse" -> "CBSE", "ncfcopy_board_ncert" -> "NCERT")) - val obj = new TestFrameworkDataEnrichment() - val result = obj.enrichFrameworkData(data) - result.metadata.contains("se_mediumIds") should be (false) - result.metadata.contains("se_boardIds") should be (false) - result.metadata.contains("se_FWIds") should be (true) - result.metadata.contains("se_boards") should be (true) - result.metadata.getOrElse("se_boards", List()).asInstanceOf[List[String]] should have length(1) - result.metadata.getOrElse("se_boards", List()).asInstanceOf[List[String]].contains("some board") should be (true) - result.metadata.contains("se_mediums") should be (true) - result.metadata.getOrElse("se_mediums", List()).asInstanceOf[List[String]] should have length(2) - result.metadata.getOrElse("se_mediums", List()).asInstanceOf[List[String]].contains("some medium 1") should be (true) - result.metadata.contains("se_gradeLevels") should be (false) - result.metadata.contains("se_subjects") should be (false) - result.metadata.contains("se_topics") should be (false) - result.metadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]] should have length(1) - result.metadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]].contains("NCF") should be (true) - } - - "getFrameworkCategoryMetadata from database" should "return touple values" in { - when(mockNeo4JUtil.getNodePropertiesWithObjectType(ArgumentMatchers.anyString())).thenReturn(getNeo4jData()) - //enrichFrameworkMasterCategoryMap() - val node : (List[String], Map[(String, String), List[String]]) = new TestFrameworkDataEnrichment().getFrameworkCategoryMetadata("domain", "Category") - node._1.asInstanceOf[List[String]] should have length(6) - node._2.asInstanceOf[Map[(String, String), List[String]]].size.equals(3) should be (true) - } - - "getFrameworkCategoryMetadata from local cache" should "return touple values" in { - //when(mockNeo4JUtil.getNodePropertiesWithObjectType(ArgumentMatchers.anyString())).thenReturn(getNeo4jData()) - enrichFrameworkMasterCategoryMap() - val node : (List[String], Map[(String, String), List[String]]) = new TestFrameworkDataEnrichment().getFrameworkCategoryMetadata("domain", "Category") - node._1.asInstanceOf[List[String]] should have length(6) - node._2.asInstanceOf[Map[(String, String), List[String]]].size.equals(3) should be (true) - FrameworkMasterCategoryMap.put("masterCategories", null) - } - - def getNeo4jData(): util.List[util.Map[String, AnyRef]] = { - util.Arrays.asList( - getCategoryNodeMap("board", "Category", "boardIds", "targetBoardIds", "se_boardIds", "se_boards"), - getCategoryNodeMap("subject", "Category", "subjectIds", "targetSubjectIds", "se_subjectIds", "se_subjects"), - getCategoryNodeMap("medium", "Category", "mediumIds", "targetMediumIds", "se_mediumIds", "se_mediums") - ) - } - def getCategoryNodeMap(identifier: String, objectType: String, orgIdFieldName: String, targetIdFieldName: String, searchIdFieldName: String, searchLabelFieldName: String): util.Map[String, AnyRef] = { - new util.HashMap[String, AnyRef]{{ - put("IL_UNIQUE_ID", identifier) - put("IL_FUNC_OBJECT_TYPE", objectType) - put("code", identifier) - put("orgIdFieldName", orgIdFieldName) - put("targetIdFieldName", targetIdFieldName) - put("searchIdFieldName", searchIdFieldName) - put("searchLabelFieldName", searchLabelFieldName) - }} - } - - def enrichFrameworkMasterCategoryMap() = { - val masterCategoriesNode: util.List[util.Map[String, AnyRef]] = getNeo4jData() - - val masterCategoryMap: List[Map[String, AnyRef]] = masterCategoriesNode.asScala.map(node => - Map("code" -> node.getOrDefault("code", "").asInstanceOf[String], - "orgIdFieldName" -> node.getOrDefault("orgIdFieldName", "").asInstanceOf[String], - "targetIdFieldName" -> node.getOrDefault("targetIdFieldName", "").asInstanceOf[String], - "searchIdFieldName" -> node.getOrDefault("searchIdFieldName", "").asInstanceOf[String], - "searchLabelFieldName" -> node.getOrDefault("searchLabelFieldName", "").asInstanceOf[String]) - ).toList - val masterCategories: Map[String, AnyRef] = masterCategoryMap.flatMap(masterCategory => Map(masterCategory.getOrElse("code", "").asInstanceOf[String] -> masterCategory)).toMap - FrameworkMasterCategoryMap.put("masterCategories", masterCategories) - } -} - -class TestFrameworkDataEnrichment extends FrameworkDataEnrichment { - -} diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectBundleSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectBundleSpec.scala deleted file mode 100644 index 86860af08..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectBundleSpec.scala +++ /dev/null @@ -1,109 +0,0 @@ -package org.sunbird.job.publish.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.commons.lang3.StringUtils -import org.mockito.Mockito -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.scalatestplus.mockito.MockitoSugar.mock -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ObjectData} -import org.sunbird.job.publish.helpers.{EcarPackageType, ObjectBundle} -import org.sunbird.job.util.{HttpUtil, Neo4JUtil} - -import scala.collection.JavaConverters._ -import scala.concurrent.ExecutionContextExecutor - -class ObjectBundleSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val httpUtil = new HttpUtil - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - // implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val ec: ExecutionContextExecutor = ExecutionContexts.global - val definitionBasePath: String = if (config.hasPath("schema.basePath")) config.getString("schema.basePath") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - val schemaSupportVersionMap = if (config.hasPath("schema.supportedVersion")) config.getObject("schema.supportedVersion").unwrapped().asScala.toMap else Map[String, AnyRef]() - implicit val defCache = new DefinitionCache() - implicit val defConfig = DefinitionConfig(schemaSupportVersionMap, definitionBasePath) - - "validUrl" should "return true for valid url input" in { - val obj = new TestObjectBundle - val result = obj.validUrl("https://sunbirddev.blob.core.windows.net/sunbird-content-dev/ecar_files/do_113105564164997120111/1-vsa-qts-2_1603203738131_do_113105564164997120111_1.0_spine.ecar") - result should be(true) - } - - "validUrl" should "return false for invalid url input" in { - val obj = new TestObjectBundle - val result = obj.validUrl("http:// abc.com") - result should be(false) - } - - "isOnline" should "return true for valid input having contentDisposition online-only" in { - val obj = new TestObjectBundle - obj.isOnline("video/mp4", "online-only") should be(true) - } - - "isOnline" should "return true for valid input having mimeType video/youtube" in { - val obj = new TestObjectBundle - obj.isOnline("video/youtube", "inline") should be(true) - } - - "isOnline" should "return false for invalid input" in { - val obj = new TestObjectBundle - obj.isOnline("application/vnd", "inline") should be(false) - } - - "getBundleFileName" should "return file name with less than 250 characters" in { - val maxAllowedContentName = publishConfig.getInt("max_allowed_content_name", 120) - val obj = new TestObjectBundle - val ecarName = obj.getBundleFileName("do_3133687719874150401162", Map("name" -> "6.10_ମାଂସପେଶୀ, ରକ୍ତ ସଂଚାଳନ ଓ ଶ୍ୱସନ ସଂସ୍ଥାନ ଉପରେ ଶାରୀରିକ କାର୍ଯ୍ୟ, ଖେଳ, କ୍ରୀଡା ଓ ଯୋଗର ପ୍ରଭାବ-eng_effects_of_physical_activities_games_sports_and_yoga_on_muscular_circulatory_and_respiratory_systems"), "SPINE") - (ecarName.length < (maxAllowedContentName + 65)) should be(true) - } - - "getBundleFileName" should "return file name" in { - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - val maxAllowedContentName = publishConfig.getInt("max_allowed_content_name", 120) - val obj = new TestObjectBundle - val ecarName = obj.getBundleFileName("do_3133687719874150401162", Map("name" -> "Sr.18\\sst\\Grade6%_/@#!-+=()\",?&6.10_ମାଂସପେଶୀ, ରକ୍ତ ସଂଚାଳନ ଓ ଶ୍ୱସନ ସଂସ୍ଥାନ ଉପରେ ଶାରୀରିକ କାର୍ଯ୍ୟ, ଖେଳ, କ୍ରୀଡା ଓ ଯୋଗର ପ୍ରଭାବ-eng_effects_of_physical_activities_games_sports_and_yoga_on_muscular_circulatory_and_respiratory_systems"), "SPINE") - (ecarName.length < (maxAllowedContentName + 65)) should be(true) - (StringUtils.contains(ecarName, "sr.18sstgrade6_-6.10_maanspeshii-rkt-sncaalln-o-shsn-snsthaan-upre-shaariirik-kaaryyy-khell-kriiddaa-o-yogr-prbhaa")) should be(true) - } - - "getHierarchyFile" should "return valid file" in { - val obj = new TestObjectBundle - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - val file = obj.getHierarchyFile(objData, "/tmp/data").get - null != file should be(true) - file.exists() should be(true) - StringUtils.endsWith(file.getName, "hierarchy.json") should be(true) - } - - "getManifestFile" should "return valid file" in { - val obj = new TestObjectBundle - val objList = List(Map("identifier" -> "do_123", "name" -> "Test QuestionSet", "status" -> "Live", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"))), Map("identifier" -> "do_234", "name" -> "Children-1", "status" -> "Live", "children" -> List(Map("identifier" -> "do_345", "name" -> "Children-2")))) - val file = obj.getManifestFile("do_123", "QuestionSet", "/tmp/data", objList) - null != file should be(true) - file.exists() should be(true) - StringUtils.endsWith(file.getName, "manifest.json") should be(true) - } - - "getObjectBundle" should " throw exception for invalid artifactUrl " in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef]), Some(Map[String, AnyRef]("appIcon" -> "someUrl"))) - val obj = new TestObjectBundle - val objList = List(Map[String, AnyRef]("publish_type" -> "public", "mediaType" -> "content", "name" -> "PDF Content edited 1", "streamingUrl" -> "http://google.com/fsdfsdfsdf", "identifier" -> "do_11338246158784921611", "primaryCategory" -> "Explanation Content", "framework" -> "NCF", "versionKey" -> "1633945983008", "mimeType" -> "application/pdf", "license" -> "CC BY 4.0", "contentType" -> "ClassroomTeachingVideo", "objectType" -> "Content", "status" -> "Draft", "lastPublishedBy" -> "anil", "contentDisposition" -> "inline", "previewUrl" -> "http://google.com/fsdfsdfsdf", "artifactUrl" -> "http://google.com/fsdfsdfsdf", "visibility" -> "Default")) - - assertThrows[InvalidInputException] { - obj.getObjectBundle(data, objList, EcarPackageType.FULL) - } - } -} - -class TestObjectBundle extends ObjectBundle { -} diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala deleted file mode 100644 index d66422709..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala +++ /dev/null @@ -1,60 +0,0 @@ -package org.sunbird.job.publish.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.ObjectEnrichment -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, Neo4JUtil} - -class ObjectEnrichmentSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val mockCassandraUtil: CassandraUtil = mock[CassandraUtil](Mockito.withSettings().serializable()) - implicit val readerConfig = ExtDataConfig("test", "test") - implicit lazy val defCache: DefinitionCache = new DefinitionCache() - implicit lazy val definitionConfig: DefinitionConfig = DefinitionConfig(Map("questionset" -> "1.0"), "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local") - - "ObjectEnrichment enrichObject" should " enrich the object with Framework data and thumbnail " in { - -// when(mockNeo4JUtil.getNodesName(any[List[String]])).thenReturn(any()) - when(mockNeo4JUtil.getNodesName(List("NCERT"))).thenReturn(Map("NCERT"-> "NCERT")) - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - - val objectEnrichment = new TestObjectEnrichment() - val result = objectEnrichment.enrichObject(objData) - val resultMetadata = result.metadata - resultMetadata.isEmpty should be(false) - - resultMetadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]].isEmpty should be(false) - resultMetadata.getOrElse("se_FWIds", List()).asInstanceOf[List[String]].contains("NCERT") should be(true) - resultMetadata.getOrElse("se_boardIds", List()).asInstanceOf[List[String]].isEmpty should be(true) - - resultMetadata.getOrElse("posterImage", "").asInstanceOf[String].isEmpty should be(false) - resultMetadata.getOrElse("appIcon", "").asInstanceOf[String].isEmpty should be(false) - } -} - -class TestObjectEnrichment extends ObjectEnrichment { - override def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig): Option[ObjectData] = None -} - diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectPdfGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectPdfGeneratorSpec.scala deleted file mode 100644 index 106b106d6..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectPdfGeneratorSpec.scala +++ /dev/null @@ -1,218 +0,0 @@ -package org.sunbird.job.publish.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.publish.helpers.QuestionPdfGenerator -import org.sunbird.job.util.{CloudStorageUtil, HTTPResponse, HttpUtil} - - -class ObjectPdfGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - implicit val mockHttpUtil: HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - - - "Object Pdf Generator getPreviewFileUrl" should "return a url of the html file after uploading it to cloud" in { - val pdfGenerator = new TestQuestionPdfGenerator() - val fileNameSuffix = System.currentTimeMillis().toString - val obj = pdfGenerator.getPreviewFileUrl(getObjectList(), getObject(), "questionSetTemplate.vm", fileNameSuffix) - obj.getOrElse("").isEmpty should be(false) - } - - "Object PDF generator getPdfFileUrl" should "return a url of the pdf file after uploading it to cloud" in { - val fileNameSuffix = System.currentTimeMillis().toString - when(mockHttpUtil.post(s"http://11.2.6.6/print/v1/print/preview/generate?fileUrl=https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_xyz/do_xyz_html_${fileNameSuffix}.html", "")).thenReturn(getHttpResponse()) - // when(mockHttpUtil.post(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.any())).thenReturn(getHttpResponse()) - val pdfGenerator = new TestQuestionPdfGenerator() - val (pdfUrl, previewUrl) = pdfGenerator.getPdfFileUrl(getObjectList(), getObject(), "questionSetTemplate.vm", "http://11.2.6.6/print", fileNameSuffix) - pdfUrl.getOrElse("").isEmpty should be(false) - previewUrl.getOrElse("").isEmpty should be(false) - - } - - "Object PDF generator getPdfFileUrl" should "return a Empty pdfUrl if failed to convert to PDF " in { - val fileNameSuffix = System.currentTimeMillis().toString - when(mockHttpUtil.post(s"http://11.2.6.6/print/v1/print/preview/generate?fileUrl=https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_xyz/do_xyz_html_${fileNameSuffix}.html", "")).thenReturn(getFailedHttpResponse()) - // when(mockHttpUtil.post(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.any())).thenReturn(getFailedHttpResponse()) - val pdfGenerator = new TestQuestionPdfGenerator() - val (pdfUrl, previewUrl) = pdfGenerator.getPdfFileUrl(getObjectList(), getObject(), "questionSetTemplate.vm", "http://11.2.6.6/print", fileNameSuffix) - pdfUrl.getOrElse("").isEmpty should be(true) - previewUrl.getOrElse("").isEmpty should be(false) - } - - private def getObjectList(): List[ObjectData] = { - val question_1 = new ObjectData("do_123", Map("primaryCategory" -> "Multiple Choice Question", "index" -> 1.asInstanceOf[AnyRef], "IL_UNIQUE_ID" -> "do_123"), - Some(Map( - "responseDeclaration" -> - """ - |{ - | "response1": { - | "maxScore": 1, - | "cardinality": "single", - | "type": "integer", - | "correctResponse": { - | "value": 1, - | "outcomes": { - | "score": 1 - | } - | } - | } - | } - """.stripMargin, - "interactions" -> - """ - |{ - | "response1": { - | "type": "choice", - | "options": [ - | { - | "label": "

2 September 1929

", - | "value": 0 - | }, - | { - | "label": "

15 October 1931

", - | "value": 1 - | }, - | { - | "label": "

15 August 1923

", - | "value": 2 - | }, - | { - | "label": "

29 February 1936

", - | "value": 3 - | } - | ] - | } - | } - """.stripMargin, - "body" -> - """
- |
When was Dr. A.P.J. Abdul Kalam born
- |
- |
""")), None) - val question_2 = new ObjectData("do_234", Map("primaryCategory" -> "Multiple Choice Question", "index" -> 2.asInstanceOf[AnyRef], "IL_UNIQUE_ID" -> "do_234"), - Some(Map( - "responseDeclaration" -> - """ - | { - | "maxScore": 3, - | "response1": { - | "cardinality": "multiple", - | "type": "integer", - | "correctResponse": { - | "value": [ - | 0, - | 1, - | 2 - | ], - | "outcomes": { - | "score": 3 - | } - | }, - | "mapping": [ - | { - | "response": 0, - | "outcomes": { - | "score": 1 - | } - | }, - | { - | "response": 1, - | "outcomes": { - | "score": 1 - | } - | }, - | { - | "response": 2, - | "outcomes": { - | "score": 1 - | } - | } - | ] - | } - | } - """.stripMargin, - "interactions" -> - """ { - | "response1": { - | "type": "choice", - | "options": [ - | { - | "label": "

Failure to Success: Legendary Lives

", - | "value": 0 - | }, - | { - | "label": "

You Are Born to Blossom

", - | "value": 1 - | }, - | { - | "label": "

Ignited Minds

", - | "value": 2 - | }, - | { - | "label": "

A House for Mr. Biswas‎

", - | "value": 3 - | } - | ] - | } - | } - | """.stripMargin, - "body" -> - """ - |
- |
Which of the following books is written by Dr. A.P.J. Abdul Kalam
- |
- |
- """.stripMargin))) - - val question_3 = new ObjectData("do_345", Map("primaryCategory" -> "Subjective Question", "index" -> 3.asInstanceOf[AnyRef], "IL_UNIQUE_ID" -> "do_345"), - Some(Map( - "body" -> "
The tenure of APJ Abdul Kalam as Indian President
", - "answer" -> "
2002 to 2007
" - ))) - List(question_1, question_2, question_3) - } - - private def getObject(): ObjectData = { - new ObjectData("do_xyz", Map("name" -> "Test Question Set", "IL_UNIQUE_ID" -> "do_xyz")) - } - - private def getHttpResponse(): HTTPResponse = { - HTTPResponse(200, - """ - |{ - |"result" : { - | "pdfUrl" : "https://dockstorage.blob.core.windows.net/sunbird-content-dock/content/do_11304066349776076815/artifact/do_11304066349776076815_1591877926475.pdf" - |} - |}""".stripMargin) - } - - private def getFailedHttpResponse(): HTTPResponse = { - HTTPResponse(400, - """ - |{ - |"result" : { - |} - |}""".stripMargin) - } - -} - -class TestQuestionPdfGenerator extends QuestionPdfGenerator { -} diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectReaderTestSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectReaderTestSpec.scala deleted file mode 100644 index 217ed6420..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectReaderTestSpec.scala +++ /dev/null @@ -1,49 +0,0 @@ -package org.sunbird.job.publish.spec - -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{ExtDataConfig, ObjectExtData} -import org.sunbird.job.publish.helpers.ObjectReader -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} - -import scala.collection.JavaConverters._ - -class ObjectReaderTestSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val mockCassandraUtil: CassandraUtil = mock[CassandraUtil](Mockito.withSettings().serializable()) - implicit val config: PublishConfig = mock[PublishConfig](Mockito.withSettings().serializable()) - - "Object Reader " should " read the metadata " in { - when(mockNeo4JUtil.getNodeProperties("do_123.img")).thenReturn(Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123.img", "IL_UNIQUE_ID" -> "do_123.img", "pkgVersion" -> 2.0.asInstanceOf[AnyRef]).asJava) - val objectReader = new TestObjectReader() - val readerConfig = ExtDataConfig("test", "test") - val obj = objectReader.getObject("do_123", 2, "", "Public", readerConfig) - val metadata = obj.metadata.asJava - metadata.isEmpty should be(false) - obj.extData should be(None) - obj.hierarchy should be(None) - } -} - -class TestObjectReader extends ObjectReader { - - override def getExtData(identifier: String, pkgVersion: Double, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] = None - - override def getHierarchy(identifier: String, pkgVersion: Double, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = None - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = None -} diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectTemplateGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectTemplateGeneratorSpec.scala deleted file mode 100644 index 3f9220f03..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectTemplateGeneratorSpec.scala +++ /dev/null @@ -1,70 +0,0 @@ -package org.sunbird.job.publish.spec - -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.publish.helpers.ObjectTemplateGenerator - - -class ObjectTemplateGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "Object Template Generator " should "generate and return a template string" in { - val objValidator = new TestTemplateGenerator() - val htmlTemplateString = objValidator.handleHtmlTemplate("questionSetTemplate.vm", getTemplateContext) - htmlTemplateString.isEmpty should be(false) - htmlTemplateString shouldEqual("
\n \n
\n\n
\n
\n

$title

\n
\n
\n
\n1.\nWhat is the color of the sky?\n
\n
\n
\n
\n2.\nWhat is the color of the leaf?\n
\n
\n
\n
\n
\n

Answers

\n
\n |
\n |
\n |1.\n |
\n |What is the color of the sky?\n |
\n |
\n |blue\n |
\n |
\n |
\n |1.\n |
\n |What is the color of the leaf?\n |
\n |
\n |grean\n |
\n
\n
\n\n
") - } - - - def getTemplateContext = { - Map("questions" -> - """ - |
- |
- |1. - |What is the color of the sky? - |
- |
- |
- |
- |2. - |What is the color of the leaf? - |
- |
- """.stripMargin, - "answers" -> - """ - |
- |
- |1. - |
- |What is the color of the sky? - |
- |
- |blue - |
- |
- |
- |1. - |
- |What is the color of the leaf? - |
- |
- |grean - |
- """) - } - -} - -class TestTemplateGenerator extends ObjectTemplateGenerator { - - -} diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectUpdaterSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectUpdaterSpec.scala deleted file mode 100644 index 413c6a203..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectUpdaterSpec.scala +++ /dev/null @@ -1,89 +0,0 @@ -package org.sunbird.job.publish.spec - -import org.mockito.ArgumentMatchers.{any, anyString} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.ObjectUpdater -import org.sunbird.job.util.{CassandraUtil, Neo4JUtil} - -import java.util - -class ObjectUpdaterSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit val mockCassandraUtil: CassandraUtil = mock[CassandraUtil](Mockito.withSettings().serializable()) - implicit val config: PublishConfig = mock[PublishConfig](Mockito.withSettings().serializable()) - implicit val readerConfig = ExtDataConfig("test", "test") - implicit lazy val defCache: DefinitionCache = new DefinitionCache() - implicit val definitionConfig: DefinitionConfig = DefinitionConfig(Map("itemset" -> "2.0"), "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local") - - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "ObjectUpdater saveOnSuccess" should " update the status for successfully published data " in { - - when(mockNeo4JUtil.executeQuery(anyString())).thenReturn(any()) - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("objectType" -> "QuestionSet", "identifier" -> "do_123","publish_type" -> "Public", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - - val obj = new TestObjectUpdater() - obj.saveOnSuccess(objData) - } - - "ObjectUpdater updateProcessingNode" should " update the status for successfully published data " in { - - when(mockNeo4JUtil.executeQuery(anyString())).thenReturn(any()); - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("objectType" -> "QuestionSet", "identifier" -> "do_123","publish_type" -> "Public", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Sr.18\\sst\\Grade6%_/@#!-+=()\",?&6.10_ମାଂସପେଶୀ, ରକ୍ତ Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - - val obj = new TestObjectUpdater() - obj.updateProcessingNode(objData) - } - - "ObjectUpdater saveOnFailure" should " update the status for failed published data " in { - - when(mockNeo4JUtil.executeQuery(anyString())).thenReturn(any()) - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("objectType" -> "QuestionSet","identifier" -> "do_123","publish_type" -> "Public", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - - val obj = new TestObjectUpdater() - obj.saveOnFailure(objData, List("Testing Save on Publish Failure"), 1) - } - - "ObjectUpdater metaDataQuery" should " give the update query " in { - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val variants = new util.HashMap[String, String] {{ - put("spine","https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_1132380439842324481319/hindi-questionset-17_1615972827743_do_1132380439842324481319_1_SPINE.ecar") - put("online","https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_1132380439842324481319/hindi-questionset-17_1615972829357_do_1132380439842324481319_1_ONLINE.ecar") - }} - val outcomeProcessing = Map("name" -> "abc") - val metadata: Map[String, AnyRef] = Map("description" -> "Hello \"World\"", "keywords" -> List("anusha"),"channel" -> "01309282781705830427","mimeType" -> "application/vnd.sunbird.questionset","showHints" -> "No","objectType" -> "QuestionSet","primaryCategory" -> "Practice Question Set","contentEncoding" -> "gzip","showSolutions" -> "No","identifier" -> "do_1132421849134858241686","visibility" -> "Default","showTimer" -> "Yes","author" -> "anusha","childNodes" -> util.Arrays.asList("do_1132421859856875521687"),"consumerId" -> "273f3b18-5dda-4a27-984a-060c7cd398d3","version" -> 1.asInstanceOf[AnyRef],"prevState" -> "Draft","IL_FUNC_OBJECT_TYPE" -> "QuestionSet","name" -> "Timer","timeLimits" -> "{\"maxTime\":\"3541\",\"warningTime\":\"2401\"}","IL_UNIQUE_ID" -> "do_1132421849134858241686","board" -> "CBSE", "variants" -> variants, "outcomeProcessing" -> outcomeProcessing) - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - val obj = new TestObjectUpdater() - println(obj.metaDataQuery(objData)(defCache, definitionConfig)) - } -} - -class TestObjectUpdater extends ObjectUpdater { - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = None -} - diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectValidatorTestSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectValidatorTestSpec.scala deleted file mode 100644 index dc1206119..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectValidatorTestSpec.scala +++ /dev/null @@ -1,32 +0,0 @@ -package org.sunbird.job.publish.spec - -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.publish.helpers.ObjectValidator - -class ObjectValidatorTestSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "Object Validator " should " validate the object and return messages" in { - val objValidator = new TestObjectValidator() - val obj = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123.img", "IL_UNIQUE_ID" -> "do_123.img", "pkgVersion" -> 2.0.asInstanceOf[AnyRef])) - val messages = objValidator.validate(obj, "do_123") - messages should have length(3) - messages should contain ("There is no mimeType defined for : do_123") - messages should contain ("There is no primaryCategory defined for : do_123") - messages should contain ("There is no code defined for : do_123") - } - -} - -class TestObjectValidator extends ObjectValidator { - -} diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala deleted file mode 100644 index a5767806f..000000000 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala +++ /dev/null @@ -1,61 +0,0 @@ -package org.sunbird.job.publish.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.ObjectData -import org.sunbird.job.publish.helpers.ThumbnailGenerator -import org.sunbird.job.util.CloudStorageUtil - - -class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers { - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - implicit val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - - "Object Thumbnail Generator generateThumbnail" should "add the thumbnail to ObjectData" in { - - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - - val thumbnailGenerator = new TestThumbnailGenerator() - val obj = thumbnailGenerator.generateThumbnail(objData) - val result = obj.getOrElse(objData) - val resultMetadata = result.metadata - resultMetadata.isEmpty should be(false) - resultMetadata.getOrElse("posterImage", "").asInstanceOf[String].isEmpty should be(false) - resultMetadata.getOrElse("appIcon", "").asInstanceOf[String].isEmpty should be(false) - resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png" - resultMetadata.getOrElse("appIcon", "").asInstanceOf[String] shouldBe "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_123/artifact/avatar_anonymous.thumb.png" - } - - "Object Thumbnail Generator generateThumbnail with google drive link" should "add the thumbnail to ObjectData" in { - val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://drive.google.com/uc?export=download&id=1-dFzAeSNmx1ZRn77CEntyQA-VcBE0PKg", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") - val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) - val thumbnailGenerator = new TestThumbnailGenerator() - val obj = thumbnailGenerator.generateThumbnail(objData) - val result = obj.getOrElse(objData) - val resultMetadata = result.metadata - resultMetadata.isEmpty should be(false) - resultMetadata.getOrElse("posterImage", "").asInstanceOf[String].isEmpty should be(false) - resultMetadata.getOrElse("appIcon", "").asInstanceOf[String].isEmpty should be(false) - resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://drive.google.com/uc?export=download&id=1-dFzAeSNmx1ZRn77CEntyQA-VcBE0PKg" - resultMetadata.getOrElse("appIcon", "").asInstanceOf[String] shouldBe "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_123/artifact/book.thumb.jpg" - } - -} - -class TestThumbnailGenerator extends ThumbnailGenerator { - -} \ No newline at end of file diff --git a/publish-pipeline/questionset-publish/pom.xml b/publish-pipeline/questionset-publish/pom.xml deleted file mode 100644 index e233131f9..000000000 --- a/publish-pipeline/questionset-publish/pom.xml +++ /dev/null @@ -1,207 +0,0 @@ - - - - publish-pipeline - org.sunbird - 1.0 - - 4.0.0 - questionset-publish - 1.0.0 - jar - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - publish-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - jackson-module-scala_${scala.version} - com.fasterxml.jackson.module - - - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.sunbird.job.questionset.task.QuestionSetPublishStreamTask - - - - reference.conf - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - questionset-publish-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/publish-pipeline/questionset-publish/src/main/resources/log4j.properties b/publish-pipeline/questionset-publish/src/main/resources/log4j.properties deleted file mode 100644 index 02d6c8285..000000000 --- a/publish-pipeline/questionset-publish/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=-questionset-publish.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/publish-pipeline/questionset-publish/src/main/resources/questionSetTemplate.vm b/publish-pipeline/questionset-publish/src/main/resources/questionSetTemplate.vm deleted file mode 100644 index baf0b5d37..000000000 --- a/publish-pipeline/questionset-publish/src/main/resources/questionSetTemplate.vm +++ /dev/null @@ -1,76 +0,0 @@ -
- -
- -
-
-

$title

-
$questions
-
-
-

Answers

-
$answers
-
- -
\ No newline at end of file diff --git a/publish-pipeline/questionset-publish/src/main/resources/questionset-publish.conf b/publish-pipeline/questionset-publish/src/main/resources/questionset-publish.conf deleted file mode 100644 index dcd68c01d..000000000 --- a/publish-pipeline/questionset-publish/src/main/resources/questionset-publish.conf +++ /dev/null @@ -1,51 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.assessment.publish.request" - post_publish.topic = "sunbirddev.content.postpublish.request" - groupId = "local-questionset-publish-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - router.parallelism = 1 -} - -question { - keyspace = "dev_question_store" - table = "question_data" -} - -questionset { - keyspace = "dev_hierarchy_store" - table = "questionset_hierarchy" - tmp_file_location = "/tmp" - template_name = "questionSetTemplate.vm" - cloud_storage.itemset.folder = "" - cloud_storage.content.folder = "" -} - -print_service.base_url="http://11.2.6.6/print" - -redis { - database { - qsCache.id = 0 - } -} - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "CONTENT_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"] -cloudstorage.metadacloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] - -cloud_storage_type="" -cloud_storage_key="" -cloud_storage_secret="" -cloud_storage_container="" -cloud_storage_endpoint="" \ No newline at end of file diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/PublishEventRouter.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/PublishEventRouter.scala deleted file mode 100644 index 510d234c3..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/PublishEventRouter.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.sunbird.job.questionset.function - -import com.google.gson.reflect.TypeToken -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.questionset.publish.domain.{Event, PublishMetadata} -import org.sunbird.job.questionset.task.QuestionSetPublishConfig -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.lang.reflect.Type - -class PublishEventRouter(config: QuestionSetPublishConfig) extends BaseProcessFunction[Event, String](config) { - - - private[this] val logger = LoggerFactory.getLogger(classOf[PublishEventRouter]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def metricsList(): List[String] = { - List(config.skippedEventCount, config.totalEventsCount) - } - - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - if (event.validEvent()) { - logger.info("PublishEventRouter :: Event: " + event) - event.objectType match { - case "Question" | "QuestionImage" => { - logger.info("PublishEventRouter :: Sending Question For Publish Having Identifier: " + event.objectId) - context.output(config.questionPublishOutTag, PublishMetadata(event.objectId, event.objectType, event.mimeType, event.pkgVersion, event.publishType, event.lastPublishedBy)) - } - case "QuestionSet" | "QuestionSetImage" => { - logger.info("PublishEventRouter :: Sending QuestionSet For Publish Having Identifier: " + event.objectId) - context.output(config.questionSetPublishOutTag, PublishMetadata(event.objectId, event.objectType, event.mimeType, event.pkgVersion, event.publishType, event.lastPublishedBy)) - } - case _ => { - metrics.incCounter(config.skippedEventCount) - logger.info("Invalid Object Type Received For Publish.| Identifier : " + event.objectId + " , objectType : " + event.objectType) - } - } - } else { - logger.warn("Event skipped for identifier: " + event.objectId + " objectType: " + event.objectType) - metrics.incCounter(config.skippedEventCount) - } - } -} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/QuestionPublishFunction.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/QuestionPublishFunction.scala deleted file mode 100644 index 502be2ad5..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/QuestionPublishFunction.scala +++ /dev/null @@ -1,86 +0,0 @@ -package org.sunbird.job.questionset.function - -import akka.dispatch.ExecutionContexts -import com.google.gson.reflect.TypeToken -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.questionset.publish.domain.PublishMetadata -import org.sunbird.job.questionset.publish.helpers.QuestionPublisher -import org.sunbird.job.questionset.task.QuestionSetPublishConfig -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} -import java.lang.reflect.Type - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.cache.{DataCache, RedisConnect} - -import scala.concurrent.ExecutionContext - -class QuestionPublishFunction(config: QuestionSetPublishConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var definitionCache: DefinitionCache = null, - @transient var definitionConfig: DefinitionConfig = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[PublishMetadata, String](config) with QuestionPublisher { - - private[this] val logger = LoggerFactory.getLogger(classOf[QuestionPublishFunction]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - private val readerConfig = ExtDataConfig(config.questionKeyspaceName, config.questionTableName) - - @transient var ec: ExecutionContext = _ - @transient var cache: DataCache = _ - private val pkgTypes = List(EcarPackageType.FULL.toString, EcarPackageType.ONLINE.toString) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - cloudStorageUtil = new CloudStorageUtil(config) - ec = ExecutionContexts.global - definitionCache = new DefinitionCache() - definitionConfig = DefinitionConfig(config.schemaSupportVersionMap, config.definitionBasePath) - cache = new DataCache(config, new RedisConnect(config), config.cacheDbId, List()) - cache.init() - } - - override def close(): Unit = { - super.close() - cassandraUtil.close() - cache.close() - } - - override def metricsList(): List[String] = { - List(config.questionPublishEventCount, config.questionPublishSuccessEventCount, config.questionPublishFailedEventCount) - } - - override def processElement(data: PublishMetadata, context: ProcessFunction[PublishMetadata, String]#Context, metrics: Metrics): Unit = { - logger.info("Question publishing started for : " + data.identifier) - metrics.incCounter(config.questionPublishEventCount) - val objData = getObject(data.identifier, data.pkgVersion, data.mimeType, data.publishType, readerConfig)(neo4JUtil, cassandraUtil, config) - val obj = if (StringUtils.isNotBlank(data.lastPublishedBy)) { - val newMeta = objData.metadata ++ Map("lastPublishedBy" -> data.lastPublishedBy) - new ObjectData(objData.identifier, newMeta, objData.extData, objData.hierarchy) - } else objData - val messages: List[String] = validate(obj, obj.identifier, validateQuestion) - if (messages.isEmpty) { - cache.del(obj.identifier) - val enrichedObj = enrichObject(obj)(neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig) - val objWithEcar = getObjectWithEcar(enrichedObj, pkgTypes)(ec, neo4JUtil, cloudStorageUtil, config, definitionCache, definitionConfig, httpUtil) - logger.info("Ecar generation done for Question: " + objWithEcar.identifier) - saveOnSuccess(objWithEcar)(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig, config) - metrics.incCounter(config.questionPublishSuccessEventCount) - logger.info("Question publishing completed successfully for : " + data.identifier) - } else { - saveOnFailure(obj, messages, data.pkgVersion)(neo4JUtil) - metrics.incCounter(config.questionPublishFailedEventCount) - logger.info("Question publishing failed for : " + data.identifier) - } - } -} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/QuestionSetPublishFunction.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/QuestionSetPublishFunction.scala deleted file mode 100644 index da6936983..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/function/QuestionSetPublishFunction.scala +++ /dev/null @@ -1,155 +0,0 @@ -package org.sunbird.job.questionset.function - -import akka.dispatch.ExecutionContexts -import com.google.gson.reflect.TypeToken -import org.apache.commons.lang3.StringUtils -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.questionset.publish.domain.PublishMetadata -import org.sunbird.job.questionset.publish.helpers.QuestionSetPublisher -import org.sunbird.job.questionset.publish.util.QuestionPublishUtil -import org.sunbird.job.questionset.task.QuestionSetPublishConfig -import org.sunbird.job.util._ -import org.sunbird.job.{BaseProcessFunction, Metrics} -import java.lang.reflect.Type - -import org.sunbird.job.cache.{DataCache, RedisConnect} - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer -import scala.concurrent.ExecutionContext - -class QuestionSetPublishFunction(config: QuestionSetPublishConfig, httpUtil: HttpUtil, - @transient var neo4JUtil: Neo4JUtil = null, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var definitionCache: DefinitionCache = null, - @transient var definitionConfig: DefinitionConfig = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[PublishMetadata, String](config) with QuestionSetPublisher { - - private[this] val logger = LoggerFactory.getLogger(classOf[QuestionSetPublishFunction]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - val liveStatus = List("Live", "Unlisted") - - @transient var ec: ExecutionContext = _ - @transient var cache: DataCache = _ - private val pkgTypes = List(EcarPackageType.SPINE.toString, EcarPackageType.ONLINE.toString, EcarPackageType.FULL.toString) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - cloudStorageUtil = new CloudStorageUtil(config) - ec = ExecutionContexts.global - definitionCache = new DefinitionCache() - definitionConfig = DefinitionConfig(config.schemaSupportVersionMap, config.definitionBasePath) - cache = new DataCache(config, new RedisConnect(config), config.cacheDbId, List()) - cache.init() - } - - override def close(): Unit = { - super.close() - cassandraUtil.close() - cache.close() - } - - override def metricsList(): List[String] = { - List(config.questionSetPublishEventCount, config.questionSetPublishSuccessEventCount, config.questionSetPublishFailedEventCount) - } - - override def processElement(data: PublishMetadata, context: ProcessFunction[PublishMetadata, String]#Context, metrics: Metrics): Unit = { - logger.info("QuestionSet publishing started for : " + data.identifier) - metrics.incCounter(config.questionSetPublishEventCount) - val definition: ObjectDefinition = definitionCache.getDefinition(data.objectType, config.schemaSupportVersionMap.getOrElse(data.objectType.toLowerCase(), "1.0").asInstanceOf[String], config.definitionBasePath) - val readerConfig = ExtDataConfig(config.questionSetKeyspaceName, config.questionSetTableName, definition.getExternalPrimaryKey, definition.getExternalProps) - val qDef: ObjectDefinition = definitionCache.getDefinition("Question", config.schemaSupportVersionMap.getOrElse("question", "1.0").asInstanceOf[String], config.definitionBasePath) - val qReaderConfig = ExtDataConfig(config.questionKeyspaceName, qDef.getExternalTable, qDef.getExternalPrimaryKey, qDef.getExternalProps) - val objData = getObject(data.identifier, data.pkgVersion, data.mimeType, data.publishType, readerConfig)(neo4JUtil, cassandraUtil, config) - val obj = if (StringUtils.isNotBlank(data.lastPublishedBy)) { - val newMeta = objData.metadata ++ Map("lastPublishedBy" -> data.lastPublishedBy) - new ObjectData(objData.identifier, newMeta, objData.extData, objData.hierarchy) - } else objData - logger.info("processElement ::: obj metadata before publish ::: " + ScalaJsonUtil.serialize(obj.metadata)) - logger.info("processElement ::: obj hierarchy before publish ::: " + ScalaJsonUtil.serialize(obj.hierarchy.getOrElse(Map()))) - val messages: List[String] = validate(obj, obj.identifier, validateQuestionSet) - if (messages.isEmpty) { - val cacheKey = s"""qs_hierarchy_${obj.identifier}""" - cache.del(cacheKey) - cache.del(obj.identifier) - // Get all the questions from hierarchy - val qList: List[ObjectData] = getQuestions(obj, qReaderConfig)(cassandraUtil) - logger.info("processElement ::: child questions list from hierarchy ::: " + qList) - // Filter out questions having visibility parent (which need to be published) - val childQuestions: List[ObjectData] = qList.filter(q => isValidChildQuestion(q, obj.getString("createdBy", ""))) - //TODO: Remove below statement - childQuestions.foreach(ch => logger.info(s"child questions which are going to be published. identifier : ${ch.identifier} , visibility: ${ch.getString("visibility", "")} , createdBy: ${ch.getString("createdBy", "")}")) - // Publish Child Questions - QuestionPublishUtil.publishQuestions(obj.identifier, childQuestions, data.pkgVersion, data.lastPublishedBy)(ec, neo4JUtil, cassandraUtil, qReaderConfig, cloudStorageUtil, definitionCache, definitionConfig, config, httpUtil) - val pubMsgs: List[String] = isChildrenPublished(childQuestions, data.publishType, qReaderConfig) - if (pubMsgs.isEmpty) { - // Enrich Object as well as hierarchy - val enrichedObj = enrichObject(obj)(neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig) - logger.info(s"processElement ::: object enrichment done for ${obj.identifier}") - logger.info("processElement ::: obj metadata post enrichment :: " + ScalaJsonUtil.serialize(enrichedObj.metadata)) - logger.info("processElement ::: obj hierarchy post enrichment :: " + ScalaJsonUtil.serialize(enrichedObj.hierarchy.get)) - // Generate ECAR - val objWithEcar = generateECAR(enrichedObj, pkgTypes)(ec, neo4JUtil, cloudStorageUtil, config, definitionCache, definitionConfig, httpUtil) - // Generate PDF URL - val updatedObj = generatePreviewUrl(objWithEcar, qList)(httpUtil, cloudStorageUtil) - saveOnSuccess(updatedObj)(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig, config) - logger.info("QuestionSet publishing completed successfully for : " + data.identifier) - metrics.incCounter(config.questionSetPublishSuccessEventCount) - } else { - saveOnFailure(obj, pubMsgs, data.pkgVersion)(neo4JUtil) - metrics.incCounter(config.questionSetPublishFailedEventCount) - logger.info("QuestionSet publishing failed for : " + data.identifier) - } - } else { - saveOnFailure(obj, messages, data.pkgVersion)(neo4JUtil) - metrics.incCounter(config.questionSetPublishFailedEventCount) - logger.info("QuestionSet publishing failed for : " + data.identifier) - } - } - - //TODO: Implement Multiple Data Read From Neo4j and Use it here. - def isChildrenPublished(children: List[ObjectData], publishType: String, readerConfig: ExtDataConfig): List[String] = { - val messages = ListBuffer[String]() - children.foreach(q => { - val id = q.identifier.replace(".img", "") - val obj = getObject(id, 0, q.mimeType, publishType, readerConfig)(neo4JUtil, cassandraUtil, config) - logger.info(s"question metadata for $id : ${obj.metadata}") - if (!List("Live", "Unlisted").contains(obj.getString("status", ""))) { - logger.info("Question publishing failed for : " + id) - messages += s"""Question publishing failed for : $id""" - } - }) - messages.toList - } - - def generateECAR(data: ObjectData, pkgTypes: List[String])(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig, httpUtil: HttpUtil): ObjectData = { - val ecarMap: Map[String, String] = generateEcar(data, pkgTypes) - val variants: java.util.Map[String, java.util.Map[String, String]] = ecarMap.map { case (key, value) => key.toLowerCase -> Map[String, String]("ecarUrl" -> value, "size" -> httpUtil.getSize(value).toString).asJava }.asJava - logger.info("QuestionSetPublishFunction ::: generateECAR ::: ecar map ::: " + ecarMap) - val meta: Map[String, AnyRef] = Map("downloadUrl" -> ecarMap.getOrElse(EcarPackageType.FULL.toString, ""), "variants" -> variants, "size" -> httpUtil.getSize(ecarMap.getOrElse(EcarPackageType.FULL.toString, "")).asInstanceOf[AnyRef]) - new ObjectData(data.identifier, data.metadata ++ meta, data.extData, data.hierarchy) - } - - def generatePreviewUrl(data: ObjectData, qList: List[ObjectData])(implicit httpUtil: HttpUtil, cloudStorageUtil: CloudStorageUtil): ObjectData = { - val (pdfUrl, previewUrl) = getPdfFileUrl(qList, data, "questionSetTemplate.vm", config.printServiceBaseUrl, System.currentTimeMillis().toString)(httpUtil, cloudStorageUtil) - logger.info("generatePreviewUrl ::: finalPdfUrl ::: " + pdfUrl.getOrElse("")) - logger.info("generatePreviewUrl ::: finalPreviewUrl ::: " + previewUrl.getOrElse("")) - new ObjectData(data.identifier, data.metadata ++ Map("previewUrl" -> previewUrl.getOrElse(""), "pdfUrl" -> pdfUrl.getOrElse("")), data.extData, data.hierarchy) - } - - def isValidChildQuestion(obj: ObjectData, createdBy: String): Boolean = { - StringUtils.equalsIgnoreCase("Parent", obj.getString("visibility", "")) || (StringUtils.equalsIgnoreCase("Default", obj.getString("visibility", "")) && !liveStatus.contains(obj.getString("status", "")) && StringUtils.equalsIgnoreCase(createdBy, obj.getString("createdBy", ""))) - } - -} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/domain/Event.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/domain/Event.scala deleted file mode 100644 index d71ec93d1..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/domain/Event.scala +++ /dev/null @@ -1,38 +0,0 @@ -package org.sunbird.job.questionset.publish.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest - -import java.util -import scala.collection.JavaConverters._ - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val jobName = "questionset-publish" - - private val objectTypes = List("Question", "QuestionImage", "QuestionSet", "QuestionSetImage") - private val mimeTypes = List("application/vnd.sunbird.question", "application/vnd.sunbird.questionset") - - def eData: Map[String, AnyRef] = readOrDefault("edata", new util.HashMap[String, AnyRef]()).asScala.toMap - - def action: String = readOrDefault[String]("edata.action", "") - - def publishType: String = readOrDefault[String]("edata.publish_type", "") - - def mimeType: String = readOrDefault[String]("edata.metadata.mimeType", "") - - def objectId: String = readOrDefault[String]("edata.metadata.identifier", "") - - def objectType: String = readOrDefault[String]("edata.metadata.objectType", "") - - def lastPublishedBy: String = readOrDefault[String]("edata.metadata.lastPublishedBy", "") - - def pkgVersion: Double = { - val pkgVersion = readOrDefault[Int]("edata.metadata.pkgVersion", 0) - pkgVersion.toDouble - } - - def validEvent(): Boolean = { - (StringUtils.equals("publish", action) && StringUtils.isNotBlank(objectId)) && (objectTypes.contains(objectType) && mimeTypes.contains(mimeType)) - } -} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/domain/Models.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/domain/Models.scala deleted file mode 100644 index 86231a8ce..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/domain/Models.scala +++ /dev/null @@ -1,3 +0,0 @@ -package org.sunbird.job.questionset.publish.domain - -case class PublishMetadata(identifier: String, objectType: String, mimeType: String, pkgVersion: Double, publishType: String, lastPublishedBy: String) diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/helpers/QuestionPublisher.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/helpers/QuestionPublisher.scala deleted file mode 100644 index 5150f2c2b..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/helpers/QuestionPublisher.scala +++ /dev/null @@ -1,183 +0,0 @@ -package org.sunbird.job.questionset.publish.helpers - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} -import org.apache.commons.lang3 -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core._ -import org.sunbird.job.publish.helpers._ -import org.sunbird.job.util._ - -import java.io.File -import java.util -import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext} - -trait QuestionPublisher extends ObjectReader with ObjectValidator with ObjectEnrichment with EcarGenerator with ObjectUpdater { - private val bundleLocation: String = "/tmp" - private val indexFileName = "index.json" - private val defaultManifestVersion = "1.2" - private[this] val logger = LoggerFactory.getLogger(classOf[QuestionPublisher]) - val extProps = List("body", "editorState", "answer", "solutions", "instructions", "hints", "media", "responseDeclaration", "interactions") - - override def getHierarchy(identifier: String, pkgVersion: Double, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = None - - override def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig): Option[ObjectData] = { - val pkgVersion = obj.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number].intValue() + 1 - val publishType = obj.getString("publish_type", "Public") - val status = if (StringUtils.equals("Private", publishType)) "Unlisted" else "Live" - val updatedMeta = obj.metadata ++ Map("pkgVersion" -> pkgVersion.asInstanceOf[AnyRef], "status" -> status) - Some(new ObjectData(obj.identifier, updatedMeta, obj.extData, obj.hierarchy)) - } - - def validateQuestion(obj: ObjectData, identifier: String): List[String] = { - logger.info("Validating Question External Data For : " + obj.identifier) - val messages = ListBuffer[String]() - if (obj.extData.getOrElse(Map()).getOrElse("body", "").asInstanceOf[String].isEmpty) messages += s"""There is no body available for : $identifier""" - if (obj.extData.getOrElse(Map()).getOrElse("editorState", "").asInstanceOf[String].isEmpty) messages += s"""There is no editorState available for : $identifier""" - val interactionTypes = obj.metadata.getOrElse("interactionTypes", new util.ArrayList[String]()).asInstanceOf[util.List[String]].asScala.toList - if (interactionTypes.nonEmpty) { - if (obj.extData.get.getOrElse("responseDeclaration", "").asInstanceOf[String].isEmpty) messages += s"""There is no responseDeclaration available for : $identifier""" - if (obj.extData.get.getOrElse("interactions", "").asInstanceOf[String].isEmpty) messages += s"""There is no interactions available for : $identifier""" - } else { - if (obj.extData.getOrElse(Map()).getOrElse("answer", "").asInstanceOf[String].isEmpty) messages += s"""There is no answer available for : $identifier""" - } - messages.toList - } - - override def getExtData(identifier: String, pkgVersion: Double, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] = { - val row: Row = Option(getQuestionData(getEditableObjId(identifier, pkgVersion), readerConfig)).getOrElse(getQuestionData(identifier, readerConfig)) - val data = if (null != row) Option(extProps.map(prop => prop -> row.getString(prop.toLowerCase())).toMap.filter(p => StringUtils.isNotBlank(p._2))) else Option(Map[String, AnyRef]()) - Option(ObjectExtData(data)) - } - - def getQuestionData(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Row = { - logger.info("QuestionPublisher ::: getQuestionData ::: Reading Question External Data For : " + identifier) - val select = QueryBuilder.select() - extProps.foreach(prop => if (lang3.StringUtils.equals("body", prop) | lang3.StringUtils.equals("answer", prop)) select.fcall("blobAsText", QueryBuilder.column(prop.toLowerCase())).as(prop.toLowerCase()) else select.column(prop.toLowerCase()).as(prop.toLowerCase())) - val selectWhere: Select.Where = select.from(readerConfig.keyspace, readerConfig.table).where().and(QueryBuilder.eq("identifier", identifier)) - logger.info("Cassandra Fetch Query :: " + selectWhere.toString) - cassandraUtil.findOne(selectWhere.toString) - } - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = { - None - } - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = { - None - } - - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val extData = obj.extData.getOrElse(Map()) - val identifier = obj.identifier.replace(".img", "") - val query = QueryBuilder.update(readerConfig.keyspace, readerConfig.table) - .where(QueryBuilder.eq("identifier", identifier)) - .`with`(QueryBuilder.set("body", QueryBuilder.fcall("textAsBlob", extData.getOrElse("body", null)))) - .and(QueryBuilder.set("answer", QueryBuilder.fcall("textAsBlob", extData.getOrElse("answer", null)))) - .and(QueryBuilder.set("editorstate", extData.getOrElse("editorState", null))) - .and(QueryBuilder.set("solutions", extData.getOrElse("solutions", null))) - .and(QueryBuilder.set("instructions", extData.getOrElse("instructions", null))) - .and(QueryBuilder.set("media", extData.getOrElse("media", null))) - .and(QueryBuilder.set("hints", extData.getOrElse("hints", null))) - .and(QueryBuilder.set("responsedeclaration", extData.getOrElse("responseDeclaration", null))) - .and(QueryBuilder.set("interactions", extData.getOrElse("interactions", null))) - - logger.info(s"Updating Question in Cassandra For $identifier : ${query.toString}") - val result = cassandraUtil.upsert(query.toString) - if (result) { - logger.info(s"Question Updated Successfully For $identifier") - } else { - val msg = s"Question Update Failed For $identifier" - logger.error(msg) - throw new Exception(msg) - } - } - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Unit = { - val query = QueryBuilder.delete().from(readerConfig.keyspace, readerConfig.table) - .where(QueryBuilder.eq("identifier", obj.dbId)) - .ifExists - cassandraUtil.session.execute(query.toString) - } - - override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = { - Some(List(obj.metadata ++ obj.extData.getOrElse(Map()).filter(p => !excludeBundleMeta.contains(p._1)))) - } - - def getObjectWithEcar(data: ObjectData, pkgTypes: List[String])(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, defCache: DefinitionCache, defConfig: DefinitionConfig, httpUtil: HttpUtil): ObjectData = { - logger.info("QuestionPublisher:generateECAR: Ecar generation done for Question: " + data.identifier) - val ecarMap: Map[String, String] = generateEcar(data, pkgTypes) - val variants: java.util.Map[String, java.util.Map[String, String]] = ecarMap.map { case (key, value) => key.toLowerCase -> Map[String, String]("ecarUrl" -> value, "size" -> httpUtil.getSize(value).toString).asJava }.asJava - logger.info("QuestionPublisher ::: generateECAR ::: ecar map ::: " + ecarMap) - val meta: Map[String, AnyRef] = Map("downloadUrl" -> ecarMap.getOrElse(EcarPackageType.FULL.toString, ""), "variants" -> variants) - new ObjectData(data.identifier, data.metadata ++ meta, data.extData, data.hierarchy) - } - - def updateArtifactUrl(obj: ObjectData, pkgType: String)(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cloudStorageUtil: CloudStorageUtil, defCache: DefinitionCache, defConfig: DefinitionConfig, config: PublishConfig, httpUtil: HttpUtil): ObjectData = { - val bundlePath = bundleLocation + File.separator + obj.identifier + File.separator + System.currentTimeMillis + "_temp" - try { - val objType = obj.getString("objectType", "") - val objList = getDataForEcar(obj).getOrElse(List()) - val (updatedObjList, dUrls) = getManifestData(obj.identifier, objType, pkgType, objList) - val downloadUrls: Map[AnyRef, List[String]] = dUrls.flatten.groupBy(_._1).map { case (k, v) => k -> v.map(_._2) } - logger.info("QuestionPublisher ::: updateArtifactUrl ::: downloadUrls :::: " + downloadUrls) - val duration: String = config.getString("media_download_duration", "300 seconds") - val downloadedMedias: List[File] = Await.result(downloadFiles(obj.identifier, downloadUrls, bundlePath), Duration.apply(duration)) - if (downloadUrls.nonEmpty && downloadedMedias.isEmpty) - throw new Exception("Error Occurred While Downloading Bundle Media Files For : " + obj.identifier) - - getIndexFile(obj.identifier, objType, bundlePath, updatedObjList) - - // create zip package - val zipFileName: String = bundlePath + File.separator + obj.identifier + "_" + System.currentTimeMillis + ".zip" - FileUtils.createZipPackage(bundlePath, zipFileName) - - // upload zip file to blob and set artifactUrl - val result: Array[String] = uploadArtifactToCloud(new File(zipFileName), obj.identifier) - - val updatedMeta = obj.metadata ++ Map("artifactUrl" -> result(1)) - new ObjectData(obj.identifier, updatedMeta, obj.extData, obj.hierarchy) - } catch { - case ex: Exception => - ex.printStackTrace() - throw new Exception(s"Error While Generating $pkgType ECAR Bundle For : " + obj.identifier, ex) - } finally { - FileUtils.deleteDirectory(new File(bundlePath)) - } - } - - @throws[Exception] - def getIndexFile(identifier: String, objType: String, bundlePath: String, objList: List[Map[String, AnyRef]]): File = { - try { - val file: File = new File(bundlePath + File.separator + indexFileName) - val header: String = s"""{"id": "sunbird.${objType.toLowerCase()}.archive", "ver": "$defaultManifestVersion" ,"ts":"$getTimeStamp", "params":{"resmsgid": "$getUUID"}, "archive":{ "count": ${objList.size}, "ttl":24, "items": """ - val mJson = header + ScalaJsonUtil.serialize(objList) + "}}" - FileUtils.writeStringToFile(file, mJson) - file - } catch { - case e: Exception => throw new Exception("Exception occurred while writing manifest file for : " + identifier, e) - } - } - - private def uploadArtifactToCloud(uploadFile: File, identifier: String)(implicit cloudStorageUtil: CloudStorageUtil): Array[String] = { - var urlArray = new Array[String](2) - // Check the cloud folder convention to store artifact.zip file with Mahesh - try { - val folder = "question" + File.separator + Slug.makeSlug(identifier, isTransliterate = true) - urlArray = cloudStorageUtil.uploadFile(folder, uploadFile) - } catch { - case e: Exception => - cloudStorageUtil.deleteFile(uploadFile.getAbsolutePath, Option(false)) - logger.error("Error while uploading the Artifact file.", e) - throw new Exception("Error while uploading the Artifact File.", e) - } - urlArray - } - -} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/helpers/QuestionSetPublisher.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/helpers/QuestionSetPublisher.scala deleted file mode 100644 index 118f0618c..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/helpers/QuestionSetPublisher.scala +++ /dev/null @@ -1,192 +0,0 @@ -package org.sunbird.job.questionset.publish.helpers - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.{Clause, Insert, QueryBuilder, Select} -import org.apache.commons.lang3 -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers._ -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, JSONUtil, Neo4JUtil, ScalaJsonUtil} - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer - -trait QuestionSetPublisher extends ObjectReader with ObjectValidator with ObjectUpdater with ObjectEnrichment with EcarGenerator with QuestionPdfGenerator { - - private[this] val logger = LoggerFactory.getLogger(classOf[QuestionSetPublisher]) - val extProps = List("body", "editorState", "answer", "solutions", "instructions", "hints", "media", "responseDeclaration", "interactions", "identifier") - - override def getExtData(identifier: String, pkgVersion: Double, mimeType: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[ObjectExtData] = { - val row: Row = Option(getQuestionSetData(getEditableObjId(identifier, pkgVersion), readerConfig)).getOrElse(getQuestionSetData(identifier, readerConfig)) - val data: Map[String, AnyRef] = if (null != row) readerConfig.propsMapping.keySet.map(prop => prop -> row.getString(prop.toLowerCase())).toMap.filter(p => StringUtils.isNotBlank(p._2.asInstanceOf[String])) else Map[String, AnyRef]() - val hierarchy: Map[String, AnyRef] = if(data.contains("hierarchy")) ScalaJsonUtil.deserialize[Map[String, AnyRef]](data.getOrElse("hierarchy", "{}").asInstanceOf[String]) else Map[String, AnyRef]() - val extData:Map[String, AnyRef] = data.filter(p => !StringUtils.equals("hierarchy", p._1)) - Option(ObjectExtData(Option(extData), Option(hierarchy))) - } - - def getQuestionSetData(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Row = { - logger.info("QuestionSetPublisher ::: getQuestionSetData ::: Reading QuestionSet External Data For : "+identifier) - val qsExtProps = readerConfig.propsMapping.keySet - val select = QueryBuilder.select() - select.column(readerConfig.primaryKey(0)).as(readerConfig.primaryKey(0)) - if (null != qsExtProps && !qsExtProps.isEmpty) { - qsExtProps.foreach(prop => { - if ("blob".equalsIgnoreCase(readerConfig.propsMapping.getOrElse(prop, "").asInstanceOf[String])) - select.fcall("blobAsText", QueryBuilder.column(prop)).as(prop) - else - select.column(prop).as(prop) - }) - } - val selectQuery = select.from(readerConfig.keyspace, readerConfig.table) - val clause: Clause = QueryBuilder.eq("identifier", identifier) - selectQuery.where.and(clause) - logger.info("Cassandra Fetch Query :: "+ selectQuery.toString) - cassandraUtil.findOne(selectQuery.toString) - } - - def validateQuestionSet(obj: ObjectData, identifier: String): List[String] = { - val messages = ListBuffer[String]() - if (obj.hierarchy.getOrElse(Map()).isEmpty) messages += s"""There is no hierarchy available for : $identifier""" - if (!StringUtils.equalsIgnoreCase(obj.getString("mimeType", ""), "application/vnd.sunbird.questionset")) - messages += s"""mimeType is invalid for : $identifier""" - if (obj.getString("visibility", "").isEmpty) messages += s"""There is no visibility available for : $identifier""" - //TODO: Add any more check, if required. - messages.toList - } - - override def getHierarchy(identifier: String, pkgVersion: Double, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil, config: PublishConfig): Option[Map[String, AnyRef]] = { - val row: Row = Option(getQuestionSetHierarchy(getEditableObjId(identifier, pkgVersion), readerConfig)).getOrElse(getQuestionSetHierarchy(identifier, readerConfig)) - if (null != row) { - val data: Map[String, AnyRef] = ScalaJsonUtil.deserialize[Map[String, AnyRef]](row.getString("hierarchy")) - Option(data) - } else Option(Map()) - } - - def getQuestionSetHierarchy(identifier: String, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil) = { - val selectWhere: Select.Where = QueryBuilder.select().all() - .from(readerConfig.keyspace, readerConfig.table). - where() - selectWhere.and(QueryBuilder.eq("identifier", identifier)) - cassandraUtil.findOne(selectWhere.toString) - } - - override def getExtDatas(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = { - val rows = getQuestionsExtData(identifiers, readerConfig)(cassandraUtil).asScala - if (rows.nonEmpty) - Option(rows.map(row => row.getString("identifier") -> extProps.map(prop => (prop -> row.getString(prop.toLowerCase()))).toMap).toMap) - else - Option(Map[String, AnyRef]()) - } - - override def getHierarchies(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil): Option[Map[String, AnyRef]] = { - None - } - - def getQuestionsExtData(identifiers: List[String], readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil) = { - logger.info("QuestionSetPublisher ::: getQuestionsExtData ::: reader config ::: keyspace: " + readerConfig.keyspace + " , table : " + readerConfig.table) - val select = QueryBuilder.select() - extProps.foreach(prop => if (lang3.StringUtils.equals("body", prop) | lang3.StringUtils.equals("answer", prop)) select.fcall("blobAsText", QueryBuilder.column(prop.toLowerCase())).as(prop.toLowerCase()) else select.column(prop.toLowerCase()).as(prop.toLowerCase())) - val selectWhere: Select.Where = select.from(readerConfig.keyspace, readerConfig.table).where() - selectWhere.and(QueryBuilder.in("identifier", identifiers.asJava)) - logger.info("QuestionSetPublisher ::: getQuestionsExtData ::: cassandra query ::: " + selectWhere.toString) - cassandraUtil.find(selectWhere.toString) - } - - override def saveExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil) = { - val identifier = obj.identifier.replace(".img", "") - val children: List[Map[String, AnyRef]] = obj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - val hierarchy: Map[String, AnyRef] = obj.metadata ++ Map("children" -> children) - val data = Map("hierarchy" -> hierarchy) ++ obj.extData.getOrElse(Map()) - val query: Insert = QueryBuilder.insertInto(readerConfig.keyspace, readerConfig.table) - query.value(readerConfig.primaryKey(0), identifier) - data.map(d => { - readerConfig.propsMapping.getOrElse(d._1, "") match { - case "blob" => query.value(d._1.toLowerCase, QueryBuilder.fcall("textAsBlob", d._2)) - case "string" => d._2 match { - case value: String => query.value(d._1.toLowerCase, value) - case _ => query.value(d._1.toLowerCase, JSONUtil.serialize(d._2)) - } - case _ => query.value(d._1, d._2) - } - }) - logger.debug(s"Saving object external data for $identifier | Query : ${query.toString}") - val result = cassandraUtil.upsert(query.toString) - if (result) { - logger.info(s"Object external data saved successfully for ${identifier}") - } else { - val msg = s"Object External Data Insertion Failed For ${identifier}" - logger.error(msg) - throw new Exception(msg) - } - - } - - def getQuestions(qsObj: ObjectData, readerConfig: ExtDataConfig) - (implicit cassandraUtil: CassandraUtil): List[ObjectData] = { - val childrenMaps: Map[String, AnyRef] = populateChildrenMapRecursively(qsObj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]], Map()) - logger.info("QuestionSetPublisher ::: getQuestions ::: child questions ::::: " + childrenMaps) - val extMap = getExtDatas(childrenMaps.keys.toList, readerConfig) - logger.info("QuestionSetPublisher ::: getQuestions ::: child questions external data ::::: " + extMap.get) - extMap.getOrElse(Map()).map(entry => { - val metadata = childrenMaps.getOrElse(entry._1, Map()).asInstanceOf[Map[String, AnyRef]] ++ Map("IL_UNIQUE_ID" -> entry._1) - val obj = new ObjectData(entry._1, metadata, Option(entry._2.asInstanceOf[Map[String, AnyRef]])) - obj - }).toList - } - - - def populateChildrenMapRecursively(children: List[Map[String, AnyRef]], childrenMap: Map[String, AnyRef]): Map[String, AnyRef] = { - val result = children.flatMap(child => { - val updatedChildrenMap: Map[String, AnyRef] = - if (child.getOrElse("objectType", "").asInstanceOf[String].equalsIgnoreCase("Question")) { - Map(child.getOrElse("identifier", "").asInstanceOf[String] -> child) ++ childrenMap - } else childrenMap - val nextChild: List[Map[String, AnyRef]] = child.getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - val map = populateChildrenMapRecursively(nextChild, updatedChildrenMap) - map ++ updatedChildrenMap - }).toMap - result - } - - override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = { - val hChildren: List[Map[String, AnyRef]] = obj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - Some(getFlatStructure(List(obj.metadata ++ obj.extData.getOrElse(Map()) ++ Map("children" -> hChildren)), List())) - } - - override def enrichObjectMetadata(obj: ObjectData)(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, config: PublishConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig): Option[ObjectData] = { - val newMetadata: Map[String, AnyRef] = obj.metadata ++ Map("pkgVersion" -> (obj.pkgVersion + 1).asInstanceOf[AnyRef], "lastPublishedOn" -> getTimeStamp, - "publishError" -> null, "variants" -> null, "downloadUrl" -> null, "compatibilityLevel" -> 5.asInstanceOf[AnyRef], "status" -> "Live") - val children: List[Map[String, AnyRef]] = obj.hierarchy.getOrElse(Map()).getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - Some(new ObjectData(obj.identifier, newMetadata, obj.extData, hierarchy = Some(Map("identifier" -> obj.identifier, "children" -> enrichChildren(children))))) - } - - def enrichChildren(children: List[Map[String, AnyRef]])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig, config: PublishConfig): List[Map[String, AnyRef]] = { - val newChildren = children.map(element => enrichMetadata(element)) - newChildren - } - - def enrichMetadata(element: Map[String, AnyRef])(implicit neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig, config: PublishConfig): Map[String, AnyRef] = { - if (StringUtils.equalsIgnoreCase(element.getOrElse("objectType", "").asInstanceOf[String], "QuestionSet") - && StringUtils.equalsIgnoreCase(element.getOrElse("visibility", "").asInstanceOf[String], "Parent")) { - val children: List[Map[String, AnyRef]] = element.getOrElse("children", List()).asInstanceOf[List[Map[String, AnyRef]]] - val enrichedChildren = enrichChildren(children) - element ++ Map("children" -> enrichedChildren, "status" -> "Live") - } else if (StringUtils.equalsIgnoreCase(element.getOrElse("objectType", "").toString, "QuestionSet") - && StringUtils.equalsIgnoreCase(element.getOrElse("visibility", "").toString, "Default")) { - val childHierarchy: Map[String, AnyRef] = getHierarchy(element.getOrElse("identifier", "").toString, 0.asInstanceOf[Double], readerConfig).getOrElse(Map()) - childHierarchy ++ Map("index" -> element.getOrElse("index", 0).asInstanceOf[AnyRef], "depth" -> element.getOrElse("depth", 0).asInstanceOf[AnyRef], "parent" -> element.getOrElse("parent", "")) - } else if (StringUtils.equalsIgnoreCase(element.getOrElse("objectType", "").toString, "Question")) { - val newObject: ObjectData = getObject(element.getOrElse("identifier", "").toString, 0.asInstanceOf[Double], element.getOrElse("mimeType", "").toString, element.getOrElse("publish_type", "Public").toString, readerConfig) - val definition: ObjectDefinition = definitionCache.getDefinition("Question", definitionConfig.supportedVersion.getOrElse("question", "1.0").asInstanceOf[String], definitionConfig.basePath) - val enMeta = newObject.metadata.filter(x => null != x._2).map(element => (element._1, convertJsonProperties(element, definition.getJsonProps()))) - logger.info("enrichMeta :::: question object meta ::: " + enMeta) - enMeta ++ Map("index" -> element.getOrElse("index", 0).asInstanceOf[AnyRef], "parent" -> element.getOrElse("parent", ""), "depth" -> element.getOrElse("depth", 0).asInstanceOf[AnyRef]) - } else Map() - } - - override def deleteExternalData(obj: ObjectData, readerConfig: ExtDataConfig)(implicit cassandraUtil: CassandraUtil) = None - -} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/util/QuestionPublishUtil.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/util/QuestionPublishUtil.scala deleted file mode 100644 index fc7c18151..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/publish/util/QuestionPublishUtil.scala +++ /dev/null @@ -1,50 +0,0 @@ -package org.sunbird.job.questionset.publish.util - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.questionset.publish.helpers.QuestionPublisher -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} - -import scala.concurrent.ExecutionContext - -object QuestionPublishUtil extends QuestionPublisher { - - private val pkgTypes = List(EcarPackageType.FULL.toString, EcarPackageType.ONLINE.toString) - - private[this] val logger = LoggerFactory.getLogger(classOf[QuestionPublishUtil]) - - def publishQuestions(identifier: String, objList: List[ObjectData], pkgVersion: Double, lastPublishedBy: String)(implicit ec: ExecutionContext, neo4JUtil: Neo4JUtil, cassandraUtil: CassandraUtil, readerConfig: ExtDataConfig, cloudStorageUtil: CloudStorageUtil, definitionCache: DefinitionCache, definitionConfig: DefinitionConfig, config: PublishConfig, httpUtil: HttpUtil): List[ObjectData] = { - logger.info("QuestionPublishUtil :::: publishing child question for questionset : " + identifier) - objList.map(qData => { - logger.info("QuestionPublishUtil :::: publishing child question : " + qData.identifier) - val objData = getObject(qData.identifier, qData.pkgVersion, qData.mimeType, qData.metadata.getOrElse("publish_type", "Public").toString, readerConfig)(neo4JUtil, cassandraUtil, config) - val obj = if (StringUtils.isNotBlank(lastPublishedBy)) { - val newMeta = objData.metadata ++ Map("lastPublishedBy" -> lastPublishedBy) - new ObjectData(objData.identifier, newMeta, objData.extData, objData.hierarchy) - } else objData - val messages: List[String] = validate(obj, obj.identifier, validateQuestion) - if (messages.isEmpty) { - val enrichedObj = enrichObject(obj)(neo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, config, definitionCache, definitionConfig) - val objWithArtifactUrl = if (enrichedObj.getString("artifactUrl", "").isEmpty) { - //create artifact zip locally, upload to cloud and update the artifact URL - updateArtifactUrl(enrichedObj, EcarPackageType.FULL.toString)(ec, neo4JUtil, cloudStorageUtil, definitionCache, definitionConfig, config, httpUtil) - } else enrichedObj - val objWithEcar = getObjectWithEcar(objWithArtifactUrl, pkgTypes)(ec, neo4JUtil, cloudStorageUtil, config, definitionCache, definitionConfig, httpUtil) - logger.info("Ecar generation done for Question: " + objWithEcar.identifier) - saveOnSuccess(objWithEcar)(neo4JUtil, cassandraUtil, readerConfig, definitionCache, definitionConfig, config) - logger.info("Question publishing completed successfully for : " + qData.identifier) - objWithEcar - } else { - saveOnFailure(obj, messages, pkgVersion)(neo4JUtil) - logger.info("Question publishing failed for : " + qData.identifier) - obj - } - }) - } -} - -class QuestionPublishUtil {} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/task/QuestionSetPublishConfig.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/task/QuestionSetPublishConfig.scala deleted file mode 100644 index e6989e86d..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/task/QuestionSetPublishConfig.scala +++ /dev/null @@ -1,65 +0,0 @@ -package org.sunbird.job.questionset.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.questionset.publish.domain.PublishMetadata - -import java.util -import scala.collection.JavaConverters._ - -class QuestionSetPublishConfig(override val config: Config) extends PublishConfig(config, "questionset-publish"){ - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val publishMetaTypeInfo: TypeInformation[PublishMetadata] = TypeExtractor.getForClass(classOf[PublishMetadata]) - - // Job Configuration - val jobEnv: String = config.getString("job.env") - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val postPublishTopic: String = config.getString("kafka.post_publish.topic") - val inputConsumerName = "questionset-publish-consumer" - - // Parallelism - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val eventRouterParallelism: Int = config.getInt("task.router.parallelism") - - // Metric List - val totalEventsCount = "total-events-count" - val skippedEventCount = "skipped-event-count" - val questionPublishEventCount = "question-publish-count" - val questionPublishSuccessEventCount = "question-publish-success-count" - val questionPublishFailedEventCount = "question-publish-failed-count" - val questionSetPublishEventCount = "questionset-publish-count" - val questionSetPublishSuccessEventCount = "questionset-publish-success-count" - val questionSetPublishFailedEventCount = "questionset-publish-failed-count" - - // Cassandra Configurations - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - val questionKeyspaceName = config.getString("question.keyspace") - val questionTableName = config.getString("question.table") - val questionSetKeyspaceName = config.getString("questionset.keyspace") - val questionSetTableName = config.getString("questionset.table") - - // Neo4J Configurations - val graphRoutePath = config.getString("neo4j.routePath") - val graphName = config.getString("neo4j.graph") - - // Cache Config - val cacheDbId: Int = if(config.hasPath("redis.database.qsCache.id")) config.getInt("redis.database.qsCache.id") else 0 - - // Out Tags - val questionPublishOutTag: OutputTag[PublishMetadata] = OutputTag[PublishMetadata]("question-publish") - val questionSetPublishOutTag: OutputTag[PublishMetadata] = OutputTag[PublishMetadata]("questionset-publish") - - // Service Urls - val printServiceBaseUrl: String = config.getString("print_service.base_url") - - val definitionBasePath: String = if (config.hasPath("schema.basePath")) config.getString("schema.basePath") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - val schemaSupportVersionMap = if (config.hasPath("schema.supportedVersion")) config.getObject("schema.supportedVersion").unwrapped().asScala.toMap else Map[String, AnyRef]() -} diff --git a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/task/QuestionSetPublishStreamTask.scala b/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/task/QuestionSetPublishStreamTask.scala deleted file mode 100644 index ee6a05947..000000000 --- a/publish-pipeline/questionset-publish/src/main/scala/org/sunbird/job/questionset/task/QuestionSetPublishStreamTask.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.questionset.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.questionset.function.{PublishEventRouter, QuestionPublishFunction, QuestionSetPublishFunction} -import org.sunbird.job.questionset.publish.domain.{Event, PublishMetadata} -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - -import java.io.File -import java.util - -class QuestionSetPublishStreamTask(config: QuestionSetPublishConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val publishMetaTypeInfo: TypeInformation[PublishMetadata] = TypeExtractor.getForClass(classOf[PublishMetadata]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - val processStreamTask = env.addSource(source).name(config.inputConsumerName) - .uid(config.inputConsumerName).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .process(new PublishEventRouter(config)) - .name("publish-event-router").uid("publish-event-router") - .setParallelism(config.eventRouterParallelism) - - processStreamTask.getSideOutput(config.questionPublishOutTag).process(new QuestionPublishFunction(config, httpUtil)) - .name("question-publish-process").uid("question-publish-process").setParallelism(1) - - processStreamTask.getSideOutput(config.questionSetPublishOutTag).process(new QuestionSetPublishFunction(config, httpUtil)) - .name("questionset-publish-process").uid("questionset-publish-process").setParallelism(1) - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object QuestionSetPublishStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("questionset-publish.conf").withFallback(ConfigFactory.systemEnvironment())) - val publishConfig = new QuestionSetPublishConfig(config) - val kafkaUtil = new FlinkKafkaConnector(publishConfig) - val httpUtil = new HttpUtil - val task = new QuestionSetPublishStreamTask(publishConfig, kafkaUtil, httpUtil) - task.process() - } -} diff --git a/publish-pipeline/questionset-publish/src/test/resources/logback-test.xml b/publish-pipeline/questionset-publish/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/publish-pipeline/questionset-publish/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/publish-pipeline/questionset-publish/src/test/resources/test.conf b/publish-pipeline/questionset-publish/src/test/resources/test.conf deleted file mode 100644 index 7f3fcdac6..000000000 --- a/publish-pipeline/questionset-publish/src/test/resources/test.conf +++ /dev/null @@ -1,35 +0,0 @@ -include "base-test.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.learning.job.request" - post_publish.topic = "sunbirddev.content.postpublish.request" - groupId = "local-questionset-publish-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - router.parallelism = 1 -} - -question { - keyspace = "dev_question_store" - table = "question_data" -} - -questionset { - keyspace = "dev_hierarchy_store" - table = "questionset_hierarchy" -} - -neo4j { - routePath = "bolt://localhost:7687" - graph = "domain" -} - - -print_service.base_url="http://11.2.6.6/print" diff --git a/publish-pipeline/questionset-publish/src/test/resources/test.cql b/publish-pipeline/questionset-publish/src/test/resources/test.cql deleted file mode 100644 index 30d8e8833..000000000 --- a/publish-pipeline/questionset-publish/src/test/resources/test.cql +++ /dev/null @@ -1,50 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS dev_hierarchy_store with replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS dev_hierarchy_store.questionset_hierarchy ( - identifier text, - hierarchy text, - instructions text, - PRIMARY KEY (identifier) -); - -CREATE KEYSPACE IF NOT EXISTS dev_question_store with replication = {'class':'SimpleStrategy','replication_factor':1}; -CREATE TABLE IF NOT EXISTS dev_question_store.question_data ( - identifier text, - body blob, - editorstate text, - answer blob, - solutions text, - instructions text, - hints text, - media text, - responsedeclaration text, - interactions text, - PRIMARY KEY (identifier) -); - - -INSERT INTO dev_question_store.question_data(identifier, body, editorstate, answer, solutions, media, responsedeclaration) VALUES ( -'do_113188615625731', -textAsBlob('

Color of Sky is?

'), -'{"question":"

Capital of india is?

","answer":"

New Delhi

"}', -textAsBlob('1'), -'[{"id": "c012a8a9-f78b-6ddb-3ac2-bd1f38c7850b", "type": "html", "value": "

Color of sky is blue 

\"do_11310507846892748812026\"
"}]', -'[{"id":"do_1127129497561497601326","type":"image","src":"/content/do_1127129497561497601326.img/artifact/sunbird_1551961194254.jpeg","baseUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev"}]', -'{"response1": {"maxScore": 1, "cardinality": "single", "type": "integer", "correctResponse": { "value": "1", "outcomes": {"SCORE": 1}}}}' -); - - -INSERT INTO dev_question_store.question_data(identifier, body, editorstate, answer, solutions, media, responsedeclaration, interactions) VALUES ( -'do_113188615625731.img', -textAsBlob('

Color of Sky is?

'), -'{"question":"

Capital of india is?

","answer":"

New Delhi

"}', -textAsBlob('1'), -'[{"id": "c012a8a9-f78b-6ddb-3ac2-bd1f38c7850b", "type": "html", "value": "

Color of sky is blue 

\"do_11310507846892748812026\"
"}]', -'[{"id": "do_11318931140144332811620", "type": "image", "src": "https://dockstorage.blob.core.windows.net/sunbird-content-dock/content/do_11318931140144332811620/artifact/i.png", baseUrl": "https://dock.sunbirded.org"}, {"id": "do_11310507846892748812026", "type": "image", "src": "https://dockstorage.blob.core.windows.net/sunbird-content-dock/content/do_11310507846892748812026/artifact/icon.png", "baseUrl": "https://dock.sunbirded.org"}]', -'{"response1": {"maxScore": 1, "cardinality": "single", "type": "integer", "correctResponse": { "value": "1", "outcomes": {"SCORE": 1}}}}', -'{"response1": {"type": "choice","options": [{"label": "

Red 

\"do_11318931140144332811620\"
","value": 0}]}}' -); - - -INSERT INTO dev_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_123', '{ "name": "QS1", "identifier": "do_123", "children": [{"identifier": "do_124", "objectType": "QuestionSet", "visibility": "Parent"}]}'); -INSERT INTO dev_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_321', '{ "name": "QS1", "identifier": "do_123", "children": [{"identifier": "do_124", "objectType": "QuestionSet", "visibility": "Parent"}]}'); -INSERT INTO dev_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_123.img', '{ "name": "QS1", "identifier": "do_123", "children": [{"identifier": "do_124", "objectType": "QuestionSet", "visibility": "Parent"}, {"identifier": "do_125", "objectType": "QuestionSet", "visibility": "Parent"}]}'); \ No newline at end of file diff --git a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index 51e462188..000000000 --- a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,10 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val QUESTION_EVENT1: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1609926636251,"mid":"LP.1609926636251.b93d8562-537e-4e52-bcf5-b9175a550391","actor":{"id":"question-publish","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"},"channel":""},"object":{"ver":"1609926299686","id":"do_113188615625731"},"edata":{"publish_type":"public","metadata":{"identifier":"do_113188615625731","mimeType":"application/vnd.sunbird.question","lastPublishedBy":null,"pkgVersion":1,"objectType":"Question"},"action":"publish","iteration":1}} - |""".stripMargin - -} diff --git a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/QuestionPublisherSpec.scala b/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/QuestionPublisherSpec.scala deleted file mode 100644 index eafdf38af..000000000 --- a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/QuestionPublisherSpec.scala +++ /dev/null @@ -1,131 +0,0 @@ -package org.sunbird.job.publish.helpers.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.commons.lang3.StringUtils -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.publish.helpers.EcarPackageType -import org.sunbird.job.questionset.publish.helpers.QuestionPublisher -import org.sunbird.job.questionset.task.QuestionSetPublishConfig -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} - -import java.util - -class QuestionPublisherSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: QuestionSetPublishConfig = new QuestionSetPublishConfig(config) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.questionKeyspaceName, jobConfig.questionTableName) - implicit val cloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val ec = ExecutionContexts.global - implicit val defCache = new DefinitionCache() - implicit val defConfig = DefinitionConfig(jobConfig.schemaSupportVersionMap, jobConfig.definitionBasePath) - implicit val httpUtil = new HttpUtil - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => { - } - } - } - - "enrichObjectMetadata" should "enrich the Question pkgVersion metadata" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef])) - val result: ObjectData = new TestQuestionPublisher().enrichObjectMetadata(data).getOrElse(data) - result.metadata.getOrElse("pkgVersion", 0.0.asInstanceOf[Number]).asInstanceOf[Number] should be(1.0.asInstanceOf[Number]) - } - "validateQuestion with invalid external data" should "return exception messages" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef]), Some(Map[String, AnyRef]("body" -> "body"))) - val result: List[String] = new TestQuestionPublisher().validateQuestion(data, data.identifier) - result.size should be(2) - } - - "validateQuestion with external data having interaction" should "validate the Question external data" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef], "interactionTypes" -> new util.ArrayList[String]() { - add("choice") - }), Some(Map[String, AnyRef]("body" -> "body", "answer" -> "answer"))) - val result: List[String] = new TestQuestionPublisher().validateQuestion(data, data.identifier) - result.size should be(3) - } - - "saveExternalData " should "save external data to cassandra table" in { - val data = new ObjectData("do_123", Map[String, AnyRef](), Some(Map[String, AnyRef]("body" -> "body", "answer" -> "answer"))) - new TestQuestionPublisher().saveExternalData(data, readerConfig) - } - - "getExtData " should "return the external data for the identifier " in { - val identifier = "do_113188615625731"; - val res: Option[ObjectExtData] = new TestQuestionPublisher().getExtData(identifier, 0.0, "", readerConfig) - val result: Option[Map[String, AnyRef]] = res.getOrElse(new ObjectExtData).data - result.getOrElse(Map()).size should be(6) - } - - "getExtData " should "return the external data for the image identifier " in { - val identifier = "do_113188615625731"; - val res: Option[ObjectExtData] = new TestQuestionPublisher().getExtData(identifier, 1.0, "", readerConfig) - val result: Option[Map[String, AnyRef]] = res.getOrElse(new ObjectExtData).data - result.getOrElse(Map()).size should be(7) - } - - "getHierarchy " should "do nothing " in { - val identifier = "do_113188615625731"; - new TestQuestionPublisher().getHierarchy(identifier, 1.0, readerConfig) - } - - "getExtDatas " should "do nothing " in { - val identifier = "do_113188615625731"; - new TestQuestionPublisher().getExtDatas(List(identifier), readerConfig) - } - - "getHierarchies " should "do nothing " in { - val identifier = "do_113188615625731"; - new TestQuestionPublisher().getHierarchies(List(identifier), readerConfig) - } - - "getDataForEcar" should "return one element in list" in { - val data = new ObjectData("do_123", Map("objectType" -> "Question"), Some(Map("responseDeclaration" -> "test")), Some(Map())) - val result: Option[List[Map[String, AnyRef]]] = new TestQuestionPublisher().getDataForEcar(data) - result.size should be(1) - } - - "getObjectWithEcar" should "return object with ecar url" in { - val data = new ObjectData("do_123", Map("objectType" -> "Question", "identifier" -> "do_123", "name" -> "Test Question"), Some(Map("responseDeclaration" -> "test", "media" -> "[{\"id\":\"do_1127129497561497601326\",\"type\":\"image\",\"src\":\"/content/do_1127129497561497601326.img/artifact/sunbird_1551961194254.jpeg\",\"baseUrl\":\"https://sunbirddev.blob.core.windows.net/sunbird-content-dev\"}]")), Some(Map())) - val result = new TestQuestionPublisher().getObjectWithEcar(data, List(EcarPackageType.FULL.toString, EcarPackageType.ONLINE.toString))(ec, mockNeo4JUtil, cloudStorageUtil, jobConfig, defCache, defConfig, httpUtil) - StringUtils.isNotBlank(result.metadata.getOrElse("downloadUrl", "").asInstanceOf[String]) - - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } -} - -class TestQuestionPublisher extends QuestionPublisher {} diff --git a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/QuestionSetPublisherSpec.scala b/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/QuestionSetPublisherSpec.scala deleted file mode 100644 index e2e31efb2..000000000 --- a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/helpers/spec/QuestionSetPublisherSpec.scala +++ /dev/null @@ -1,131 +0,0 @@ -package org.sunbird.job.publish.helpers.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData, ObjectExtData} -import org.sunbird.job.questionset.publish.helpers.QuestionSetPublisher -import org.sunbird.job.questionset.task.QuestionSetPublishConfig -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, Neo4JUtil} - -class QuestionSetPublisherSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val jobConfig: QuestionSetPublishConfig = new QuestionSetPublishConfig(config) - implicit val cloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.questionSetKeyspaceName, jobConfig.questionSetTableName, List("identifier"), Map("hierarchy"->"string","instructions"->"string")) - val questionReaderConfig: ExtDataConfig = ExtDataConfig(jobConfig.questionKeyspaceName, jobConfig.questionTableName) - implicit val defCache = new DefinitionCache() - implicit val defConfig = DefinitionConfig(jobConfig.schemaSupportVersionMap, jobConfig.definitionBasePath) - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => { - } - } - } - - "validateQuestionSet with no hierarchy, mimeType and visibility" should "return 3 error message" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef])) - val result: List[String] = new TestQuestionSetPublisher().validateQuestionSet(data, data.identifier) - result.size should be(3) - } - - "saveExternalData" should "save external data like hierarchy" in { - val data = new ObjectData("do_123", Map[String, AnyRef]("name" -> "Content Name", "identifier" -> "do_123", "pkgVersion" -> 0.0.asInstanceOf[AnyRef])) - new TestQuestionSetPublisher().saveExternalData(data, readerConfig) - } - - "populateChildrenMapRecursively with two children" should "return a map with one data" in { - val result: Map[String, AnyRef] = new TestQuestionSetPublisher().populateChildrenMapRecursively(List(Map[String, AnyRef] ("identifier" -> "do_123", "objectType" -> "Question"), Map[String, AnyRef] ("identifier" -> "do_124", "objectType" -> "QuestionSet")), Map()) - result.size should be (1) - } - - "getDataForEcar with empty children" should "return one element in list" in { - val data = new ObjectData("do_123", Map(), Some(Map()), Some(Map[String, AnyRef]("identifier" -> "do_123", "children" -> List(Map())))) - val result: Option[List[Map[String, AnyRef]]] = new TestQuestionSetPublisher().getDataForEcar(data) - result.size should be (1) - } - - "enrichObjectMetadata with valid children" should "return updated pkgVersion" in { - val data = new ObjectData("do_123", Map[String, AnyRef] ("name" -> "QS1"), Some(Map()), Some(Map[String, AnyRef]("identifier" -> "do_123", "children" -> List(Map[String, AnyRef]("identifier" -> "do_124", "objectType"->"QuestionSet", "visibility"-> "Parent"), Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType"->"Question", "visibility"-> "Default"))))) - val result: Option[ObjectData] = new TestQuestionSetPublisher().enrichObjectMetadata(data) - result.getOrElse(new ObjectData("do_123", Map())).pkgVersion should be (1.asInstanceOf[Number]) - result.size should be (1) - } - - "getHierarchy " should " return the hierarchy of the Question Set for the image id " in { - val identifier = "do_123" - val result = new TestQuestionSetPublisher().getHierarchy(identifier, 1.0, readerConfig) - result.getOrElse(Map()).size should be(3) - result.getOrElse(Map()).getOrElse("children", List(Map())).asInstanceOf[List[Map[String, AnyRef]]].size should be (2) - } - - "getHierarchy " should " return the hierarchy of the Question Set " in { - val identifier = "do_321" - val result = new TestQuestionSetPublisher().getHierarchy(identifier, 1.0, readerConfig) - result.getOrElse(Map()).size should be(3) - result.getOrElse(Map()).getOrElse("children", List(Map())).asInstanceOf[List[Map[String, AnyRef]]].size should be (1) - } - - "getQuestions " should " return the question data from the hierarchy of the Question Set " in { - val data = new ObjectData("do_123", Map[String, AnyRef] ("name" -> "QS1"), Some(Map()), Some(Map[String, AnyRef]("identifier" -> "do_123", "children" -> List(Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType"->"Question", "visibility"-> "Parent"))))) - val result = new TestQuestionSetPublisher().getQuestions(data, questionReaderConfig) - result.size should be(1) - } - - "getExtDatas " should " return the External data of the provided identifiers " in { - val result = new TestQuestionSetPublisher().getExtDatas(List("do_113188615625731", "do_113188615625731.img"), questionReaderConfig) - result.getOrElse(List(Map())).size should be(2) - } - - "deleteExternalData with valid ObjectData" should "should do nothing" in { - new TestQuestionSetPublisher().deleteExternalData(new ObjectData("do_123", Map()), readerConfig) - } - - "deleteExternalData " should "do nothing " in { - val objData = new ObjectData("do_113188615625731", Map()); - new TestQuestionSetPublisher().deleteExternalData(objData, readerConfig) - } - - "getExtData " should " return ObjectExtData having hierarchy" in { - val identifier = "do_321" - val result = new TestQuestionSetPublisher().getExtData(identifier, 1.0,"", readerConfig) - result.getOrElse(new ObjectExtData).asInstanceOf[ObjectExtData].hierarchy.getOrElse(Map()).contains("do_321") - result.getOrElse(new ObjectExtData).asInstanceOf[ObjectExtData].hierarchy.getOrElse(Map()).getOrElse("children", List(Map())).asInstanceOf[List[Map[String, AnyRef]]].size should be (1) - } - "getHierarchies " should "do nothing " in { - val identifier = "do_113188615625731"; - new TestQuestionSetPublisher().getHierarchies(List(identifier), readerConfig) - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } -} - -class TestQuestionSetPublisher extends QuestionSetPublisher{} \ No newline at end of file diff --git a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/util/spec/QuestionPublishUtilSpec.scala b/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/util/spec/QuestionPublishUtilSpec.scala deleted file mode 100644 index 1e518ebfa..000000000 --- a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/publish/util/spec/QuestionPublishUtilSpec.scala +++ /dev/null @@ -1,74 +0,0 @@ -package org.sunbird.job.publish.util.spec - -import akka.dispatch.ExecutionContexts -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.publish.core.{DefinitionConfig, ExtDataConfig, ObjectData} -import org.sunbird.job.questionset.publish.util.QuestionPublishUtil -import org.sunbird.job.questionset.task.QuestionSetPublishConfig -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} - -import scala.collection.JavaConverters._ - -class QuestionPublishUtilSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - implicit val ec = ExecutionContexts.global - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - implicit var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: QuestionSetPublishConfig = new QuestionSetPublishConfig(config) - implicit val readerConfig: ExtDataConfig = ExtDataConfig(jobConfig.questionKeyspaceName, jobConfig.questionTableName) - implicit val cloudStorageUtil = new CloudStorageUtil(jobConfig) - implicit val defCache = new DefinitionCache() - implicit val defConfig = DefinitionConfig(jobConfig.schemaSupportVersionMap, jobConfig.definitionBasePath) - implicit val publishConfig: PublishConfig = new PublishConfig(config, "") - implicit val httpUtil = new HttpUtil - - override protected def beforeAll(): Unit = { - super.beforeAll() - - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - delay(10000) - } catch { - case ex: Exception => { - } - } - } - - def delay(time: Long): Unit = { - try { - Thread.sleep(time) - } catch { - case ex: Exception => print("") - } - } - - "publishQuestions " should " publish questions in questionset " in { - when(mockNeo4JUtil.getNodeProperties("do_113188615625731")).thenReturn(Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Question", "IL_FUNC_OBJECT_TYPE" -> "Question", "visibility" -> "Parent", "mimeType" -> "application/vnd.sunbird.question", "primaryCategory" -> "some category", "name" -> "Some Question", "code" -> "some code").asJava) - - val identifier = "do_123"; - val obj1: ObjectData = new ObjectData("do_113188615625731", Map[String, AnyRef]("identifier" -> "do_113188615625731", "objectType" -> "Question", "visibility" -> "Parent", "mimeType" -> "application/vnd.sunbird.question", "primaryCategory" -> "some category", "name" -> "Some Question", "code" -> "some code")) - val objList: List[ObjectData] = List(obj1) - val publishedQuestions = QuestionPublishUtil.publishQuestions(identifier, objList, 1, "sunbird")(ec, mockNeo4JUtil, cassandraUtil, readerConfig, cloudStorageUtil, defCache, defConfig, jobConfig, httpUtil) - publishedQuestions.map(record => record.getString("artifactUrl","") shouldNot be(null)) - } - -} diff --git a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/spec/QuestionSetPublishStreamTaskSpec.scala b/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/spec/QuestionSetPublishStreamTaskSpec.scala deleted file mode 100644 index 5606b56be..000000000 --- a/publish-pipeline/questionset-publish/src/test/scala/org/sunbird/job/spec/QuestionSetPublishStreamTaskSpec.scala +++ /dev/null @@ -1,96 +0,0 @@ -package org.sunbird.job.spec - -import com.google.gson.Gson -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.publish.config.PublishConfig -import org.sunbird.job.questionset.publish.domain.Event -import org.sunbird.job.questionset.task.{QuestionSetPublishConfig, QuestionSetPublishStreamTask} -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, HttpUtil, Neo4JUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import java.util - -class QuestionSetPublishStreamTaskSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - implicit val jobConfig: QuestionSetPublishConfig = new QuestionSetPublishConfig(config) - - val mockHttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - val publishConfig: PublishConfig = new PublishConfig(config, "") - val cloudStorageUtil: CloudStorageUtil = new CloudStorageUtil(publishConfig) - - override protected def beforeAll(): Unit = { - super.beforeAll() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - super.afterAll() - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - flinkCluster.after() - } - - //TODO: provide test cases. - def initialize(): Unit = { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new QuestionPublishEventSource) - } - - ignore should " publish the question " in { - initialize - new QuestionSetPublishStreamTask(jobConfig, mockKafkaUtil, mockHttpUtil).process() - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.questionPublishEventCount}").getValue() should be(1) - } -} - -private class QuestionPublishEventSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(jsonToEvent(EventFixture.QUESTION_EVENT1)) - } - - override def cancel() = {} - - def jsonToEvent(json: String): Event = { - val gson = new Gson() - val data = gson.fromJson(json, new util.LinkedHashMap[String, Any]().getClass).asInstanceOf[util.Map[String, Any]] - val metadataMap = data.get("edata").asInstanceOf[util.Map[String, Any]].get("metadata").asInstanceOf[util.Map[String, Any]] - metadataMap.put("pkgVersion",metadataMap.get("pkgVersion").asInstanceOf[Double].toInt) - new Event(data, 0, 10) - } - -} diff --git a/qrcode-image-generator/README.md b/qrcode-image-generator/README.md deleted file mode 100644 index cd6477f4b..000000000 --- a/qrcode-image-generator/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# QR Code Image Generator - -QR Code Image Generator job - -## Getting Started - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. -### Prerequisites - -1. Download flink-1.12.0-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.12.0/flink-1.12.0-bin-scala_2.12.tgz). -2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. -3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. -4. Docker installed. -5. A running yarn cluster or a kubernetes cluster. - -### Build - -mvn clean install - -## Deployment - -### Yarn - -Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. - -``` -./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /qrcode-image-generator/target/qrcode-image-generator-0.0.1.jar -``` - -### Kubernetes - -``` -# Create a single node cluster -k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 -# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. -export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" - -# Only for Mac OSX -# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl -rm /usr/local/bin/kubectl -brew link --overwrite kubernetes-cli - -# Create a configmap using the flink-configuration-configmap.yaml -kubectl create -f knowledge-platform-job/kubernetes/flink-configuration-configmap.yaml - -# Create pods for jobmanager-service, job-manager and task-manager using the yaml files -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-service.yaml -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-deployment.yaml -kubectl create -f knowledge-platform-job/kubernetes/taskmanager-deployment.yaml - -# Create a port-forwarding for accessing the job-manager UI on localhost:8081 -kubectl port-forward deployment/flink-jobmanager 8081:8081 - -# Submit the job to the Kubernetes single node cluster flink-cluster -./bin/flink run -m localhost:8081 /qrcode-image-generator/target/qrcode-image-generator-0.0.1.jar - -# Commands to delete the pods created in the cluster -kubectl delete deployment/flink-jobmanager -kubectl delete deployment/flink-taskmanager -kubectl delete service/flink-jobmanager -kubectl delete configmaps/flink-config - -# Command to stop the single-node cluster -k3d stop --name="flink-cluster" -``` diff --git a/qrcode-image-generator/pom.xml b/qrcode-image-generator/pom.xml deleted file mode 100644 index 68305d0dd..000000000 --- a/qrcode-image-generator/pom.xml +++ /dev/null @@ -1,226 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - qrcode-image-generator - 1.0.0 - jar - qrcode-image-generator - - QR Code Image Generator Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - com.google.zxing - core - 3.3.3 - - - com.google.zxing - javase - 3.3.3 - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - org.sunbird.job.qrimagegenerator.task.QRCodeImageGeneratorTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - qrcode-image-generator-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/qrcode-image-generator/src/main/resources/Verdana.ttf b/qrcode-image-generator/src/main/resources/Verdana.ttf deleted file mode 100755 index 18ef6e8f1..000000000 Binary files a/qrcode-image-generator/src/main/resources/Verdana.ttf and /dev/null differ diff --git a/qrcode-image-generator/src/main/resources/log4j.properties b/qrcode-image-generator/src/main/resources/log4j.properties deleted file mode 100644 index 09460618f..000000000 --- a/qrcode-image-generator/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=qrcode-image-generator.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/qrcode-image-generator/src/main/resources/qrcode-image-generator.conf b/qrcode-image-generator/src/main/resources/qrcode-image-generator.conf deleted file mode 100644 index 7ad19a6cb..000000000 --- a/qrcode-image-generator/src/main/resources/qrcode-image-generator.conf +++ /dev/null @@ -1,47 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.qrimage.request" - groupId = "sunbirddev-qrcode-image-generator-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - window.time = 60 -} - -lp.tmp.file.location="/tmp" - -qr.image { - imageFormat="png" - bottomMargin=0 - margin=1 -} - -lms-cassandra { - keyspace = "dialcodes" - table { - image = "dialcode_images" - batch = "dialcode_batch" - } -} - -# Default value is 120 -max_allowed_character_for_file_name = 120 - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "DIAL_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net/dial","https://obj.dev.sunbird.org/dial"] -cloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] - -cloud_storage_type="" -cloud_storage_key="" -cloud_storage_secret="" -cloud_storage_container="" -cloud_storage_endpoint="" diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/domain/Event.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/domain/Event.scala deleted file mode 100644 index 016c8703c..000000000 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/domain/Event.scala +++ /dev/null @@ -1,47 +0,0 @@ -package org.sunbird.job.qrimagegenerator.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest -import org.sunbird.job.qrimagegenerator.task.QRCodeImageGeneratorConfig - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "qrcode-image-generator" - - def eid: String = readOrDefault[String]("eid", "") - - def processId: String = readOrDefault[String]("processId", "") - - def objectId: String = readOrDefault[String]("objectId", "") - - def storageContainer: String = readOrDefault[String]("storage.container", "") - - def storagePath: String = readOrDefault[String]("storage.path", "") - - def storageFileName: String = readOrDefault[String]("storage.fileName", "") - - def imageConfigMap: Map[String, AnyRef] = readOrDefault("config", Map[String, AnyRef]()) - - def dialCodes: List[Map[String, AnyRef]] = readOrDefault[List[Map[String, AnyRef]]]("dialcodes", List()) - - def isValid(config: QRCodeImageGeneratorConfig): Boolean = { - eid.nonEmpty && StringUtils.equalsIgnoreCase(config.eid, eid) && dialCodes.nonEmpty - } - - def imageConfig(config: QRCodeImageGeneratorConfig): ImageConfig = { - ImageConfig( - imageConfigMap.getOrElse("errorCorrectionLevel", "").asInstanceOf[String], - imageConfigMap.getOrElse("pixelsPerBlock", 0).asInstanceOf[Int], - imageConfigMap.getOrElse("qrCodeMargin", 0).asInstanceOf[Int], - imageConfigMap.getOrElse("textFontName", "").asInstanceOf[String], - imageConfigMap.getOrElse("textFontSize", 0).asInstanceOf[Int], - imageConfigMap.getOrElse("textCharacterSpacing", 0).asInstanceOf[Double], - imageConfigMap.getOrElse("imageFormat", config.imageFormat).asInstanceOf[String], - imageConfigMap.getOrElse("colourModel", "").asInstanceOf[String], - imageConfigMap.getOrElse("imageBorderSize", 0).asInstanceOf[Int], - imageConfigMap.getOrElse("qrCodeMarginBottom", config.imageMarginBottom).asInstanceOf[Int], - imageConfigMap.getOrElse("imageMargin", config.imageMargin).asInstanceOf[Int] - ) - } - -} diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/domain/Models.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/domain/Models.scala deleted file mode 100644 index 243276f13..000000000 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/domain/Models.scala +++ /dev/null @@ -1,17 +0,0 @@ -package org.sunbird.job.qrimagegenerator.domain - -case class ImageConfig(errorCorrectionLevel: String, - pixelsPerBlock: Int, - qrCodeMargin: Int, - textFontName: String, - textFontSize: Int, - textCharacterSpacing: Double, - imageFormat: String, - colourModel: String, - imageBorderSize: Int, - qrCodeMarginBottom: Int, - imageMargin: Int) - -case class QRCodeImageGeneratorRequest(dialCodes: List[Map[String, AnyRef]], - imageConfig: ImageConfig, - tempFilePath: String) diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala deleted file mode 100644 index 348ab95f2..000000000 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala +++ /dev/null @@ -1,144 +0,0 @@ -package org.sunbird.job.qrimagegenerator.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.qrimagegenerator.domain.{Event, ImageConfig, QRCodeImageGeneratorRequest} -import org.sunbird.job.qrimagegenerator.task.QRCodeImageGeneratorConfig -import org.sunbird.job.qrimagegenerator.util.QRCodeImageGeneratorUtil -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, ElasticSearchUtil, FileUtils} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.io.File -import scala.collection.mutable.ListBuffer - -class QRCodeImageGeneratorFunction(config: QRCodeImageGeneratorConfig, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var esUtil: ElasticSearchUtil = null, - @transient var qRCodeImageGeneratorUtil: QRCodeImageGeneratorUtil = null) - (implicit val stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) { - - private val logger = LoggerFactory.getLogger(classOf[QRCodeImageGeneratorFunction]) - - override def open(parameters: Configuration): Unit = { - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - cloudStorageUtil = new CloudStorageUtil(config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) - qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil) - super.open(parameters) - } - - override def close(): Unit = { - cassandraUtil.close() - super.close() - } - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount, config.dbFailureEventCount, - config.dbHitEventCount, config.cloudDbHitCount, config.cloudDbFailCount) - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - - val availableImages = ListBuffer[File]() - var zipFile: File = null - try { - logger.info("QRCodeImageGeneratorService:processMessage: Processing request for processId : " + event.processId + " and objectId: " + event.objectId) - logger.info("QRCodeImageGeneratorService:processMessage: Starting message processing at " + System.currentTimeMillis()) - if (event.isValid(config)) { - val tempFilePath = config.lpTempFileLocation - val imageConfig: ImageConfig = event.imageConfig(config) - - event.dialCodes.filter(f => !f.getOrElse("location", "").asInstanceOf[String].isBlank) - .foreach { dialcode => - try { - val fileName = dialcode("id").asInstanceOf[String] - val downloadUrl = dialcode("location").asInstanceOf[String] - val file: File = FileUtils.downloadFile(downloadUrl, tempFilePath) - val newFile: File = new File(s"""$tempFilePath${File.separator}$fileName.${imageConfig.imageFormat}""") - FileUtils.copyFile(file, newFile) - file.delete() - logger.info("QRCodeImageGeneratorService:processMessage: created file - " + newFile.getAbsolutePath) - metrics.incCounter(config.cloudDbHitCount) - availableImages += newFile - } catch { - case e: Exception => - metrics.incCounter(config.cloudDbFailCount) - throw new InvalidEventException(e.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), e) - } - } - logger.info("availableImages after W/0 Loc: " + availableImages) - - val dialCodes: List[Map[String, AnyRef]] = event.dialCodes.filter(dialcode => dialcode.getOrElse("location", "").asInstanceOf[String].isEmpty) - val qrGenRequest: QRCodeImageGeneratorRequest = QRCodeImageGeneratorRequest(dialCodes, imageConfig, config.lpTempFileLocation) - logger.info("QRCodeImageGeneratorRequest: " + qrGenRequest + " storageContainer: "+event.storageContainer+" storagePath: "+event.storagePath) - val generatedImages: ListBuffer[File] = qRCodeImageGeneratorUtil.createQRImages(qrGenRequest, event.storageContainer, event.storagePath, metrics) - - if (!event.processId.isBlank) { - val maxAllowedCharacter: Int = config.getInt("max_allowed_character_for_file_name", 120) - logger.info("QRCodeImageGeneratorService:processMessage: Generating zip for QR codes with processId " + event.processId) - val storageFileName = if (event.storageFileName.isBlank) event.processId else event.storageFileName - logger.info("QRCodeImageGeneratorService:processMessage: storageFileName " + storageFileName) - val qrZipFileName = if (storageFileName.length > maxAllowedCharacter) storageFileName.substring(0, maxAllowedCharacter) else storageFileName - logger.info("QRCodeImageGeneratorService:processMessage: qrZipFileName - " + qrZipFileName + " tempFilePath - "+tempFilePath) - - // Merge available and generated image list - generatedImages.foreach(f => availableImages += f) - - val zipFileName: String = tempFilePath + File.separator + qrZipFileName + ".zip" - logger.info("QRCodeImageGeneratorService:processMessage: zipFileName - " + zipFileName) - - val fileList: List[String] = availableImages.map(f => f.getName).toList - FileUtils.zipIt(zipFileName, fileList, tempFilePath) - - zipFile = new File(zipFileName) - logger.info("QRCodeImageGeneratorService:processMessage: event.storagePath - " + event.storagePath + " event.storageContainer - "+ event.storageContainer) - val zipDownloadUrl = cloudStorageUtil.uploadFile(event.storagePath.replace("/", ""), zipFile, Some(false), container = event.storageContainer) - logger.info("QRCodeImageGeneratorService:processMessage: zipDownloadUrl - " + zipDownloadUrl(1)) - logger.info("QRCodeImageGeneratorService:processMessage: config.cloudStorageEndpoint - " + config.cloudStorageEndpoint+" config.cloudStorageProxyHost - "+config.cloudStorageProxyHost) - metrics.incCounter(config.cloudDbHitCount) - if(config.cloudStorageEndpoint.nonEmpty){ - var newDownloadUrl = zipDownloadUrl(1).replaceAll(config.cloudStorageEndpoint, config.cloudStorageProxyHost) - logger.info("QRCodeImageGeneratorService:processMessage: newDownloadUrl - " + newDownloadUrl) - logger.info("QRCodeImageGeneratorService:processMessage: event - " + event) - qRCodeImageGeneratorUtil.updateCassandra(config.cassandraDialCodeBatchTable, 2, newDownloadUrl, "processid", event.processId, metrics) - } else { - qRCodeImageGeneratorUtil.updateCassandra(config.cassandraDialCodeBatchTable, 2, zipDownloadUrl(1), "processid", event.processId, metrics) - } - - } - else { - logger.info("QRCodeImageGeneratorService:processMessage: Skipping zip creation due to missing processId.") - } - - if(config.indexImageURL) - context.output(config.indexImageUrlOutTag, event) - logger.info("QRCodeImageGeneratorService:processMessage: Message processed successfully at " + System.currentTimeMillis) - } else { - logger.info("QRCodeImageGeneratorService: Eid other than BE_QR_IMAGE_GENERATOR or Dialcodes not present") - metrics.incCounter(config.skippedEventCount) - } - } catch { - case e: Exception => - e.printStackTrace() - qRCodeImageGeneratorUtil.updateCassandra(config.cassandraDialCodeBatchTable, 3, "", "processid", event.processId, metrics) - logger.info("QRCodeImageGeneratorService:CassandraUpdateFailure: " + e.getMessage) - metrics.incCounter(config.failedEventCount) - throw new InvalidEventException(e.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), e) - } finally { - if (null != zipFile) { - zipFile.delete() - } - availableImages.filter(imageFile => null != imageFile).foreach(imageFile => imageFile.delete()) - } - } - -} diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala deleted file mode 100644 index c5d2a4001..000000000 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala +++ /dev/null @@ -1,57 +0,0 @@ -package org.sunbird.job.qrimagegenerator.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.qrimagegenerator.domain.Event -import org.sunbird.job.qrimagegenerator.task.QRCodeImageGeneratorConfig -import org.sunbird.job.qrimagegenerator.util.QRCodeImageGeneratorUtil -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, ElasticSearchUtil, ScalaJsonUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import scala.collection.mutable - -class QRCodeIndexImageUrlFunction(config: QRCodeImageGeneratorConfig, - @transient var cassandraUtil: CassandraUtil = null, - @transient var cloudStorageUtil: CloudStorageUtil = null, - @transient var esUtil: ElasticSearchUtil = null, - @transient var qRCodeImageGeneratorUtil: QRCodeImageGeneratorUtil = null) - (implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[Event, String](config) { - - private val logger = LoggerFactory.getLogger(classOf[QRCodeIndexImageUrlFunction]) - - override def open(parameters: Configuration): Unit = { - cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) - qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil) - super.open(parameters) - } - - override def close(): Unit = { - cassandraUtil.close() - super.close() - } - - override def metricsList(): List[String] = { - List() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - event.dialCodes.foreach { dialcode => - try { - val text = dialcode("text").asInstanceOf[String] - qRCodeImageGeneratorUtil.indexImageInDocument(text)(esUtil, cassandraUtil) - } catch { - case e: Exception => e.printStackTrace() - throw new InvalidEventException(e.getMessage) - } - } - - } - -} diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/task/QRCodeImageGeneratorConfig.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/task/QRCodeImageGeneratorConfig.scala deleted file mode 100644 index 3fe4f5b10..000000000 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/task/QRCodeImageGeneratorConfig.scala +++ /dev/null @@ -1,65 +0,0 @@ -package org.sunbird.job.qrimagegenerator.task - -import com.typesafe.config.Config -import org.apache.commons.lang3.StringUtils -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.qrimagegenerator.domain.Event - -class QRCodeImageGeneratorConfig(override val config: Config) extends BaseJobConfig(config, "qrcode-image-generator") { - - // Job Configuration - val jobEnv: String = config.getString("job.env") - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - implicit val qrImageTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val dbHitEventCount = "db-hit-events-count" - val dbFailureEventCount = "db-failure-events-count" - val skippedEventCount = "skipped-events-count" - val cloudDbHitCount = "cloud-db-hit-events-count" - val cloudDbFailCount = "cloud-db-hit-failure-count" - - //Tags - val indexImageUrlOutTag: OutputTag[Event] = OutputTag[Event]("index-imageUrl") - - // ES Configs - val esConnectionInfo = config.getString("es.basePath") - - val dialcodeExternalIndex: String = if (config.hasPath("dialcode.index.name")) config.getString("dialcode.index.name") else "dialcode" - val dialcodeExternalIndexType: String = "dc" - - // Consumers - val eventConsumer = "qrcode-image-generator-consumer" - val qrCodeImageGeneratorFunction = "qrcode-image-generator-function" - - val configVersion = "1.0" - - val eid = "BE_QR_IMAGE_GENERATOR" - val lpTempFileLocation: String = if (config.hasPath("lp.tmp.file.location")) config.getString("lp.tmp.file.location") else "/tmp" - - // default image config - val imageFormat: String = if (config.hasPath("imageFormat")) config.getString("imageFormat") else "png" - val imageMarginBottom: Int = if (config.hasPath("qr.image.bottomMargin")) config.getInt("qr.image.bottomMargin") else 0 - val imageMargin: Int = if (config.hasPath("qr.image.margin")) config.getInt("qr.image.margin") else 1 - - //cassandra config - val cassandraHost: String = config.getString("lms-cassandra.host") - val cassandraPort: Int = config.getInt("lms-cassandra.port") - val cassandraKeyspace: String = config.getString("lms-cassandra.keyspace") - val cassandraDialCodeImageTable: String = config.getString("lms-cassandra.table.image") - val cassandraDialCodeBatchTable: String = config.getString("lms-cassandra.table.batch") - - val cloudStorageEndpoint: String = if (config.hasPath("cloud_storage_endpoint")) config.getString("cloud_storage_endpoint") else "" - val cloudStorageProxyHost: String = if (config.hasPath("cloud_storage_proxy_host")) config.getString("cloud_storage_endpoint") else "" - val indexImageURL: Boolean = if (config.hasPath("qr.image.indexImageUrl")) config.getBoolean("qr.image.indexImageUrl") else true -} diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/task/QRCodeImageGeneratorTask.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/task/QRCodeImageGeneratorTask.scala deleted file mode 100644 index 694f81e16..000000000 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/task/QRCodeImageGeneratorTask.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.qrimagegenerator.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.qrimagegenerator.domain.Event -import org.sunbird.job.qrimagegenerator.functions.{QRCodeImageGeneratorFunction, QRCodeIndexImageUrlFunction} -import org.sunbird.job.util.FlinkUtil - -import java.io.File - - -class QRCodeImageGeneratorTask(config: QRCodeImageGeneratorConfig, kafkaConnector: FlinkKafkaConnector) { - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - val streamTask = env.addSource(source) - .name(config.eventConsumer) - .uid(config.eventConsumer) - .rebalance - .process(new QRCodeImageGeneratorFunction(config)) - .setParallelism(config.kafkaConsumerParallelism) - .name(config.qrCodeImageGeneratorFunction) - .uid(config.qrCodeImageGeneratorFunction) - .setParallelism(config.parallelism) - - if(config.indexImageURL) - streamTask.getSideOutput(config.indexImageUrlOutTag).process(new QRCodeIndexImageUrlFunction(config)) - .name("index-imageUrl-process").uid("index-imageUrl-process").setParallelism(config.parallelism) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object QRCodeImageGeneratorTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("qrcode-image-generator.conf").withFallback(ConfigFactory.systemEnvironment())) - val qrCodeImageGeneratorConfig = new QRCodeImageGeneratorConfig(config) - val kafkaUtil = new FlinkKafkaConnector(qrCodeImageGeneratorConfig) - val task = new QRCodeImageGeneratorTask(qrCodeImageGeneratorConfig, kafkaUtil) - task.process() - } -} - -// $COVERAGE-ON$ diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/util/QRCodeImageGeneratorUtil.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/util/QRCodeImageGeneratorUtil.scala deleted file mode 100644 index 22aa842db..000000000 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/util/QRCodeImageGeneratorUtil.scala +++ /dev/null @@ -1,306 +0,0 @@ -package org.sunbird.job.qrimagegenerator.util - -import com.datastax.driver.core.Row -import com.datastax.driver.core.querybuilder.QueryBuilder -import com.google.zxing.client.j2se.BufferedImageLuminanceSource -import com.google.zxing.common.{BitMatrix, HybridBinarizer} -import com.google.zxing.qrcode.QRCodeWriter -import com.google.zxing.qrcode.decoder.ErrorCorrectionLevel -import com.google.zxing.{BarcodeFormat, EncodeHintType, NotFoundException, WriterException} -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.qrimagegenerator.domain.{ImageConfig, QRCodeImageGeneratorRequest} -import org.sunbird.job.qrimagegenerator.task.QRCodeImageGeneratorConfig -import org.sunbird.job.util._ - -import java.awt.font.TextAttribute -import java.awt.image.BufferedImage -import java.awt.{Color, Font, FontFormatException, Graphics2D, RenderingHints} -import java.io.{File, IOException, InputStream} -import java.util.UUID -import javax.imageio.ImageIO -import scala.collection.mutable -import scala.collection.mutable.ListBuffer - -class QRCodeImageGeneratorUtil(config: QRCodeImageGeneratorConfig, cassandraUtil: CassandraUtil, cloudStorageUtil: CloudStorageUtil, esUtil: ElasticSearchUtil) { - private val qrCodeWriter = new QRCodeWriter() - private val fontStore: java.util.Map[String, Font] = new java.util.HashMap[String, Font]() - private val logger = LoggerFactory.getLogger(classOf[QRCodeImageGeneratorUtil]) - val isrRelativePathEnabled = config.getBoolean("cloudstorage.metadata.replace_absolute_path", false) - - @throws[WriterException] - @throws[IOException] - @throws[NotFoundException] - @throws[FontFormatException] - def createQRImages(req: QRCodeImageGeneratorRequest, container: String, path: String, metrics: Metrics): ListBuffer[File] = { - val fileList = ListBuffer[File]() - val imageConfig: ImageConfig = req.imageConfig - val dialCodes: List[Map[String, AnyRef]] = req.dialCodes - - dialCodes.foreach { dialcode => - val data = dialcode("data").asInstanceOf[String] - val text = dialcode("text").asInstanceOf[String] - val fileName = dialcode("id").asInstanceOf[String] - logger.info("QRCodeImageGeneratorUtil:createQRImages: data - " + data+" text - "+text+" fileName - "+fileName) - var qrImage = generateBaseImage(data, imageConfig.errorCorrectionLevel, imageConfig.pixelsPerBlock, imageConfig.qrCodeMargin, imageConfig.colourModel) - if (null != text || !text.isBlank) { - val textImage = getTextImage(text, imageConfig.textFontName, imageConfig.textFontSize, imageConfig.textCharacterSpacing, imageConfig.colourModel) - qrImage = addTextToBaseImage(qrImage, textImage, imageConfig.colourModel, imageConfig.qrCodeMargin, imageConfig.pixelsPerBlock, imageConfig.qrCodeMarginBottom, imageConfig.imageMargin) - } - logger.info("QRCodeImageGeneratorUtil:createQRImages: qrImage - " + qrImage) - if (imageConfig.imageBorderSize > 0) drawBorder(qrImage, imageConfig.imageBorderSize, imageConfig.imageMargin) - val finalImageFile = new File(req.tempFilePath + File.separator + fileName + "." + imageConfig.imageFormat) - logger.info("QRCodeImageGeneratorUtil:createQRImages: creating file - " + finalImageFile.getAbsolutePath) - finalImageFile.createNewFile - ImageIO.write(qrImage, imageConfig.imageFormat, finalImageFile) - fileList += finalImageFile - try { - logger.info("QRCodeImageGeneratorUtil:createQRImages: path before - " + path) - logger.info("QRCodeImageGeneratorUtil:createQRImages: path after - " + path.replace("/", "")) - val imageDownloadUrl = cloudStorageUtil.uploadFile(path.replace("/", ""), finalImageFile, Some(false), container = container) - logger.info("QRCodeImageGeneratorUtil:createQRImages: imageDownloadUrl - " + imageDownloadUrl(1)) - if(config.cloudStorageEndpoint.nonEmpty){ - logger.info("QRCodeImageGeneratorUtil:createQRImages: config.cloudStorageEndpoint - " + config.cloudStorageEndpoint+" config.cloudStorageProxyHost - "+config.cloudStorageProxyHost) - var newDownloadUrl = imageDownloadUrl(1).replaceAll(config.cloudStorageEndpoint, config.cloudStorageProxyHost) - logger.info("QRCodeImageGeneratorService:processMessage: newDownloadUrl before - " + newDownloadUrl) - updateCassandra(config.cassandraDialCodeImageTable, 2, newDownloadUrl, "filename", fileName, metrics) - } else { - updateCassandra(config.cassandraDialCodeImageTable, 2, imageDownloadUrl(1), "filename", fileName, metrics) - } - - } catch { - case e: Exception => - metrics.incCounter(config.dbFailureEventCount) - e.printStackTrace() - logger.info("QRCodeImageGeneratorUtil: Failure while uploading download URL Query:: " + e.getMessage) - } - } - fileList - } - - def updateCassandra(table: String, status: Int, downloadURL: String, whereClauseKey: String, whereClauseValue: String, metrics: Metrics): Unit = { - val value = if (whereClauseKey.equalsIgnoreCase("processid")) UUID.fromString(whereClauseValue) else whereClauseValue - val updateQuery: String = QueryBuilder.update(config.cassandraKeyspace, table) - .`with`(QueryBuilder.set("status", status)) - .and(QueryBuilder.set("url", downloadURL)) - .where(QueryBuilder.eq(whereClauseKey, value)).toString - cassandraUtil.upsert(updateQuery) - metrics.incCounter(config.dbHitEventCount) - } - - @throws[NotFoundException] - private def addTextToBaseImage(qrImage: BufferedImage, textImage: BufferedImage, colorModel: String, qrMargin: Int, pixelsPerBlock: Int, qrMarginBottom: Int, imageMargin: Int) = { - val qrSource = new BufferedImageLuminanceSource(qrImage) - val qrBinarizer = new HybridBinarizer(qrSource) - var qrBits = qrBinarizer.getBlackMatrix - val textSource = new BufferedImageLuminanceSource(textImage) - val textBinarizer = new HybridBinarizer(textSource) - var textBits = textBinarizer.getBlackMatrix - if (qrBits.getWidth > textBits.getWidth) { - val tempTextMatrix = new BitMatrix(qrBits.getWidth, textBits.getHeight) - copyMatrixDataToBiggerMatrix(textBits, tempTextMatrix) - textBits = tempTextMatrix - } - else if (qrBits.getWidth < textBits.getWidth) { - val tempQrMatrix = new BitMatrix(textBits.getWidth, qrBits.getHeight) - copyMatrixDataToBiggerMatrix(qrBits, tempQrMatrix) - qrBits = tempQrMatrix - } - val mergedMatrix = mergeMatricesOfSameWidth(qrBits, textBits, qrMargin, pixelsPerBlock, qrMarginBottom, imageMargin) - getImage(mergedMatrix, colorModel) - } - - @throws[WriterException] - private def generateBaseImage(data: String, errorCorrectionLevel: String, pixelsPerBlock: Int, qrMargin: Int, colorModel: String) = { - val hintsMap = getHintsMap(errorCorrectionLevel, qrMargin) - val defaultBitMatrix = getDefaultBitMatrix(data, hintsMap) - logger.info(s"defaultBitMatrix: width= ${defaultBitMatrix.getWidth}, height= ${defaultBitMatrix.getHeight}") - val largeBitMatrix = getBitMatrix(data, defaultBitMatrix.getWidth * pixelsPerBlock, defaultBitMatrix.getHeight * pixelsPerBlock, hintsMap) - logger.info(s"largeBitMatrix: width= ${largeBitMatrix.getWidth}, height= ${largeBitMatrix.getHeight}") - getImage(largeBitMatrix, colorModel) - } - - //To remove extra spaces between text and qrcode, margin below qrcode is removed - //Parameter, qrCodeMarginBottom, is introduced to add custom margin(in pixels) between qrcode and text - //Parameter, imageMargin is introduced, to add custom margin(in pixels) outside the black border of the image - private def mergeMatricesOfSameWidth(firstMatrix: BitMatrix, secondMatrix: BitMatrix, qrMargin: Int, pixelsPerBlock: Int, qrMarginBottom: Int, imageMargin: Int) = { - val mergedWidth = firstMatrix.getWidth + (2 * imageMargin) - val mergedHeight = firstMatrix.getHeight + secondMatrix.getHeight + (2 * imageMargin) - val defaultBottomMargin = pixelsPerBlock * qrMargin - val marginToBeRemoved = if (qrMarginBottom > defaultBottomMargin) 0 - else defaultBottomMargin - qrMarginBottom - val mergedMatrix = new BitMatrix(mergedWidth, mergedHeight - marginToBeRemoved) - for (x <- 0 until firstMatrix.getWidth) { - for (y <- 0 until firstMatrix.getHeight) { - if (firstMatrix.get(x, y)) mergedMatrix.set(x + imageMargin, y + imageMargin) - } - } - - for (x <- 0 until secondMatrix.getWidth) { - for (y <- 0 until secondMatrix.getHeight) { - if (secondMatrix.get(x, y)) mergedMatrix.set(x + imageMargin, y + firstMatrix.getHeight - marginToBeRemoved + imageMargin) - } - } - mergedMatrix - } - - private def copyMatrixDataToBiggerMatrix(fromMatrix: BitMatrix, toMatrix: BitMatrix): Unit = { - val widthDiff = toMatrix.getWidth - fromMatrix.getWidth - val leftMargin = widthDiff / 2 - - for (x <- 0 until fromMatrix.getWidth) { - for (y <- 0 until fromMatrix.getHeight) { - if (fromMatrix.get(x, y)) toMatrix.set(x + leftMargin, y) - } - } - } - - private def drawBorder(image: BufferedImage, borderSize: Int, imageMargin: Int): Unit = { - image.createGraphics - val graphics = image.getGraphics.asInstanceOf[Graphics2D] - graphics.setColor(Color.BLACK) - for (i <- 0 until borderSize) { - graphics.drawRect(i + imageMargin, i + imageMargin, image.getWidth - 1 - (2 * i) - (2 * imageMargin), image.getHeight - 1 - (2 * i) - (2 * imageMargin)) - } - graphics.dispose() - } - - private def getImage(bitMatrix: BitMatrix, colorModel: String) = { - val imageWidth = bitMatrix.getWidth - val imageHeight = bitMatrix.getHeight - val image = new BufferedImage(imageWidth, imageHeight, getImageType(colorModel)) - image.createGraphics() - val graphics = image.getGraphics.asInstanceOf[Graphics2D] - graphics.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_OFF) - graphics.setColor(Color.WHITE) - graphics.fillRect(0, 0, imageWidth, imageHeight) - graphics.setColor(Color.BLACK) - logger.info(s"imageWidth: $imageWidth, imageHeight: $imageHeight") - for (i <- 0 until imageWidth) { - for (j <- 0 until imageHeight) { - if (bitMatrix.get(i, j)) { - graphics.fillRect(i, j, 1, 1) - } - } - } - graphics.dispose() - image - } - - @throws[WriterException] - private def getBitMatrix(data: String, width: Int, height: Int, hintsMap: java.util.Map[_, _]) = { - qrCodeWriter.encode(data, BarcodeFormat.QR_CODE, width, height, hintsMap.asInstanceOf[java.util.Map[EncodeHintType, _]]) - } - - @throws[WriterException] - private def getDefaultBitMatrix(data: String, hintsMap: java.util.Map[_, _]) = { - qrCodeWriter.encode(data, BarcodeFormat.QR_CODE, 0, 0, hintsMap.asInstanceOf[java.util.Map[EncodeHintType, _]]) - } - - private def getHintsMap(errorCorrectionLevel: String, qrMargin: Int) = { - val hintsMap = new java.util.HashMap[EncodeHintType, AnyRef]() - errorCorrectionLevel match { - case "H" => hintsMap.put(EncodeHintType.ERROR_CORRECTION, ErrorCorrectionLevel.H) - case "Q" => hintsMap.put(EncodeHintType.ERROR_CORRECTION, ErrorCorrectionLevel.Q) - case "M" => hintsMap.put(EncodeHintType.ERROR_CORRECTION, ErrorCorrectionLevel.M) - case "L" => hintsMap.put(EncodeHintType.ERROR_CORRECTION, ErrorCorrectionLevel.L) - } - hintsMap.put(EncodeHintType.MARGIN, qrMargin.asInstanceOf[AnyRef]) - hintsMap - } - - //Sample = 2A42UH , Verdana, 11, 0.1, Grayscale - @throws[IOException] - @throws[FontFormatException] - private def getTextImage(text: String, fontName: String, fontSize: Int, tracking: Double, colorModel: String) = { - var image = new BufferedImage(1, 1, getImageType(colorModel)) - val basicFont = getFontFromStore(fontName) - val attributes: java.util.Map[TextAttribute, Any] = new java.util.HashMap[TextAttribute, Any] - attributes.put(TextAttribute.TRACKING, tracking) - attributes.put(TextAttribute.WEIGHT, TextAttribute.WEIGHT_BOLD) - attributes.put(TextAttribute.SIZE, fontSize) - logger.info("attributes: " + attributes) - val font = basicFont.deriveFont(attributes) - var graphics2d = image.createGraphics - graphics2d.setFont(font) - var fontmetrics = graphics2d.getFontMetrics - val width = fontmetrics.stringWidth(text) - val height = fontmetrics.getHeight - graphics2d.dispose() - image = new BufferedImage(width, height, getImageType(colorModel)) - graphics2d = image.createGraphics - graphics2d.setRenderingHint(RenderingHints.KEY_ALPHA_INTERPOLATION, RenderingHints.VALUE_ALPHA_INTERPOLATION_QUALITY) - graphics2d.setRenderingHint(RenderingHints.KEY_COLOR_RENDERING, RenderingHints.VALUE_COLOR_RENDER_QUALITY) - graphics2d.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_OFF) - graphics2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_OFF) - graphics2d.setColor(Color.WHITE) - graphics2d.fillRect(0, 0, image.getWidth, image.getHeight) - graphics2d.setColor(Color.BLACK) - graphics2d.setFont(font) - fontmetrics = graphics2d.getFontMetrics - graphics2d.drawString(text, 0, fontmetrics.getAscent) - graphics2d.dispose() - image - } - - private def getImageType(colorModel: String) = if (colorModel.equalsIgnoreCase("RGB")) BufferedImage.TYPE_INT_RGB - else BufferedImage.TYPE_BYTE_GRAY - - @throws[IOException] - @throws[FontFormatException] - //load the packaged font file from the root dir - private def loadFontStore(fontName: String): Font = { - val fontFile = fontName + ".ttf" - logger.info("fontFile: " + fontFile) - var basicFont: Font = null - var fontStream: InputStream = null - val classLoader: ClassLoader = this.getClass.getClassLoader - try { - fontStream = classLoader.getResourceAsStream(fontFile) - logger.info("input stream value is not null for fontfile " + fontFile + " " + fontStream) - basicFont = Font.createFont(Font.TRUETYPE_FONT, fontStream) - } catch { - case e: Exception => - e.printStackTrace() - logger.debug("Exception occurred during font creation " + e.getMessage) - } - fontStore.put(fontName, basicFont) - basicFont - } - - @throws[IOException] - @throws[FontFormatException] - private def getFontFromStore(fontName: String): Font = { - fontStore.getOrDefault(fontName, loadFontStore(fontName)) - } - - - def indexImageInDocument(id: String)(esUtil: ElasticSearchUtil, cassandraUtil: CassandraUtil): Unit = { - val documentJson: String = esUtil.getDocumentAsString(id) - val indexDocument = if (documentJson != null && documentJson.nonEmpty) ScalaJsonUtil.deserialize[mutable.Map[String, AnyRef]](documentJson) else mutable.Map[String, AnyRef]() - logger.info("QRCodeImageGeneratorUtil::indexImageInDocument:: indexDocument:: " + indexDocument) - if(indexDocument!=null && indexDocument.nonEmpty && !indexDocument.contains("url")) { - val query = QueryBuilder.select("url").from(config.cassandraKeyspace, config.cassandraDialCodeImageTable) - .allowFiltering() - .where(QueryBuilder.eq("dialcode", id)) - logger.info("QRCodeImageGeneratorUtil::indexImageInDocument:: query:: " + query) - val row: Row = cassandraUtil.findOne(query.toString) - if(null != row && !row.isNull("url")) { - val imageUrl = row.getString("url") - logger.info("QRCodeImageGeneratorUtil::indexImageInDocument:: imageUrl:: " + imageUrl) - val absoluteImageUrl = if (isrRelativePathEnabled) CSPMetaUtil.updateAbsolutePath(imageUrl)(config) else imageUrl - logger.info("QRCodeImageGeneratorUtil::indexImageInDocument:: absoluteImageUrl:: " + absoluteImageUrl) - val updatedDocString = ScalaJsonUtil.serialize(indexDocument + ("imageUrl" -> absoluteImageUrl)) - logger.info("QRCodeImageGeneratorUtil:indexImageInDocument: updatedDocString:: " + updatedDocString) - esUtil.updateDocument(id, updatedDocString) - } - } else { - throw new InvalidInputException("ElasticSearch Document not found for " + id) - } - - } - - -} - diff --git a/qrcode-image-generator/src/test/resources/logback-test.xml b/qrcode-image-generator/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/qrcode-image-generator/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/qrcode-image-generator/src/test/resources/test.conf b/qrcode-image-generator/src/test/resources/test.conf deleted file mode 100644 index 8816c0472..000000000 --- a/qrcode-image-generator/src/test/resources/test.conf +++ /dev/null @@ -1,35 +0,0 @@ -include "base-test.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.qrimage.request" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - window.time = 60 -} - -lms-cassandra { - keyspace = "dialcodes" - table { - image = "dialcode_images" - batch = "dialcode_batch" - } -} - -lp.tmp.file.location="/tmp" - -qr.image { - imageFormat="png" - bottomMargin=0 - margin=1 -} - -cloud_storage_endpoint="https://example.com" │ -cloud_storage_proxy_host="https://example.com" -indexImageUrl = false diff --git a/qrcode-image-generator/src/test/resources/test.cql b/qrcode-image-generator/src/test/resources/test.cql deleted file mode 100644 index 6bd783335..000000000 --- a/qrcode-image-generator/src/test/resources/test.cql +++ /dev/null @@ -1,25 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS dialcodes WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': '2'} AND durable_writes = true; - -CREATE TABLE IF NOT EXISTS dialcodes.dialcode_images ( - filename text PRIMARY KEY, - channel text, - config map, - created_on timestamp, - dialcode text, - publisher text, - status int, - url text -); - -CREATE TABLE IF NOT EXISTS dialcodes.dialcode_batch ( - processid uuid PRIMARY KEY, - channel text, - config map, - created_on timestamp, - dialcodes list, - publisher text, - status int, - url text -); - -INSERT INTO dialcodes.dialcode_images(filename, channel, dialcode, url) VALUES ('0_Q1I5I3', 'b00bc992ef25f1a9a8d63291e20efc8d', 'Q1I5I3', 'https://sunbirddev.blob.core.windows.net/sunbird-content-dev/in.ekstep/0_Q1I5I3.png') ; diff --git a/qrcode-image-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/qrcode-image-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index 75077b28d..000000000 --- a/qrcode-image-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,20 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"eid":"BE_QR_IMAGE_GENERATOR","processId":"e101e2cf-d592-4095-89fe-38acc0b8b276","objectId":"do_2132430402739240961409","dialcodes":[{"data":"https://preprod.ntp.net.in/dial/N3X6Y3","text":"N3X6Y3","id":"2_N3X6Y3"},{"data":"https://preprod.ntp.net.in/dial/U3J1J9","text":"U3J1J9","id":"0_U3J1J9"},{"data":"https://preprod.ntp.net.in/dial/J9Z5R4","text":"J9Z5R4","id":"1_J9Z5R4"}],"storage":{"container":"dial","path":"01269878797503692810/","fileName":"do_2132430402739240961409_english_class_10_english_1616582317028"},"config":{"errorCorrectionLevel":"H","pixelsPerBlock":2,"qrCodeMargin":3,"textFontName":"Verdana","textFontSize":11,"textCharacterSpacing":0.1,"imageFormat":"png","colourModel":"Grayscale","imageBorderSize":1}} - |""".stripMargin - - val EVENT_2: String = - """ - |{"eid":"BE_QR_IMAGE_GENERATOR","processId":"d36c130b-d5cb-4bec-9224-734d1d89bb00","objectId":"do_2132491902071767041339","dialcodes":[{"data":"https://preprod.ntp.net.in/dial/V2B5A2","text":"V2B5A2","id":"1_V2B5A2","location":"https://preprod.ntp.net.in"},{"data":"https://preprod.ntp.net.in/dial/F6J3E7","text":"F6J3E7","id":"0_F6J3E7","location":"https://preprod.ntp.net.in"}],"storage":{"container":"dial","path":"01272777697873100812/","fileName":"do_2132491902071767041339_sanskrit_class_12_home_science_1617332855179"},"config":{"errorCorrectionLevel":"H","pixelsPerBlock":2,"qrCodeMargin":3,"textFontName":"Verdana","textFontSize":11,"textCharacterSpacing":0.1,"imageFormat":"png","colourModel":"Grayscale","imageBorderSize":1}} - |""".stripMargin - - // storageFileName length too long - val EVENT_3: String = - """ - |{"eid":"BE_QR_IMAGE_GENERATOR","processId":"d36c130b-d5cb-4bec-9224-734d1d89bb00","objectId":"do_2132491902071767041339","dialcodes":[{"data":"https://preprod.ntp.net.in/dial/V2B5A2","text":"V2B5A2","id":"1_V2B5A2","location":"https://preprod.ntp.net.in"},{"data":"https://preprod.ntp.net.in/dial/F6J3E7","text":"F6J3E7","id":"0_F6J3E7","location":"https://preprod.ntp.net.in"}],"storage":{"container":"dial","path":"01272777697873100812/","fileName":"do_2132491902071767041339_kannada_english_marathi_hindi_sanskrit_telugu_gujarati_odia_urdu_bengali_tamil_punjabi_malayalam_cpd_class_10_class_11_class_1_class_12_class_2_class_3_class_4_class_5_class_6_class_7_class_8_class_9_others_cbse_training_mathematics_cpd_social_science_information_and_communication_technology_urdu_ict_political_science_civics_geography_heritage_crafts_biology_economics_health_and_physical_education_sociology_chemistry_science_home_science_informatics_practices_graphic_design_business_studies_creative_writing_and_translation_physics_education_computer_science_sanskrit_workbook_english_workbook_statistics_sanskrit_accountancy_english_political_science_hindi_graphics_design_psychology_environmental_studies_history_fine_arts_1617332855179"},"config":{"errorCorrectionLevel":"H","pixelsPerBlock":2,"qrCodeMargin":3,"textFontName":"Verdana","textFontSize":11,"textCharacterSpacing":0.1,"imageFormat":"png","colourModel":"Grayscale","imageBorderSize":1}} - |""".stripMargin -} \ No newline at end of file diff --git a/qrcode-image-generator/src/test/scala/org/sunbird/job/spec/QRCodeImageGeneratorTaskTestSpec.scala b/qrcode-image-generator/src/test/scala/org/sunbird/job/spec/QRCodeImageGeneratorTaskTestSpec.scala deleted file mode 100644 index 19310cc29..000000000 --- a/qrcode-image-generator/src/test/scala/org/sunbird/job/spec/QRCodeImageGeneratorTaskTestSpec.scala +++ /dev/null @@ -1,91 +0,0 @@ -package org.sunbird.job.spec - -import java.util -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.qrimagegenerator.domain.Event -import org.sunbird.job.qrimagegenerator.task.{QRCodeImageGeneratorConfig, QRCodeImageGeneratorTask} -import org.sunbird.job.util.{CassandraUtil, CloudStorageUtil, ElasticSearchUtil, JSONUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -class QRCodeImageGeneratorTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: QRCodeImageGeneratorConfig = new QRCodeImageGeneratorConfig(config) - val cloudStorageUtil:CloudStorageUtil = new CloudStorageUtil(jobConfig) - var cassandraUtils: CassandraUtil = _ - val mockElasticUtil: ElasticSearchUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - var currentMilliSecond = 1605816926271L - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtils = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtils.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - flinkCluster.after() - super.afterAll() - } - - - ignore should "generate event" in { - - val N3X6Y3Json = """{"identifier":"N3X6Y3", "filename":"2_N3X6Y3", "channel":"b00bc992ef25f1a9a8d63291e20efc8d"}""" - val U3J1J9Json = """{"identifier":"U3J1J9", "filename":"0_U3J1J9", "channel":"b00bc992ef25f1a9a8d63291e20efc8d"}""" - val V2B5A2Json = """{"identifier":"V2B5A2", "filename":"1_V2B5A2", "channel":"b00bc992ef25f1a9a8d63291e20efc8d"}""" - val F6J3E7Json = """{"identifier":"F6J3E7", "filename":"0_F6J3E7", "channel":"b00bc992ef25f1a9a8d63291e20efc8d"}""" - val Q1I5I3Json = """{"identifier":"Q1I5I3", "filename":"0_Q1I5I3", "channel":"b00bc992ef25f1a9a8d63291e20efc8d"}""" - when(mockElasticUtil.getDocumentAsString("Q1I5I3")).thenReturn(Q1I5I3Json) - when(mockElasticUtil.getDocumentAsString("N3X6Y3")).thenReturn(N3X6Y3Json) - when(mockElasticUtil.getDocumentAsString("U3J1J9")).thenReturn(U3J1J9Json) - when(mockElasticUtil.getDocumentAsString("V2B5A2")).thenReturn(V2B5A2Json) - when(mockElasticUtil.getDocumentAsString("F6J3E7")).thenReturn(F6J3E7Json) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new QRCodeImageGeneratorMapSource) - new QRCodeImageGeneratorTask(jobConfig, mockKafkaUtil).process() -// assertThrows[JobExecutionException] { -// new QRCodeImageGeneratorTask(jobConfig, mockKafkaUtil).process() -// } - - } -} - -class QRCodeImageGeneratorMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]): Unit = { - // Valid event - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1), 0, 10)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2), 0, 11)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3), 0, 12)) - // Invalid event -// ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_4))) - } - - override def cancel(): Unit = {} -} \ No newline at end of file diff --git a/qrcode-image-generator/src/test/scala/org/sunbird/job/util/QRCodeImageGeneratorUtilSpec.scala b/qrcode-image-generator/src/test/scala/org/sunbird/job/util/QRCodeImageGeneratorUtilSpec.scala deleted file mode 100644 index 8920cf4f0..000000000 --- a/qrcode-image-generator/src/test/scala/org/sunbird/job/util/QRCodeImageGeneratorUtilSpec.scala +++ /dev/null @@ -1,55 +0,0 @@ -package org.sunbird.job.util - -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.scalatestplus.mockito.MockitoSugar -import org.sunbird.job.exception.InvalidInputException -import org.sunbird.job.qrimagegenerator.task.QRCodeImageGeneratorConfig -import org.sunbird.job.qrimagegenerator.util.QRCodeImageGeneratorUtil - -class QRCodeImageGeneratorUtilSpec extends FlatSpec with BeforeAndAfterAll with Matchers with MockitoSugar { - - val config: Config = ConfigFactory.load("test.conf").withFallback(ConfigFactory.systemEnvironment()) - val jobConfig: QRCodeImageGeneratorConfig = new QRCodeImageGeneratorConfig(config) - implicit val mockNeo4JUtil: Neo4JUtil = mock[Neo4JUtil](Mockito.withSettings().serializable()) - var cassandraUtil: CassandraUtil = _ - implicit val mockCloudUtil: CloudStorageUtil = mock[CloudStorageUtil](Mockito.withSettings().serializable()) - val mockElasticUtil: ElasticSearchUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - - override protected def beforeAll(): Unit = { - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.cassandraHost, jobConfig.cassandraPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session) - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) - - super.beforeAll() - } - - override protected def afterAll(): Unit = { - cassandraUtil.close() - super.afterAll() - - } - - "QRCodeImageGeneratorFunction" should "return QR Code Document" in { - val qrCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(jobConfig, cassandraUtil, mockCloudUtil, mockElasticUtil) - assertThrows[InvalidInputException] { - qrCodeImageGeneratorUtil.indexImageInDocument("Q1I5I3")(mockElasticUtil, cassandraUtil) - } - val Q1I5I3Json = """{"identifier":"Q1I5I3", "filename":"0_Q1I5I3", "channel":"b00bc992ef25f1a9a8d63291e20efc8d"}""" - when(mockElasticUtil.getDocumentAsString("Q1I5I3")).thenReturn(Q1I5I3Json) - qrCodeImageGeneratorUtil.indexImageInDocument("Q1I5I3")(mockElasticUtil, cassandraUtil) - } - - -} - - - - diff --git a/search-indexer/pom.xml b/search-indexer/pom.xml deleted file mode 100644 index a865f9b0b..000000000 --- a/search-indexer/pom.xml +++ /dev/null @@ -1,209 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - search-indexer - 1.0.0 - jar - search-indexer - - Search Indexer Flink Job - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - pl.allegro.tech - embedded-elasticsearch - 2.7.0 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.searchindexer.task.SearchIndexerStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - search-indexer-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/search-indexer/src/main/resources/log4j.properties b/search-indexer/src/main/resources/log4j.properties deleted file mode 100644 index 4a2ed773e..000000000 --- a/search-indexer/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=post-publish-processor.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/search-indexer/src/main/resources/search-indexer.conf b/search-indexer/src/main/resources/search-indexer.conf deleted file mode 100644 index 904bd96ca..000000000 --- a/search-indexer/src/main/resources/search-indexer.conf +++ /dev/null @@ -1,33 +0,0 @@ -include "base-config.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.learning.graph.events" - error.topic = "sunbirddev.learning.events.failed" - groupId = "local-search-indexer-group" -} - -task { - consumer.parallelism = 1 - router.parallelism = 1 - compositeSearch.parallelism = 1 - dialcodeIndexer.parallelism = 1 - dialcodemetricsIndexer.parallelism = 1 -} - -compositesearch.index.name = "compositesearch" -nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions"] -schema.definition_cache.expiry = 14400 -restrict { - metadata.objectTypes = [] - objectTypes = ["EventSet", "Questionnaire", "Misconception", "FrameworkType", "EventSet", "Event"] -} - -cloudstorage.metadata.replace_absolute_path=false -cloudstorage.relative_path_prefix= "CONTENT_STORAGE_BASE_PATH" -cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net" -cloudstorage.mecloudstorage.metadata.list=["appIcon","posterImage","artifactUrl","downloadUrl","variants","previewUrl","pdfUrl", "streamingUrl", "toc_url"] -cloud_storage_container="sunbird-content-dev" \ No newline at end of file diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/domain/Event.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/domain/Event.scala deleted file mode 100644 index 57e28aaa0..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/domain/Event.scala +++ /dev/null @@ -1,34 +0,0 @@ -package org.sunbird.job.searchindexer.compositesearch.domain - -import org.apache.commons.lang3.BooleanUtils -import org.sunbird.job.domain.reader.JobRequest - -import java.util - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - val jobName = "SearchIndexer" - - def index: Boolean = { - val index = eventMap.get("index") - if (index == null) true else { - index match { - case bool: Boolean => bool - case _ => BooleanUtils.toBoolean(index.toString) - } - } - } - - def operationType: String = readOrDefault("operationType", "") - - def id: String = readOrDefault("nodeUniqueId", "") - - def nodeType: String = readOrDefault("nodeType", "") - - def objectType: String = readOrDefault("objectType", "") - - def validEvent(restrictObjectTypes: util.List[String]): Boolean = { - (operationType != null) && index && (!restrictObjectTypes.contains(objectType)) - } - -} diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala deleted file mode 100644 index 0915868e3..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala +++ /dev/null @@ -1,126 +0,0 @@ -package org.sunbird.job.searchindexer.compositesearch.helpers - -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import org.sunbird.job.searchindexer.models.CompositeIndexer -import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} - -import scala.collection.JavaConverters._ - -trait CompositeSearchIndexerHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[CompositeSearchIndexerHelper]) - - def createCompositeSearchIndex()(esUtil: ElasticSearchUtil): Boolean = { - val settings = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1500"}},"analysis":{"filter":{"mynGram":{"token_chars":["letter","digit","whitespace","punctuation","symbol"],"min_gram":"1","type":"nGram","max_gram":"30"}},"analyzer":{"cs_index_analyzer":{"filter":["lowercase","mynGram"],"type":"custom","tokenizer":"standard"},"keylower":{"filter":"lowercase","tokenizer":"keyword"},"cs_search_analyzer":{"filter":["standard","lowercase"],"type":"custom","tokenizer":"standard"}}}}""" - val mappings = """{"dynamic_templates":[{"nested":{"match_mapping_type":"object","mapping":{"type":"nested","fields":{"type":"nested"}}}},{"longs":{"match_mapping_type":"long","mapping":{"type":"long","fields":{"raw":{"type":"long"}}}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"type":"boolean","fields":{"raw":{"type":"boolean"}}}}},{"doubles":{"match_mapping_type":"double","mapping":{"type":"double","fields":{"raw":{"type":"double"}}}}},{"dates":{"match_mapping_type":"date","mapping":{"type":"date","fields":{"raw":{"type":"date"}}}}},{"strings":{"match_mapping_type":"string","mapping":{"type":"text","copy_to":"all_fields","analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}],"properties":{"screenshots":{"type":"text","index":false},"body":{"type":"text","index":false},"appIcon":{"type":"text","index":false},"all_fields":{"type":"text","analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}""" - esUtil.addIndex(settings, mappings) - } - - private def getIndexDocument(identifier: String)(esUtil: ElasticSearchUtil): scala.collection.mutable.Map[String, AnyRef] = { - val documentJson: String = esUtil.getDocumentAsString(identifier) - val indexDocument = if (documentJson != null && documentJson.nonEmpty) ScalaJsonUtil.deserialize[scala.collection.mutable.Map[String, AnyRef]](documentJson) else scala.collection.mutable.Map[String, AnyRef]() - indexDocument - } - - def getIndexDocument(message: Map[String, Any], isUpdate: Boolean, definition: ObjectDefinition, nestedFields: List[String], ignoredFields: List[String])(esUtil: ElasticSearchUtil): Map[String, AnyRef] = { - val identifier = message.getOrElse("nodeUniqueId", "").asInstanceOf[String] - val indexDocument = if (isUpdate) getIndexDocument(identifier)(esUtil) else scala.collection.mutable.Map[String, AnyRef]() - val transactionData = message.getOrElse("transactionData", Map[String, Any]()).asInstanceOf[Map[String, Any]] - - if (transactionData.nonEmpty) { - val addedProperties = transactionData.getOrElse("properties", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - addedProperties.foreach(property => { - if (!definition.externalProperties.contains(property._1)) { - val propertyNewValue: AnyRef = property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) match { - case propVal: List[AnyRef] => if(propVal.isEmpty) null else propVal - case _ => property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) - } - if (propertyNewValue == null) indexDocument.remove(property._1) else indexDocument.put(property._1, addMetadataToDocument(property._1, propertyNewValue, nestedFields)) - } - }) - - val addedRelations = transactionData.getOrElse("addedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - if (addedRelations.nonEmpty) { - addedRelations.foreach(rel => { - val direction = rel.getOrElse("dir", "").asInstanceOf[String] - val relationType = rel.getOrElse("rel", "").asInstanceOf[String] - val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] - val title = definition.relationLabel(targetObjType, direction, relationType) - if (title.nonEmpty) { - val list = indexDocument.getOrElse(title.get, List[String]()).asInstanceOf[List[String]] - val id = rel.getOrElse("id", "").asInstanceOf[String] - if (!list.contains(id)) indexDocument.put(title.get, (id :: list).asInstanceOf[AnyRef]) - } - }) - } - - val removedRelations = transactionData.getOrElse("removedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - removedRelations.foreach(rel => { - val direction = rel.getOrElse("dir", "").asInstanceOf[String] - val relationType = rel.getOrElse("rel", "").asInstanceOf[String] - val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] - val title = definition.relationLabel(targetObjType, direction, relationType) - if (title.nonEmpty) { - val list = indexDocument.getOrElse(title.get, List[String]()).asInstanceOf[List[String]] - val id = rel.getOrElse("id", "").asInstanceOf[String] - if (list.contains(id)) { - val updatedList = list diff List(id) - indexDocument.put(title.get, updatedList.asInstanceOf[AnyRef]) - } - } - }) - } - - //Ignored fields are removed-> it can be a propertyName or relation Name - indexDocument --= ignoredFields - - indexDocument.put("graph_id", message.getOrElse("graphId", "").asInstanceOf[String]) - indexDocument.put("node_id", message.getOrElse("nodeGraphId", 0).asInstanceOf[Integer]) - indexDocument.put("identifier", message.getOrElse("nodeUniqueId", "").asInstanceOf[String]) - indexDocument.put("objectType", message.getOrElse("objectType", "").asInstanceOf[String]) - indexDocument.put("nodeType", message.getOrElse("nodeType", "").asInstanceOf[String]) - indexDocument.toMap - } - - def upsertDocument(identifier: String, jsonIndexDocument: String)(esUtil: ElasticSearchUtil): Unit = { - esUtil.addDocument(identifier, jsonIndexDocument) - } - - def processESMessage(compositeObject: CompositeIndexer)(esUtil: ElasticSearchUtil, defCache: DefinitionCache): Unit = { - val definition = defCache.getDefinition(compositeObject.objectType, compositeObject.getVersionAsString(), compositeObject.getDefinitionBasePath()) - - val compositeMap = compositeObject.message.asScala.toMap - upsertDocument(compositeObject.identifier, compositeMap, definition, compositeObject.getNestedFields(), compositeObject.getIgnoredFields())(esUtil) - } - - - private def upsertDocument(identifier: String, message: Map[String, Any], definition: ObjectDefinition, nestedFields: List[String], ignoredFields: List[String])(esUtil: ElasticSearchUtil): Unit = { - val operationType = message.getOrElse("operationType", "").asInstanceOf[String] - operationType match { - case "CREATE" => - val indexDocument = getIndexDocument(message, false, definition, nestedFields, ignoredFields)(esUtil) - val jsonIndexDocument = ScalaJsonUtil.serialize(indexDocument) - upsertDocument(identifier, jsonIndexDocument)(esUtil) - case "UPDATE" => - val indexDocument = getIndexDocument(message, true, definition, nestedFields, ignoredFields)(esUtil) - val jsonIndexDocument = ScalaJsonUtil.serialize(indexDocument) - upsertDocument(identifier, jsonIndexDocument)(esUtil) - case "DELETE" => - val id = message.getOrElse("nodeUniqueId", "").asInstanceOf[String] - val indexDocument = getIndexDocument(id)(esUtil) - val visibility = indexDocument.getOrElse("visibility", "").asInstanceOf[String] - if (StringUtils.equalsIgnoreCase("Parent", visibility)) logger.info(s"Not deleting the document (visibility: Parent) with ID: $id") - else esUtil.deleteDocument(identifier) - case _ => - logger.info(s"Unknown Operation Type : $operationType for the identifier: $identifier.") - } - } - - private def addMetadataToDocument(propertyName: String, propertyValue: AnyRef, nestedFields: List[String]): AnyRef = { - val propertyNewValue = if (nestedFields.contains(propertyName)) ScalaJsonUtil.deserialize[AnyRef](propertyValue.asInstanceOf[String]) else propertyValue - propertyNewValue - } - -} diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala deleted file mode 100644 index 5b5050ede..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala +++ /dev/null @@ -1,61 +0,0 @@ -package org.sunbird.job.searchindexer.compositesearch.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} - -import scala.collection.mutable - -trait DIALCodeIndexerHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeIndexerHelper]) - - def createDialCodeIndex()(esUtil: ElasticSearchUtil): Boolean = { - val settings: String = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1050"}},"analysis":{"analyzer":{"dc_index_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","mynGram"]},"dc_search_analyzer":{"type":"custom","tokenizer":"standard","filter":["standard","lowercase"]},"keylower":{"tokenizer":"keyword","filter":"lowercase"}},"filter":{"mynGram":{"type":"nGram","min_gram":1,"max_gram":30,"token_chars":["letter","digit","whitespace","punctuation","symbol"]}}}}""" - val mappings: String = """{"dynamic_templates":[{"longs":{"match_mapping_type":"long","mapping":{"type":"long","fields":{"raw":{"type":"long"}}}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"type":"boolean","fields":{"raw":{"type":"boolean"}}}}},{"doubles":{"match_mapping_type":"double","mapping":{"type":"double","fields":{"raw":{"type":"double"}}}}},{"dates":{"match_mapping_type":"date","mapping":{"type":"date","fields":{"raw":{"type":"date"}}}}},{"strings":{"match_mapping_type":"string","mapping":{"type":"text","copy_to":"all_fields","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}],"properties":{"all_fields":{"type":"text","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}""" - esUtil.addIndex(settings, mappings) - } - - private def getIndexDocument(id: String)(esUtil: ElasticSearchUtil): mutable.Map[String, AnyRef] = { - val documentJson: String = esUtil.getDocumentAsString(id) - val indexDocument = if (documentJson != null && !documentJson.isEmpty) ScalaJsonUtil.deserialize[mutable.Map[String, AnyRef]](documentJson) else mutable.Map[String, AnyRef]() - indexDocument - } - - def getDocument(message: Map[String, Any], updateRequest: Boolean)(esUtil: ElasticSearchUtil): Map[String, AnyRef] = { - val identifier: String = message.getOrElse("nodeUniqueId", "").asInstanceOf[String] - val indexDocument = if (updateRequest) getIndexDocument(identifier)(esUtil) else mutable.Map[String, AnyRef]() - val transactionData: Map[String, AnyRef] = message.getOrElse("transactionData", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - if (!transactionData.isEmpty) { - val addedProperties: Map[String, AnyRef] = transactionData.getOrElse("properties", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - addedProperties.foreach(property => { - val propertyNewValue: AnyRef = property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) - if (propertyNewValue == null) indexDocument.remove(property._1) else indexDocument.put(property._1, propertyNewValue) - }) - } - indexDocument.put("identifier", message.getOrElse("nodeUniqueId", "").asInstanceOf[String]) - indexDocument.put("objectType", message.getOrElse("objectType", "").asInstanceOf[String]) - indexDocument.toMap - } - - private def upsertDocument(identifier: String, jsonIndexDocument: String)(esUtil: ElasticSearchUtil): Unit = { - esUtil.addDocument(identifier, jsonIndexDocument) - } - - def upsertDIALCodeDocument(identifier: String, message: Map[String, Any])(esUtil: ElasticSearchUtil): Unit = { - val operationType: String = message.getOrElse("operationType", "").asInstanceOf[String] - operationType match { - case "CREATE" => - val doc: Map[String, AnyRef] = getDocument(message, false)(esUtil) - val jsonDoc: String = ScalaJsonUtil.serialize(doc) - upsertDocument(identifier, jsonDoc)(esUtil) - case "UPDATE" => - val doc: Map[String, AnyRef] = getDocument(message, true)(esUtil) - val jsonDoc: String = ScalaJsonUtil.serialize(doc) - upsertDocument(identifier, jsonDoc)(esUtil) - case "DELETE" => - esUtil.deleteDocument(identifier) - case _ => - logger.info(s"Unknown Operation : ${operationType} for the identifier : ${identifier}.") - } - } -} diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala deleted file mode 100644 index 3c0a403af..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala +++ /dev/null @@ -1,61 +0,0 @@ -package org.sunbird.job.searchindexer.compositesearch.helpers - -import org.slf4j.LoggerFactory -import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} - -import scala.collection.mutable - -trait DIALCodeMetricsIndexerHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeMetricsIndexerHelper]) - - def createDialCodeIndex()(esUtil: ElasticSearchUtil): Boolean = { - val settings: String = """{"number_of_shards":5}""" - val mappings: String = """{"dcm":{"dynamic":false,"properties":{"dial_code":{"type":"keyword"},"total_dial_scans_local":{"type":"double"},"total_dial_scans_global":{"type":"double"},"average_scans_per_day":{"type":"double"},"last_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"first_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"}}}}""" - esUtil.addIndex(settings, mappings) - } - - private def getIndexDocument(id: String)(esUtil: ElasticSearchUtil): mutable.Map[String, AnyRef] = { - val documentJson: String = esUtil.getDocumentAsString(id) - val indexDocument = if (documentJson != null && !documentJson.isEmpty) ScalaJsonUtil.deserialize[mutable.Map[String, AnyRef]](documentJson) else mutable.Map[String, AnyRef]() - indexDocument - } - - def getIndexDocument(message: Map[String, Any], updateRequest: Boolean)(esUtil: ElasticSearchUtil): Map[String, AnyRef] = { - val identifier: String = message.getOrElse("nodeUniqueId", "").asInstanceOf[String] - val indexDocument = if (updateRequest) getIndexDocument(identifier)(esUtil) else mutable.Map[String, AnyRef]() - val transactionData: Map[String, AnyRef] = message.getOrElse("transactionData", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - if (!transactionData.isEmpty) { - val addedProperties: Map[String, AnyRef] = transactionData.getOrElse("properties", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - addedProperties.foreach(property => { - val propertyNewValue: AnyRef = property._2.asInstanceOf[Map[String, AnyRef]].getOrElse("nv", null) - if (propertyNewValue == null) indexDocument.remove(property._1) else indexDocument.put(property._1, propertyNewValue) - }) - } - indexDocument.put("dial_code", message.getOrElse("nodeUniqueId", "").asInstanceOf[String]) - indexDocument.put("objectType", message.getOrElse("objectType", "").asInstanceOf[String]) - indexDocument.toMap - } - - private def upsertDocument(identifier: String, jsonIndexDocument: String)(esUtil: ElasticSearchUtil): Unit = { - esUtil.addDocument(identifier, jsonIndexDocument) - } - - def upsertDialcodeMetricDocument(identifier: String, message: Map[String, Any])(esUtil: ElasticSearchUtil): Unit = { - val operationType: String = message.getOrElse("operationType", "").asInstanceOf[String] - operationType match { - case "CREATE" => - val indexDocument: Map[String, AnyRef] = getIndexDocument(message, false)(esUtil) - val jsonIndexDocument: String = ScalaJsonUtil.serialize(indexDocument) - upsertDocument(identifier, jsonIndexDocument)(esUtil) - case "UPDATE" => - val indexDocument: Map[String, AnyRef] = getIndexDocument(message, true)(esUtil) - val jsonIndexDocument: String = ScalaJsonUtil.serialize(indexDocument) - upsertDocument(identifier, jsonIndexDocument)(esUtil) - case "DELETE" => - esUtil.deleteDocument(identifier) - case _ => - logger.info(s"Unknown Operation : ${operationType} for the identifier: ${identifier}.") - } - } -} diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala deleted file mode 100644 index 998dce0f3..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala +++ /dev/null @@ -1,70 +0,0 @@ -package org.sunbird.job.searchindexer.functions - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.searchindexer.compositesearch.domain.Event -import org.sunbird.job.searchindexer.compositesearch.helpers.CompositeSearchIndexerHelper -import org.sunbird.job.searchindexer.models.CompositeIndexer -import org.sunbird.job.searchindexer.task.SearchIndexerConfig -import org.sunbird.job.util.{CSPMetaUtil, ElasticSearchUtil, ScalaJsonUtil} -import org.sunbird.job.{BaseProcessFunction, Metrics} - - -class CompositeSearchIndexerFunction(config: SearchIndexerConfig, - @transient var elasticUtil: ElasticSearchUtil = null) - extends BaseProcessFunction[Event, String](config) with CompositeSearchIndexerHelper with FailedEventHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[CompositeSearchIndexerFunction]) - lazy val defCache: DefinitionCache = new DefinitionCache() - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndex, config.compositeSearchIndexType) - createCompositeSearchIndex()(elasticUtil) - } - - override def close(): Unit = { - elasticUtil.close() - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.compositeSearchEventCount) - try { - val compositeObject = getCompositeIndexerObject(event) - processESMessage(compositeObject)(elasticUtil, defCache) - metrics.incCounter(config.successCompositeSearchEventCount) - } catch { - case ex: Throwable => - logger.error(s"Error while processing message for identifier : ${event.id}. Partition: ${event.partition} and Offset: ${event.offset}. Error : ", ex) - metrics.incCounter(config.failedCompositeSearchEventCount) - val failedEvent = getFailedEvent(event.jobName, event.getMap(), ex) - context.output(config.failedEventOutTag, failedEvent) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - def getCompositeIndexerObject(event: Event): CompositeIndexer = { - val objectType = event.readOrDefault("objectType", "") - val graphId = event.readOrDefault("graphId", "") - val uniqueId = event.readOrDefault("nodeUniqueId", "") - val messageId = event.readOrDefault("mid", "") - - val updateEvent: java.util.Map[String, Any] = if(config.isrRelativePathEnabled) { - val json = CSPMetaUtil.updateAbsolutePath(event.getJson())(config) - ScalaJsonUtil.deserialize[java.util.Map[String, Any]](json) - } else event.getMap() - - CompositeIndexer(graphId, objectType, uniqueId, messageId, updateEvent, config) - } - - override def metricsList(): List[String] = { - List(config.successCompositeSearchEventCount, config.failedCompositeSearchEventCount, config.compositeSearchEventCount) - } - -} \ No newline at end of file diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala deleted file mode 100644 index 41f4e1cfd..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala +++ /dev/null @@ -1,52 +0,0 @@ -package org.sunbird.job.searchindexer.functions - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.searchindexer.compositesearch.domain.Event -import org.sunbird.job.searchindexer.compositesearch.helpers.DIALCodeIndexerHelper -import org.sunbird.job.searchindexer.task.SearchIndexerConfig -import org.sunbird.job.util.ElasticSearchUtil -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import scala.collection.JavaConverters._ - -class DIALCodeIndexerFunction(config: SearchIndexerConfig, - @transient var elasticUtil: ElasticSearchUtil = null) - extends BaseProcessFunction[Event, String](config) with DIALCodeIndexerHelper with FailedEventHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeIndexerFunction]) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) - createDialCodeIndex()(elasticUtil) - } - - override def close(): Unit = { - elasticUtil.close() - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.dialcodeExternalEventCount) - try { - upsertDIALCodeDocument(event.id, event.getMap().asScala.toMap)(elasticUtil) - metrics.incCounter(config.successDialcodeExternalEventCount) - } catch { - case ex: Throwable => - logger.error(s"Error while processing message for identifier : ${event.id}. Error : ", ex) - metrics.incCounter(config.failedDialcodeExternalEventCount) - val failedEvent = getFailedEvent(event.jobName, event.getMap(), ex) - context.output(config.failedEventOutTag, failedEvent) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - override def metricsList(): List[String] = { - List(config.successDialcodeExternalEventCount, config.failedDialcodeExternalEventCount, config.dialcodeExternalEventCount) - } -} \ No newline at end of file diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala deleted file mode 100644 index eeb6684d2..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala +++ /dev/null @@ -1,53 +0,0 @@ -package org.sunbird.job.searchindexer.functions - -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.helper.FailedEventHelper -import org.sunbird.job.searchindexer.compositesearch.domain.Event -import org.sunbird.job.searchindexer.compositesearch.helpers.DIALCodeMetricsIndexerHelper -import org.sunbird.job.searchindexer.task.SearchIndexerConfig -import org.sunbird.job.util.ElasticSearchUtil -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import scala.collection.JavaConverters._ - -class DIALCodeMetricsIndexerFunction(config: SearchIndexerConfig, - @transient var elasticUtil: ElasticSearchUtil = null) - extends BaseProcessFunction[Event, String](config) with DIALCodeMetricsIndexerHelper with FailedEventHelper { - - private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeMetricsIndexerFunction]) - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeMetricIndex, config.dialcodeMetricIndexType) - createDialCodeIndex()(elasticUtil) - } - - override def close(): Unit = { - elasticUtil.close() - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.dialcodeMetricEventCount) - try { - upsertDialcodeMetricDocument(event.id, event.getMap().asScala.toMap)(elasticUtil) - metrics.incCounter(config.successDialcodeMetricEventCount) - } catch { - case ex: Throwable => - logger.error(s"Error while processing message for identifier : ${event.id}. Error : ", ex) - metrics.incCounter(config.failedDialcodeMetricEventCount) - val failedEvent = getFailedEvent(event.jobName, event.getMap(), ex) - context.output(config.failedEventOutTag, failedEvent) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - override def metricsList(): List[String] = { - List(config.successDialcodeMetricEventCount, config.failedDialcodeMetricEventCount, config.dialcodeMetricEventCount) - } - -} \ No newline at end of file diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/TransactionEventRouter.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/TransactionEventRouter.scala deleted file mode 100644 index ab1e2e6b9..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/TransactionEventRouter.scala +++ /dev/null @@ -1,49 +0,0 @@ -package org.sunbird.job.searchindexer.functions - -import com.google.gson.reflect.TypeToken -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.searchindexer.compositesearch.domain.Event -import org.sunbird.job.searchindexer.task.SearchIndexerConfig -import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} - -import java.lang.reflect.Type - -class TransactionEventRouter(config: SearchIndexerConfig) - extends BaseProcessKeyedFunction[String, Event, String](config) { - - private[this] val logger = LoggerFactory.getLogger(classOf[TransactionEventRouter]) - val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - override def processElement(event: Event, context: KeyedProcessFunction[String, Event, String]#Context, metrics: Metrics): Unit = { - metrics.incCounter(config.totalEventsCount) - if (event.validEvent(config.restrictObjectTypes)) { - event.nodeType match { - case "SET" | "DATA_NODE" => context.output(config.compositeSearchDataOutTag, event) - case "EXTERNAL" => context.output(config.dialCodeExternalOutTag, event) - case "DIALCODE_METRICS" => context.output(config.dialCodeMetricOutTag, event) - case _ => { - logger.info(s"UNKNOWN EVENT NODETYPE : ${event.nodeType} for Identifier : ${event.id}.") - metrics.incCounter(config.skippedEventCount) - } - } - } else { - metrics.incCounter(config.skippedEventCount) - logger.info(s"Event not qualified for indexing for Identifier : ${event.id}.") - } - } - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.skippedEventCount) - } - -} \ No newline at end of file diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/models/Models.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/models/Models.scala deleted file mode 100644 index a0c9cb31f..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/models/Models.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.sunbird.job.searchindexer.models - -import org.sunbird.job.searchindexer.task.SearchIndexerConfig - -import java.util -import scala.collection.JavaConverters._ - -case class CompositeIndexer(graphId: String, objectType: String, identifier: String, messageId: String, message: util.Map[String, Any], config: SearchIndexerConfig) { - def getNestedFields(): List[String] = config.nestedFields.asScala.toList - def getDefinitionBasePath(): String = config.definitionBasePath - def getVersionAsString(): String = config.schemaSupportVersionMap.getOrElse(objectType.toLowerCase(), "1.0").asInstanceOf[String] - def getRestrictMetadataObjectTypes(): List[String] = config.restrictMetadataObjectTypes.asScala.toList - def getIgnoredFields(): List[String] = config.ignoredFields -} diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala deleted file mode 100644 index 9adad0e00..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala +++ /dev/null @@ -1,75 +0,0 @@ -package org.sunbird.job.searchindexer.task - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.searchindexer.compositesearch.domain.Event - -import java.util -import scala.collection.JavaConverters._ - -class SearchIndexerConfig(override val config: Config) extends BaseJobConfig(config, "search-indexer") { - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Job Configuration - val jobEnv: String = config.getString("job.env") - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val kafkaErrorTopic: String = config.getString("kafka.error.topic") - - // Parallelism - val eventRouterParallelism: Int = config.getInt("task.router.parallelism") - val compositeSearchIndexerParallelism: Int = config.getInt("task.compositeSearch.parallelism") - val dialCodeExternalIndexerParallelism: Int = config.getInt("task.dialcodeIndexer.parallelism") - val dialCodeMetricIndexerParallelism: Int = config.getInt("task.dialcodemetricsIndexer.parallelism") - - // Consumers - val searchIndexerConsumer = "search-indexer-consumer" - val transactionEventRouter = "transaction-event-router" - - // Metric List - val totalEventsCount = "total-events-count" - val skippedEventCount = "skipped-event-count" - val compositeSearchEventCount = "composite-search-event-count" - val successCompositeSearchEventCount = "composite-search-event-success-count" - val failedCompositeSearchEventCount = "composite-search-event-failed-count" - val dialcodeMetricEventCount = "dialcode-metric-event-count" - val successDialcodeMetricEventCount = "dialcode-metric-event-success-count" - val failedDialcodeMetricEventCount = "dialcode-metric-event-failed-count" - val dialcodeExternalEventCount = "dialcode-external-event-count" - val successDialcodeExternalEventCount = "dialcode-external-event-success-count" - val failedDialcodeExternalEventCount = "dialcode-external-event-failed-count" - - // Tags - val compositeSearchDataOutTag: OutputTag[Event] = OutputTag[Event]("composite-search-data") - val dialCodeExternalOutTag: OutputTag[Event] = OutputTag[Event]("dialcode-external") - val dialCodeMetricOutTag: OutputTag[Event] = OutputTag[Event]("dialcode-metric") - val failedEventOutTag: OutputTag[String] = OutputTag[String]("failed-event") - - // ES Configs - val esConnectionInfo = config.getString("es.basePath") - - val compositeSearchIndex: String = if (config.hasPath("compositesearch.index.name")) config.getString("compositesearch.index.name") else "compositesearch" - val compositeSearchIndexType: String = "cs" - - val dialcodeExternalIndex: String = if (config.hasPath("dialcode.index.name")) config.getString("dialcode.index.name") else "dialcode" - val dialcodeExternalIndexType: String = "dc" - - val dialcodeMetricIndex: String = if (config.hasPath("dailcodemetrics.index.name")) config.getString("dailcodemetrics.index.name") else "dialcodemetrics" - val dialcodeMetricIndexType: String = "dcm" - - val restrictMetadataObjectTypes: util.List[String] = if (config.hasPath("restrict.metadata.objectTypes")) config.getStringList("restrict.metadata.objectTypes") else new util.ArrayList[String] - val nestedFields: util.List[String] = if (config.hasPath("nested.fields")) config.getStringList("nested.fields") else new util.ArrayList[String] - val definitionBasePath: String = if (config.hasPath("schema.basePath")) config.getString("schema.basePath") else "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - val schemaSupportVersionMap = if (config.hasPath("schema.supportedVersion")) config.getObject("schema.supportedVersion").unwrapped().asScala else Map[String, AnyRef]() - val definitionCacheExpiry: Int = if (config.hasPath("schema.definition_cache.expiry")) config.getInt("schema.definition_cache.expiry") else 14400 - val restrictObjectTypes: util.List[String] = if(config.hasPath("restrict.objectTypes")) config.getStringList("restrict.objectTypes") else new util.ArrayList[String] - val ignoredFields: List[String] = if (config.hasPath("ignored.fields")) config.getStringList("ignored.fields").asScala.toList else List("responseDeclaration", "body") - - val isrRelativePathEnabled: Boolean = if (config.hasPath("cloudstorage.metadata.replace_absolute_path")) config.getBoolean("cloudstorage.metadata.replace_absolute_path") else false -} diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerStreamTask.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerStreamTask.scala deleted file mode 100644 index b7aad178a..000000000 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerStreamTask.scala +++ /dev/null @@ -1,69 +0,0 @@ -package org.sunbird.job.searchindexer.task - -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.functions.KeySelector -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.searchindexer.compositesearch.domain.Event -import org.sunbird.job.searchindexer.functions.{CompositeSearchIndexerFunction, DIALCodeIndexerFunction, DIALCodeMetricsIndexerFunction, TransactionEventRouter} -import org.sunbird.job.util.FlinkUtil - -import java.io.File -import java.util - -class SearchIndexerStreamTask(config: SearchIndexerConfig, kafkaConnector: FlinkKafkaConnector) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - val processStreamTask = env.addSource(source).name(config.searchIndexerConsumer) - .uid(config.searchIndexerConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .keyBy(new SearchIndexerKeySelector()) - .process(new TransactionEventRouter(config)) - .name(config.transactionEventRouter).uid(config.transactionEventRouter) - .setParallelism(config.eventRouterParallelism) - - val compositeSearchStream = processStreamTask.getSideOutput(config.compositeSearchDataOutTag).process(new CompositeSearchIndexerFunction(config)) - .name("composite-search-indexer").uid("composite-search-indexer").setParallelism(config.compositeSearchIndexerParallelism) - val dialcodeExternalStream = processStreamTask.getSideOutput(config.dialCodeExternalOutTag).process(new DIALCodeIndexerFunction(config)) - .name("dialcode-external-indexer").uid("dialcode-external-indexer").setParallelism(config.dialCodeExternalIndexerParallelism) - val dialcodeMetricStream = processStreamTask.getSideOutput(config.dialCodeMetricOutTag).process(new DIALCodeMetricsIndexerFunction(config)) - .name("dialcode-metric-indexer").uid("dialcode-metric-indexer").setParallelism(config.dialCodeMetricIndexerParallelism) - - compositeSearchStream.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaErrorTopic)) - dialcodeExternalStream.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaErrorTopic)) - dialcodeMetricStream.getSideOutput(config.failedEventOutTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaErrorTopic)) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object SearchIndexerStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("search-indexer.conf").withFallback(ConfigFactory.systemEnvironment())) - val searchIndexerConfig = new SearchIndexerConfig(config) - val kafkaUtil = new FlinkKafkaConnector(searchIndexerConfig) - val task = new SearchIndexerStreamTask(searchIndexerConfig, kafkaUtil) - task.process() - } - -} - -// $COVERAGE-ON$ - -class SearchIndexerKeySelector extends KeySelector[Event, String] { - override def getKey(in: Event): String = in.id.replace(".img", "") -} \ No newline at end of file diff --git a/search-indexer/src/test/resources/test.conf b/search-indexer/src/test/resources/test.conf deleted file mode 100644 index 1faacbda4..000000000 --- a/search-indexer/src/test/resources/test.conf +++ /dev/null @@ -1,35 +0,0 @@ -include "base-test.conf" - -job { - env = "sunbirddev" -} - -kafka { - input.topic = "sunbirddev.learning.graph.events" - error.topic = "sunbirddev.learning.events.failed" - groupId = "local-composite-search-indexer-group" -} - -task { - consumer.parallelism = 1 - router.parallelism = 1 - compositeSearch.parallelism = 1 - dialcodeIndexer.parallelism = 1 - dialcodemetricsIndexer.parallelism = 1 -} - -compositesearch.index.name = "compositesearch" -nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials"] - -restrict { - metadata.objectTypes = [] - objectTypes = ["EventSet", "Questionnaire", "Misconception", "FrameworkType", "EventSet", "Event"] -} - -schema { - basePath = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" - supportedVersion = { - itemset = "2.0" - } -} -schema.definition_cache.expiry = 14400 \ No newline at end of file diff --git a/search-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/search-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index c2b94db9e..000000000 --- a/search-indexer/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,95 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val DATA_NODE_CREATE: String = - """ - |{"ets":1614346609623,"channel":"channel-1","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"org.sunbird.zf7fcK"},"credentials":{"ov":null,"nv":"{\"enabled\":\"No\"}"},"subject":{"ov":null,"nv":["Geography"]},"channel":{"ov":null,"nv":"channel-1"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/vnd.ekstep.content-collection"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"primaryCategory":{"ov":null,"nv":"Digital Textbook"},"contentDisposition":{"ov":null,"nv":"inline"},"additionalCategories":{"ov":null,"nv":["Textbook"]},"lastUpdatedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"contentEncoding":{"ov":null,"nv":"gzip"},"dialcodeRequired":{"ov":null,"nv":"No"},"contentType":{"ov":null,"nv":"TextBook"},"trackable":{"ov":null,"nv":"{\"enabled\":\"No\",\"autoBatch\":\"No\"}"},"subjectIds":{"ov":null,"nv":["ncf_subject_geography"]},"lastStatusChangedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"audience":{"ov":null,"nv":["Student"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"consumerId":{"ov":null,"nv":"7411b6bd-89f3-40ec-98d1-229dc64ce77d"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"version":{"ov":null,"nv":2},"versionKey":{"ov":null,"nv":"1614346609592"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"license":{"ov":null,"nv":"CC BY-SA 4.0"},"framework":{"ov":null,"nv":"NCF"},"createdBy":{"ov":null,"nv":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8"},"compatibilityLevel":{"ov":null,"nv":1},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Collection"},"userConsent":{"ov":null,"nv":"Yes"},"name":{"ov":null,"nv":"Test"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_1132247274257203201191"},"status":{"ov":null,"nv":"Draft"}}},"mid":"62d07bca-ac01-4faf-8042-af25979bd902","label":"Test","nodeType":"DATA_NODE","userId":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","createdOn":"2021-02-26T13:36:49.623+0000","objectType":"Collection","nodeUniqueId":"do_1132247274257203201191","requestId":null,"operationType":"CREATE","nodeGraphId":509674,"graphId":"domain"} - |""".stripMargin - - val DATA_NODE_UPDATE: String = - """ - |{"ets":1614346671374,"channel":"channel-1","transactionData":{"properties":{"description":{"ov":null,"nv":"updated description"},"language":{"ov":["English"],"nv":null},"lastUpdatedOn":{"ov":"2021-02-26T13:36:49.592+0000","nv":"2021-02-26T13:37:51.285+0000"},"versionKey":{"ov":"1614346609592","nv":"1614346671285"}}},"mid":"6395492e-8ecc-4ad0-a6a6-bf524b8d2ac0","label":"Test","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-02-26T13:37:51.373+0000","objectType":"Collection","nodeUniqueId":"do_1132247274257203201191","requestId":null,"operationType":"UPDATE","nodeGraphId":509674,"graphId":"domain"} - |""".stripMargin - - - val DATA_NODE_DELETE: String = - """ - |{"ets":1614346963589,"channel":"channel-1","transactionData":{"properties":{"ownershipType":{"ov":["createdBy"],"nv":null},"code":{"ov":"org.sunbird.zf7fcK","nv":null},"credentials":{"ov":"{\"enabled\":\"No\"}","nv":null},"subject":{"ov":["Geography"],"nv":null},"channel":{"ov":"channel-1","nv":null},"description":{"ov":"updated description","nv":null},"language":{"ov":["English"],"nv":null},"mimeType":{"ov":"application/vnd.ekstep.content-collection","nv":null},"idealScreenSize":{"ov":"normal","nv":null},"createdOn":{"ov":"2021-02-26T13:36:49.592+0000","nv":null},"primaryCategory":{"ov":"Digital Textbook","nv":null},"additionalCategories":{"ov":["Textbook"],"nv":null},"contentDisposition":{"ov":"inline","nv":null},"contentEncoding":{"ov":"gzip","nv":null},"lastUpdatedOn":{"ov":"2021-02-26T13:37:51.285+0000","nv":null},"contentType":{"ov":"TextBook","nv":null},"dialcodeRequired":{"ov":"No","nv":null},"trackable":{"ov":"{\"enabled\":\"No\",\"autoBatch\":\"No\"}","nv":null},"audience":{"ov":["Student"],"nv":null},"subjectIds":{"ov":["ncf_subject_geography"],"nv":null},"lastStatusChangedOn":{"ov":"2021-02-26T13:36:49.592+0000","nv":null},"visibility":{"ov":"Default","nv":null},"os":{"ov":["All"],"nv":null},"IL_SYS_NODE_TYPE":{"ov":"DATA_NODE","nv":null},"consumerId":{"ov":"7411b6bd-89f3-40ec-98d1-229dc64ce77d","nv":null},"mediaType":{"ov":"content","nv":null},"osId":{"ov":"org.ekstep.quiz.app","nv":null},"version":{"ov":2,"nv":null},"versionKey":{"ov":"1614346671285","nv":null},"license":{"ov":"CC BY-SA 4.0","nv":null},"idealScreenDensity":{"ov":"hdpi","nv":null},"framework":{"ov":"NCF","nv":null},"createdBy":{"ov":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","nv":null},"compatibilityLevel":{"ov":1,"nv":null},"userConsent":{"ov":"Yes","nv":null},"IL_FUNC_OBJECT_TYPE":{"ov":"Collection","nv":null},"name":{"ov":"Test","nv":null},"IL_UNIQUE_ID":{"ov":"do_1132247274257203201191","nv":null},"status":{"ov":"Draft","nv":null}}},"mid":"9759b9a4-3b07-473b-b38b-53cdce944288","label":"Test","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-02-26T13:42:43.589+0000","objectType":"Collection","nodeUniqueId":"do_1132247274257203201191","requestId":null,"operationType":"DELETE","nodeGraphId":509674,"graphId":"domain"} - |""".stripMargin - - val DATA_NODE_UNKNOWN: String = - """ - |{"ets":1614346609623,"channel":"channel-1","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"org.sunbird.zf7fcK"},"credentials":{"ov":null,"nv":"{\"enabled\":\"No\"}"},"subject":{"ov":null,"nv":["Geography"]},"channel":{"ov":null,"nv":"channel-1"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/vnd.ekstep.content-collection"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"primaryCategory":{"ov":null,"nv":"Digital Textbook"},"contentDisposition":{"ov":null,"nv":"inline"},"additionalCategories":{"ov":null,"nv":["Textbook"]},"lastUpdatedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"contentEncoding":{"ov":null,"nv":"gzip"},"dialcodeRequired":{"ov":null,"nv":"No"},"contentType":{"ov":null,"nv":"TextBook"},"trackable":{"ov":null,"nv":"{\"enabled\":\"No\",\"autoBatch\":\"No\"}"},"subjectIds":{"ov":null,"nv":["ncf_subject_geography"]},"lastStatusChangedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"audience":{"ov":null,"nv":["Student"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"consumerId":{"ov":null,"nv":"7411b6bd-89f3-40ec-98d1-229dc64ce77d"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"version":{"ov":null,"nv":2},"versionKey":{"ov":null,"nv":"1614346609592"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"license":{"ov":null,"nv":"CC BY-SA 4.0"},"framework":{"ov":null,"nv":"NCF"},"createdBy":{"ov":null,"nv":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8"},"compatibilityLevel":{"ov":null,"nv":1},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Collection"},"userConsent":{"ov":null,"nv":"Yes"},"name":{"ov":null,"nv":"Test"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_1132247274257203201191"},"status":{"ov":null,"nv":"Draft"}}},"mid":"62d07bca-ac01-4faf-8042-af25979bd902","label":"Test","nodeType":"DATA_NODE","userId":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","createdOn":"2021-02-26T13:36:49.623+0000","objectType":"Collection","nodeUniqueId":"do_1132247274257203201191","requestId":null,"operationType":"UNKNOWN","nodeGraphId":509674,"graphId":"domain"} - |""".stripMargin - - val DATA_NODE_CREATE_WITH_RELATION: String = - """ - |{"ets":1502102183388,"nodeUniqueId":"do_112276071067320320114","requestId":null,"transactionData":{"removedTags":[],"addedRelations":[{"rel":"hasSequenceMember","id":"do_1123032073439723521148","label":"Test unit 11","dir":"IN","type":"Content"}],"removedRelations":[],"addedTags":[],"properties":{}},"operationType":"CREATE","nodeGraphId":105631,"label":"collaborator test","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-08-07T10:36:23.388+0000","objectType":"Content"} - |""".stripMargin - - val DATA_NODE_UPDATE_WITH_RELATION: String = - """ - |{"ets":1502102183388,"nodeUniqueId":"do_112276071067320320114","requestId":null,"transactionData":{"removedTags":[],"addedRelations":[],"removedRelations":[{"rel":"hasSequenceMember","id":"do_1123032073439723521148","label":"Test unit 11","dir":"IN","type":"Content"}],"addedTags":[],"properties":{}},"operationType":"UPDATE","nodeGraphId":105631,"label":"collaborator test","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-08-07T10:36:23.388+0000","objectType":"Content"} - |""".stripMargin - - val DIALCODE_METRIC_CREATE: String = - """ - |{"ets":1543561000015,"nodeUniqueId":"QR1234","transactionData":{"properties":{"average_scans_per_day":{"nv":2},"last_scan":{"nv":1541456052000},"total_dial_scans_local":{"nv":25},"first_scan":{"nv":1540469152000}}},"objectType":"","operationType":"CREATE","nodeType":"DIALCODE_METRICS","graphId":"domain","nodeGraphId":0} - |""".stripMargin - - val DIALCODE_METRIC_UPDATE: String = - """ - |{"ets":1543561000015,"nodeUniqueId":"QR1234","transactionData":{"properties":{"average_scans_per_day":{"nv":2},"last_scan":{"nv":1541456052000},"total_dial_scans_global":{"nv":25},"total_dial_scans_local":{"nv":null},"first_scan":{"nv":1540469152000}}},"objectType":"","operationType":"UPDATE","nodeType":"DIALCODE_METRICS","graphId":"domain","nodeGraphId":0} - |""".stripMargin - - val DIALCODE_METRIC_DELETE: String = - """ - |{"ets":1543561000015,"nodeUniqueId":"QR1234","transactionData":{"properties":{"average_scans_per_day":{"nv":null},"last_scan":{"nv":null},"total_dial_scans_local":{"nv":null},"first_scan":{"nv":null}}},"objectType":"","operationType":"DELETE","nodeType":"DIALCODE_METRICS","graphId":"domain","nodeGraphId":0} - |""".stripMargin - - val DIALCODE_METRIC_UNKNOWN: String = - """ - |{"ets":1543561000015,"nodeUniqueId":"QR1234","transactionData":{"properties":{"average_scans_per_day":{"nv":2},"last_scan":{"nv":1541456052000},"total_dial_scans_local":{"nv":25},"first_scan":{"nv":1540469152000}}},"objectType":"","operationType":"UNKNOWN","nodeType":"DIALCODE_METRICS","graphId":"domain","nodeGraphId":0} - |""".stripMargin - - val DIALCODE_EXTERNAL_CREATE: String = - """ - |{"nodeUniqueId":"X8R3W4","ets":1613072768797,"requestId":null,"audit":false,"transactionData":{"properties":{"dialcode_index":{"ov":null,"nv":9071809.0},"identifier":{"ov":null,"nv":"X8R3W4"},"channel":{"ov":null,"nv":"channelTest"},"batchcode":{"ov":null,"nv":"testPub0001.20210212T011555"},"publisher":{"ov":null,"nv":"testPub0001"},"generated_on":{"ov":null,"nv":"2021-02-12T01:16:07.750+0530"},"status":{"ov":null,"nv":"Draft"}}},"channel":"in.ekstep","index":true,"operationType":"CREATE","nodeType":"EXTERNAL","userId":"ANONYMOUS","createdOn":"2021-02-12T01:16:08.797+0530","objectType":"DialCode"} - |""".stripMargin - - val DIALCODE_EXTERNAL_UPDATE: String = - """ - |{"nodeUniqueId":"X8R3W4","ets":1613072768797,"requestId":null,"audit":false,"transactionData":{"properties":{"dialcode_index":{"ov":null,"nv":9071809.0},"identifier":{"ov":null,"nv":"X8R3W4"},"channel":{"ov":null,"nv":"channelTest Updated"},"batchcode":{"ov":"testPub0001.20210212T011555","nv":null},"publisher":{"ov":null,"nv":"testPub0001"},"generated_on":{"ov":null,"nv":"2021-02-12T01:16:07.750+0530"},"status":{"ov":null,"nv":"Draft"}}},"channel":"in.ekstep","index":true,"operationType":"UPDATE","nodeType":"EXTERNAL","userId":"ANONYMOUS","createdOn":"2021-02-12T01:16:08.797+0530","objectType":"DialCode"} - |""".stripMargin - - val DIALCODE_EXTERNAL_DELETE: String = - """ - |{"nodeUniqueId":"X8R3W4","ets":1613072768797,"requestId":null,"audit":false,"transactionData":{"properties":{"dialcode_index":{"nv":null},"identifier":{"nv":null},"channel":{"nv":null},"batchcode":{"nv":null},"publisher":{"nv":null},"generated_on":{"nv":null},"status":{"nv":null}}},"channel":"in.ekstep","index":true,"operationType":"DELETE","nodeType":"EXTERNAL","userId":"ANONYMOUS","createdOn":"2021-02-12T01:16:08.797+0530","objectType":"DialCode"} - |""".stripMargin - - val DIALCODE_EXTERNAL_UNKNOWN: String = - """ - |{"nodeUniqueId":"X8R3W4","ets":1613072768797,"requestId":null,"audit":false,"transactionData":{"properties":{"dialcode_index":{"ov":null,"nv":9071809.0},"identifier":{"ov":null,"nv":"X8R3W4"},"channel":{"ov":null,"nv":"channelTest"},"batchcode":{"ov":null,"nv":"testPub0001.20210212T011555"},"publisher":{"ov":null,"nv":"testPub0001"},"generated_on":{"ov":null,"nv":"2021-02-12T01:16:07.750+0530"},"status":{"ov":null,"nv":"Draft"}}},"channel":"in.ekstep","index":true,"operationType":"UNKNOWN","nodeType":"EXTERNAL","userId":"ANONYMOUS","createdOn":"2021-02-12T01:16:08.797+0530","objectType":"DialCode"} - |""".stripMargin - - val UNKNOWN_NODE_TYPE: String = - """ - |{"nodeUniqueId":"X8R3W4","ets":1613072768797,"requestId":null,"audit":false,"transactionData":{"properties":{"dialcode_index":{"ov":null,"nv":9071809.0},"identifier":{"ov":null,"nv":"X8R3W4"},"channel":{"ov":null,"nv":"channelTest"},"batchcode":{"ov":null,"nv":"testPub0001.20210212T011555"},"publisher":{"ov":null,"nv":"testPub0001"},"generated_on":{"ov":null,"nv":"2021-02-12T01:16:07.750+0530"},"status":{"ov":null,"nv":"Draft"}}},"channel":"in.ekstep","index":true,"operationType":"CREATE","nodeType":"UNKNOWN","userId":"ANONYMOUS","createdOn":"2021-02-12T01:16:08.797+0530","objectType":"DialCode"} - |""".stripMargin - - val INDEX_FALSE: String = - """ - |{"nodeUniqueId":"X8R3W4","ets":1613072768797,"requestId":null,"audit":false,"transactionData":{"properties":{"dialcode_index":{"ov":null,"nv":9071809.0},"identifier":{"ov":null,"nv":"X8R3W4"},"channel":{"ov":null,"nv":"channelTest"},"batchcode":{"ov":null,"nv":"testPub0001.20210212T011555"},"publisher":{"ov":null,"nv":"testPub0001"},"generated_on":{"ov":null,"nv":"2021-02-12T01:16:07.750+0530"},"status":{"ov":null,"nv":"Draft"}}},"channel":"in.ekstep","index":false,"operationType":"UNKNOWN","nodeType":"EXTERNAL","userId":"ANONYMOUS","createdOn":"2021-02-12T01:16:08.797+0530","objectType":"DialCode"} - |""".stripMargin - - val DATA_NODE_FAILED: String = - """ - |{"ets":1614346609623,"channel":"channel-1","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"org.sunbird.zf7fcK"},"credentials":{"ov":null,"nv":"{\"enabled\":\"No\"}"},"subject":{"ov":null,"nv":["Geography"]},"channel":{"ov":null,"nv":"channel-1"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/vnd.ekstep.content-collection"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"primaryCategory":{"ov":null,"nv":"Digital Textbook"},"contentDisposition":{"ov":null,"nv":"inline"},"additionalCategories":{"ov":null,"nv":["Textbook"]},"lastUpdatedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"contentEncoding":{"ov":null,"nv":"gzip"},"dialcodeRequired":{"ov":null,"nv":"No"},"contentType":{"ov":null,"nv":"TextBook"},"trackable":{"ov":null,"nv":"{\"enabled\":\"No\",\"autoBatch\":\"No\"}"},"subjectIds":{"ov":null,"nv":["ncf_subject_geography"]},"lastStatusChangedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"audience":{"ov":null,"nv":["Student"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"consumerId":{"ov":null,"nv":"7411b6bd-89f3-40ec-98d1-229dc64ce77d"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"version":{"ov":null,"nv":2},"versionKey":{"ov":null,"nv":"1614346609592"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"license":{"ov":null,"nv":"CC BY-SA 4.0"},"framework":{"ov":null,"nv":"NCF"},"createdBy":{"ov":null,"nv":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8"},"compatibilityLevel":{"ov":null,"nv":1},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Collection"},"userConsent":{"ov":null,"nv":"Yes"},"name":{"ov":null,"nv":"Test"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_1132247274257203201191"},"status":{"ov":null,"nv":"Draft"}}},"mid":"62d07bca-ac01-4faf-8042-af25979bd902","label":"Test","nodeType":"DATA_NODE","userId":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","createdOn":"2021-02-26T13:36:49.623+0000","objectType":"UNKNOWN","nodeUniqueId":"do_1132247274257203201191","requestId":null,"operationType":"CREATE","nodeGraphId":509674,"graphId":"domain"} - |""".stripMargin - - val DATA_NODE_IGNORE: String = - """ - |{"ets":1614346609623,"channel":"channel-1","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"org.sunbird.zf7fcK"},"credentials":{"ov":null,"nv":"{\"enabled\":\"No\"}"},"subject":{"ov":null,"nv":["Geography"]},"channel":{"ov":null,"nv":"channel-1"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/vnd.ekstep.content-collection"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"primaryCategory":{"ov":null,"nv":"Digital Textbook"},"contentDisposition":{"ov":null,"nv":"inline"},"additionalCategories":{"ov":null,"nv":["Textbook"]},"lastUpdatedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"contentEncoding":{"ov":null,"nv":"gzip"},"dialcodeRequired":{"ov":null,"nv":"No"},"contentType":{"ov":null,"nv":"TextBook"},"trackable":{"ov":null,"nv":"{\"enabled\":\"No\",\"autoBatch\":\"No\"}"},"subjectIds":{"ov":null,"nv":["ncf_subject_geography"]},"lastStatusChangedOn":{"ov":null,"nv":"2021-02-26T13:36:49.592+0000"},"audience":{"ov":null,"nv":["Student"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"consumerId":{"ov":null,"nv":"7411b6bd-89f3-40ec-98d1-229dc64ce77d"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"version":{"ov":null,"nv":2},"versionKey":{"ov":null,"nv":"1614346609592"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"license":{"ov":null,"nv":"CC BY-SA 4.0"},"framework":{"ov":null,"nv":"NCF"},"createdBy":{"ov":null,"nv":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8"},"compatibilityLevel":{"ov":null,"nv":1},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Collection"},"userConsent":{"ov":null,"nv":"Yes"},"name":{"ov":null,"nv":"Test"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_1132247274257203201191"},"status":{"ov":null,"nv":"Draft"}}},"mid":"62d07bca-ac01-4faf-8042-af25979bd902","label":"Test","nodeType":"DATA_NODE","userId":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","createdOn":"2021-02-26T13:36:49.623+0000","objectType":"EventSet","nodeUniqueId":"do_1132247274257203201191","requestId":null,"operationType":"CREATE","nodeGraphId":509674,"graphId":"domain"} - |""".stripMargin -} \ No newline at end of file diff --git a/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala b/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala deleted file mode 100644 index d8bc1ee6b..000000000 --- a/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala +++ /dev/null @@ -1,578 +0,0 @@ -package org.sunbird.job.spec - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.sink.SinkFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.mockito.ArgumentMatchers.anyString -import org.mockito.Mockito -import org.mockito.Mockito.{doNothing, times, verify, when} -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.domain.`object`.DefinitionCache -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.searchindexer.compositesearch.domain.Event -import org.sunbird.job.searchindexer.functions.{CompositeSearchIndexerFunction, DIALCodeIndexerFunction, DIALCodeMetricsIndexerFunction} -import org.sunbird.job.searchindexer.task.{SearchIndexerConfig, SearchIndexerStreamTask} -import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} -import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic - -import java.util -import scala.collection.JavaConverters._ - - -class SearchIndexerTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig = new SearchIndexerConfig(config) - val mockElasticUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - var embeddedElastic: EmbeddedElastic = _ - val defCache = new DefinitionCache() - - override protected def beforeAll(): Unit = { - super.beforeAll() - - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion("6.8.22") - .withEsJavaOpts("-Xms128m -Xmx512m") - .build() - .start() - flinkCluster.before() - } - - override protected def afterAll(): Unit = { - super.afterAll() - embeddedElastic.stop() - flinkCluster.after() - } - - "getCompositeIndexerObject" should " return Composite Object for the event" in { - val event = getEvent(EventFixture.DATA_NODE_CREATE, 509674) - val compositeObject = new CompositeSearchIndexerFunction(jobConfig).getCompositeIndexerObject(event) - compositeObject.objectType should be("Collection") - compositeObject.getVersionAsString() should be("1.0") - compositeObject.identifier should be("do_1132247274257203201191") - } - - - it should "return the indexable document for the provided object" in { - val definition = defCache.getDefinition("Collection", "1.0", jobConfig.definitionBasePath) - val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) - val message = getEvent(EventFixture.DATA_NODE_CREATE, 509674).getMap().asScala.toMap - val indexDocument: Map[String, AnyRef] = compositeFunc.getIndexDocument(message, false, definition, jobConfig.nestedFields.asScala.toList, jobConfig.ignoredFields)(mockElasticUtil) - val trackable = indexDocument.getOrElse("trackable", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - indexDocument.isEmpty should be(false) - indexDocument.getOrElse("identifier", "").asInstanceOf[String] should be("do_1132247274257203201191") - trackable.getOrElse("enabled", "") should be("No") - indexDocument.getOrElse("objectType", "").asInstanceOf[String] should be("Collection") - } - - it should "return the indexable document with the added relation for the provided object" in { - val definition = defCache.getDefinition("Collection", "1.0", jobConfig.definitionBasePath) - val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) - val message = getEvent(EventFixture.DATA_NODE_CREATE_WITH_RELATION, 509674).getMap().asScala.toMap - val indexDocument: Map[String, AnyRef] = compositeFunc.getIndexDocument(message, false, definition, jobConfig.nestedFields.asScala.toList, jobConfig.ignoredFields)(mockElasticUtil) - indexDocument.isEmpty should be(false) - indexDocument.getOrElse("identifier", "").asInstanceOf[String] should be("do_112276071067320320114") - indexDocument.getOrElse("objectType", "").asInstanceOf[String] should be("Content") - indexDocument.getOrElse("collections", List[String]()).asInstanceOf[List[String]] should contain("do_1123032073439723521148") - } - - it should "return the indexable document without the deleted relation for the provided object" in { - val documentJson = """{"identifier":"do_112276071067320320114","graph_id":"domain","node_id":105631,"collections":["do_1123032073439723521148", "do_1123032073439723521149"],"objectType":"Content","nodeType":"DATA_NODE"}""" - when(mockElasticUtil.getDocumentAsString(anyString())).thenReturn(documentJson) - - val definition = defCache.getDefinition("Collection", "1.0", jobConfig.definitionBasePath) - val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) - val message = getEvent(EventFixture.DATA_NODE_UPDATE_WITH_RELATION, 509674).getMap().asScala.toMap - val indexDocument: Map[String, AnyRef] = compositeFunc.getIndexDocument(message, true, definition, jobConfig.nestedFields.asScala.toList, jobConfig.ignoredFields)(mockElasticUtil) - val collections = indexDocument.getOrElse("collections", List[String]()).asInstanceOf[List[String]] - indexDocument.isEmpty should be(false) - indexDocument.getOrElse("identifier", "").asInstanceOf[String] should be("do_112276071067320320114") - indexDocument.getOrElse("objectType", "").asInstanceOf[String] should be("Content") - collections.length should be(1) - collections should contain("do_1123032073439723521149") - indexDocument.getOrElse("collections", List[String]()).asInstanceOf[List[String]] should not contain ("do_1123032073439723521148") - } - - it should "return the indexable document for the provided update object" in { - val documentJson = """{"ownershipType":["createdBy"],"code":"org.sunbird.zf7fcK","credentials":{"enabled":"No"},"subject":["Geography"],"channel":"channel-1","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-02-26T13:36:49.592+0000","objectType":"Collection","primaryCategory":"Digital Textbook","contentDisposition":"inline","additionalCategories":["Textbook"],"lastUpdatedOn":"2021-02-26T13:36:49.592+0000","contentEncoding":"gzip","dialcodeRequired":"No","contentType":"TextBook","trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_1132247274257203201191","subjectIds":["ncf_subject_geography"],"lastStatusChangedOn":"2021-02-26T13:36:49.592+0000","audience":["Student"],"IL_SYS_NODE_TYPE":"DATA_NODE","os":["All"],"visibility":"Default","consumerId":"7411b6bd-89f3-40ec-98d1-229dc64ce77d","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","version":2,"versionKey":"1614346609592","idealScreenDensity":"hdpi","license":"CC BY-SA 4.0","framework":"NCF","createdBy":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","compatibilityLevel":1,"IL_FUNC_OBJECT_TYPE":"Collection","userConsent":"Yes","name":"Test","IL_UNIQUE_ID":"do_1132247274257203201191","status":"Draft","node_id":509674}""" - when(mockElasticUtil.getDocumentAsString(anyString())).thenReturn(documentJson) - - val definition = defCache.getDefinition("Collection", "1.0", jobConfig.definitionBasePath) - val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) - val message = getEvent(EventFixture.DATA_NODE_UPDATE, 509674).getMap().asScala.toMap - val indexDocument: Map[String, AnyRef] = compositeFunc.getIndexDocument(message, true, definition, jobConfig.nestedFields.asScala.toList, jobConfig.ignoredFields)(mockElasticUtil) - indexDocument.isEmpty should be(false) - indexDocument.getOrElse("identifier", "").asInstanceOf[String] should be("do_1132247274257203201191") - indexDocument.getOrElse("objectType", "").asInstanceOf[String] should be("Collection") - indexDocument.getOrElse("description", "").asInstanceOf[String] should be("updated description") - } - - it should " give the document for indexing the DIAL Code metrics " in { - val documentJson = """{"last_scan":1541456052000,"dial_code":"QR1234","first_scan":1540469152000,"total_dial_scans_local":25,"objectType":"","average_scans_per_day":2}""" - when(mockElasticUtil.getDocumentAsString(anyString())).thenReturn(documentJson) - - val event = getEvent(EventFixture.DIALCODE_METRIC_UPDATE, 509674) - val dialcodeMetricFunc = new DIALCodeMetricsIndexerFunction(jobConfig) - val response = dialcodeMetricFunc.getIndexDocument(event.getMap().asScala.toMap, true)(mockElasticUtil) - response.isEmpty should be(false) - response.getOrElse("total_dial_scans_global", 0).asInstanceOf[Integer] should be(25) - } - - it should " give the document for indexing the dialcode external " in { - val event = getEvent(EventFixture.DIALCODE_EXTERNAL_CREATE, 509674) - val dialcodeExternalFunc = new DIALCodeIndexerFunction(jobConfig) - val response = dialcodeExternalFunc.getDocument(event.getMap().asScala.toMap, false)(mockElasticUtil) - response.isEmpty should be(false) - response.getOrElse("identifier", "").asInstanceOf[String] should be("X8R3W4") - response.getOrElse("objectType", "").asInstanceOf[String] should be("DialCode") - response.getOrElse("batchcode", "").asInstanceOf[String] should be("testPub0001.20210212T011555") - } - -// -// "processESMessage " should " update the indexed event for the new values of the properties" in { -// Mockito.reset(mockElasticUtil) -// doNothing().when(mockElasticUtil).addDocumentWithId(anyString(), anyString()) -// val documentJson = """{"ownershipType":["createdBy"],"code":"org.sunbird.zf7fcK","credentials":{"enabled":"No"},"subject":["Geography"],"channel":"channel-1","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-02-26T13:36:49.592+0000","objectType":"Collection","primaryCategory":"Digital Textbook","contentDisposition":"inline","additionalCategories":["Textbook"],"lastUpdatedOn":"2021-02-26T13:36:49.592+0000","contentEncoding":"gzip","dialcodeRequired":"No","contentType":"TextBook","trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_1132247274257203201191","subjectIds":["ncf_subject_geography"],"lastStatusChangedOn":"2021-02-26T13:36:49.592+0000","audience":["Student"],"IL_SYS_NODE_TYPE":"DATA_NODE","os":["All"],"visibility":"Default","consumerId":"7411b6bd-89f3-40ec-98d1-229dc64ce77d","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","version":2,"versionKey":"1614346609592","idealScreenDensity":"hdpi","license":"CC BY-SA 4.0","framework":"NCF","createdBy":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","compatibilityLevel":1,"IL_FUNC_OBJECT_TYPE":"Collection","userConsent":"Yes","name":"Test","IL_UNIQUE_ID":"do_1132247274257203201191","status":"Draft","node_id":509674}""" -// when(mockElasticUtil.getDocumentAsStringById(anyString())).thenReturn(documentJson) -// -// val event = getEvent(EventFixture.DATA_NODE_UPDATE, 509674) -// val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) -// val compositeObject = compositeFunc.getCompositeIndexerObject(event) -// compositeFunc.processESMessage(compositeObject)(mockElasticUtil, definitionUtil) -// -// verify(mockElasticUtil, times(1)).addDocumentWithId(anyString(), anyString()) -// verify(mockElasticUtil, times(1)).getDocumentAsStringById(anyString()) -// } -// -// "processESMessage " should " delete the indexed event " in { -// Mockito.reset(mockElasticUtil) -// doNothing().when(mockElasticUtil).deleteDocument(anyString()) -// val documentJson = """{"ownershipType":["createdBy"],"code":"org.sunbird.zf7fcK","credentials":{"enabled":"No"},"subject":["Geography"],"channel":"channel-1","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-02-26T13:36:49.592+0000","objectType":"Collection","primaryCategory":"Digital Textbook","contentDisposition":"inline","additionalCategories":["Textbook"],"lastUpdatedOn":"2021-02-26T13:36:49.592+0000","contentEncoding":"gzip","dialcodeRequired":"No","contentType":"TextBook","trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_1132247274257203201191","subjectIds":["ncf_subject_geography"],"lastStatusChangedOn":"2021-02-26T13:36:49.592+0000","audience":["Student"],"IL_SYS_NODE_TYPE":"DATA_NODE","os":["All"],"visibility":"Default","consumerId":"7411b6bd-89f3-40ec-98d1-229dc64ce77d","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","version":2,"versionKey":"1614346609592","idealScreenDensity":"hdpi","license":"CC BY-SA 4.0","framework":"NCF","createdBy":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","compatibilityLevel":1,"IL_FUNC_OBJECT_TYPE":"Collection","userConsent":"Yes","name":"Test","IL_UNIQUE_ID":"do_1132247274257203201191","status":"Draft","node_id":509674}""" -// when(mockElasticUtil.getDocumentAsStringById(anyString())).thenReturn(documentJson) -// -// val event = getEvent(EventFixture.DATA_NODE_DELETE, 509674) -// val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) -// val compositeObject = compositeFunc.getCompositeIndexerObject(event) -// compositeFunc.processESMessage(compositeObject)(mockElasticUtil, definitionUtil) -// -// verify(mockElasticUtil, times(1)).getDocumentAsStringById(anyString()) -// verify(mockElasticUtil, times(1)).deleteDocument(anyString()) -// } -// -// "processESMessage " should " not delete the indexed event with visibility Parent " in { -// Mockito.reset(mockElasticUtil) -// val documentJson = """{"ownershipType":["createdBy"],"code":"org.sunbird.zf7fcK","credentials":{"enabled":"No"},"subject":["Geography"],"channel":"channel-1","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","idealScreenSize":"normal","createdOn":"2021-02-26T13:36:49.592+0000","objectType":"Collection","primaryCategory":"Digital Textbook","contentDisposition":"inline","additionalCategories":["Textbook"],"lastUpdatedOn":"2021-02-26T13:36:49.592+0000","contentEncoding":"gzip","dialcodeRequired":"No","contentType":"TextBook","trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_1132247274257203201191","subjectIds":["ncf_subject_geography"],"lastStatusChangedOn":"2021-02-26T13:36:49.592+0000","audience":["Student"],"IL_SYS_NODE_TYPE":"DATA_NODE","os":["All"],"visibility":"Parent","consumerId":"7411b6bd-89f3-40ec-98d1-229dc64ce77d","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","version":2,"versionKey":"1614346609592","idealScreenDensity":"hdpi","license":"CC BY-SA 4.0","framework":"NCF","createdBy":"95e4942d-cbe8-477d-aebd-ad8e6de4bfc8","compatibilityLevel":1,"IL_FUNC_OBJECT_TYPE":"Collection","userConsent":"Yes","name":"Test","IL_UNIQUE_ID":"do_1132247274257203201191","status":"Draft","node_id":509674}""" -// when(mockElasticUtil.getDocumentAsStringById(anyString())).thenReturn(documentJson) -// -// val event = getEvent(EventFixture.DATA_NODE_DELETE, 509674) -// val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) -// val compositeObject = compositeFunc.getCompositeIndexerObject(event) -// compositeFunc.processESMessage(compositeObject)(mockElasticUtil, definitionUtil) -// -// verify(mockElasticUtil, times(1)).getDocumentAsStringById(anyString()) -// verify(mockElasticUtil, times(0)).deleteDocument(anyString()) -// } -// -// "processESMessage " should " index the event with the added Relations" in { -// Mockito.reset(mockElasticUtil) -// doNothing().when(mockElasticUtil).addDocumentWithId(anyString(), anyString()) -// -// val event = getEvent(EventFixture.DATA_NODE_CREATE_WITH_RELATION, 509674) -// val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) -// val compositeObject = compositeFunc.getCompositeIndexerObject(event) -// compositeFunc.processESMessage(compositeObject)(mockElasticUtil, definitionUtil) -// -// verify(mockElasticUtil, times(1)).addDocumentWithId(anyString(), anyString()) -// verify(mockElasticUtil, times(0)).getDocumentAsStringById(anyString()) -// } - - it should " index the DIAL Code metrics " in { - Mockito.reset(mockElasticUtil) - doNothing().when(mockElasticUtil).addDocument(anyString(), anyString()) - - val event = getEvent(EventFixture.DIALCODE_METRIC_CREATE, 509674) - val dialcodeMetricFunc = new DIALCodeMetricsIndexerFunction(jobConfig) - val uniqueId = event.readOrDefault("nodeUniqueId", "") - dialcodeMetricFunc.upsertDialcodeMetricDocument(uniqueId, event.getMap().asScala.toMap)(mockElasticUtil) - - verify(mockElasticUtil, times(1)).addDocument(anyString(), anyString()) - verify(mockElasticUtil, times(0)).getDocumentAsString(anyString()) - } - - "upsertDialcodeMetricDocument " should " update the indexed the dialcode metrics " in { - Mockito.reset(mockElasticUtil) - doNothing().when(mockElasticUtil).addDocument(anyString(), anyString()) - val documentJson = """{"last_scan":1541456052000,"dial_code":"QR1234","first_scan":1540469152000,"total_dial_scans_local":25,"objectType":"","average_scans_per_day":2}""" - when(mockElasticUtil.getDocumentAsString(anyString())).thenReturn(documentJson) - - val event = getEvent(EventFixture.DIALCODE_METRIC_UPDATE, 509674) - val dialcodeMetricFunc = new DIALCodeMetricsIndexerFunction(jobConfig) - val uniqueId = event.readOrDefault("nodeUniqueId", "") - dialcodeMetricFunc.upsertDialcodeMetricDocument(uniqueId, event.getMap().asScala.toMap)(mockElasticUtil) - - verify(mockElasticUtil, times(1)).addDocument(anyString(), anyString()) - verify(mockElasticUtil, times(1)).getDocumentAsString(anyString()) - } - - "upsertDialcodeMetricDocument " should " delete the indexed dialcode metric event " in { - Mockito.reset(mockElasticUtil) - val event = getEvent(EventFixture.DIALCODE_METRIC_DELETE, 509674) - val dialcodeMetricFunc = new DIALCodeMetricsIndexerFunction(jobConfig) - val uniqueId = event.readOrDefault("nodeUniqueId", "") - dialcodeMetricFunc.upsertDialcodeMetricDocument(uniqueId, event.getMap().asScala.toMap)(mockElasticUtil) - - verify(mockElasticUtil, times(0)).getDocumentAsString(anyString()) - verify(mockElasticUtil, times(1)).deleteDocument(anyString()) - } - - "upsertExternalDocument " should " index the dialcode external " in { - Mockito.reset(mockElasticUtil) - doNothing().when(mockElasticUtil).addDocument(anyString(), anyString()) - - val event = getEvent(EventFixture.DIALCODE_EXTERNAL_CREATE, 509674) - val dialcodeExternalFunc = new DIALCodeIndexerFunction(jobConfig) - val uniqueId = event.readOrDefault("nodeUniqueId", "") - dialcodeExternalFunc.upsertDIALCodeDocument(uniqueId, event.getMap().asScala.toMap)(mockElasticUtil) - - verify(mockElasticUtil, times(1)).addDocument(anyString(), anyString()) - verify(mockElasticUtil, times(0)).getDocumentAsString(anyString()) - } - - "upsertExternalDocument " should " update and index the dialcode external " in { - Mockito.reset(mockElasticUtil) - doNothing().when(mockElasticUtil).addDocument(anyString(), anyString()) - val documentJson = """{"channel":"channelTest","generated_on":"2021-02-12T01:16:07.750+0530","identifier":"X8R3W4","dialcode_index":9071809.0,"batchcode":"testPub0001.20210212T011555","objectType":"DialCode","status":"Draft","publisher":"testPub0001"}""" - when(mockElasticUtil.getDocumentAsString(anyString())).thenReturn(documentJson) - - val event = getEvent(EventFixture.DIALCODE_EXTERNAL_UPDATE, 509674) - val dialcodeExternalFunc = new DIALCodeIndexerFunction(jobConfig) - val uniqueId = event.readOrDefault("nodeUniqueId", "") - dialcodeExternalFunc.upsertDIALCodeDocument(uniqueId, event.getMap().asScala.toMap)(mockElasticUtil) - - verify(mockElasticUtil, times(1)).addDocument(anyString(), anyString()) - verify(mockElasticUtil, times(1)).getDocumentAsString(anyString()) - } - - "upsertExternalDocument " should " delete the indexed dialcode external event " in { - Mockito.reset(mockElasticUtil) - val event = getEvent(EventFixture.DIALCODE_EXTERNAL_DELETE, 509674) - val dialcodeExternalFunc = new DIALCodeIndexerFunction(jobConfig) - val uniqueId = event.readOrDefault("nodeUniqueId", "") - dialcodeExternalFunc.upsertDIALCodeDocument(uniqueId, event.getMap().asScala.toMap)(mockElasticUtil) - - verify(mockElasticUtil, times(0)).getDocumentAsString(anyString()) - verify(mockElasticUtil, times(1)).deleteDocument(anyString()) - } - - "createCompositeSearchIndex" should "create the elastic search index for composite Search" in { - Mockito.reset(mockElasticUtil) - when(mockElasticUtil.addIndex(anyString(), anyString(), anyString())).thenReturn(false) - val compositeFunc = new CompositeSearchIndexerFunction(jobConfig) - val check = compositeFunc.createCompositeSearchIndex()(mockElasticUtil) - check should be(false) - } - - "createDialCodeIndex" should "create the elastic search index for dialcode" in { - Mockito.reset(mockElasticUtil) - when(mockElasticUtil.addIndex(anyString(), anyString(), anyString())).thenReturn(false) - val compositeFunc = new DIALCodeIndexerFunction(jobConfig) - val check = compositeFunc.createDialCodeIndex()(mockElasticUtil) - check should be(false) - } - - "createDialCodeIndex" should "create the elastic search index for dialcode metric" in { - Mockito.reset(mockElasticUtil) - when(mockElasticUtil.addIndex(anyString(), anyString(), anyString())).thenReturn(false) - val compositeFunc = new DIALCodeMetricsIndexerFunction(jobConfig) - val check = compositeFunc.createDialCodeIndex()(mockElasticUtil) - check should be(false) - } - - "DialCodeMetricIndexerFunction" should "return the event with error message" in { - val compositeFunc = new DIALCodeMetricsIndexerFunction(jobConfig) - val event = getEvent(EventFixture.DATA_NODE_DELETE, 509674) - val exception = new Exception(s"Test Exception Handling") - val failedEventString = compositeFunc.getFailedEvent(event.jobName, event.getMap(), exception) - failedEventString.isEmpty should be(false) - failedEventString.contains("failInfo") should be(true) - failedEventString.contains("jobName") should be(true) - } - - "Event.index" should "return whether event is indexable " in { - var eventMap = new util.HashMap[String, Any]() - eventMap.put("index", "true") - var event = new Event(eventMap,0, 10) - event.index should be(true) - - eventMap.put("index", "false") - event = new Event(eventMap,0, 11) - event.index should be(false) - - eventMap.put("index", null) - event = new Event(eventMap,0, 12) - event.index should be(true) - - eventMap.put("index", true) - event = new Event(eventMap,0, 13) - event.index should be(true) - - eventMap.put("index", false) - event = new Event(eventMap,0, 14) - event.index should be(false) - } - - "Composite Search Indexer" should " sync the Data Node " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_CREATE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) - val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") - data.isEmpty should be(false) - data.contains("do_1132247274257203201191") should be(true) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successCompositeSearchEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.compositeSearchEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedCompositeSearchEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " update the Data Node " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_UPDATE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) - val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") - data.isEmpty should be(false) - data.contains("do_1132247274257203201191") should be(true) - data.contains("updated description") should be(true) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successCompositeSearchEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.compositeSearchEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedCompositeSearchEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " create and delete the Data Node " in { - embeddedElastic.deleteIndices() - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_CREATE, EventFixture.DATA_NODE_DELETE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) - val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") - data should be(null) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successCompositeSearchEventCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.compositeSearchEventCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedCompositeSearchEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " do nothing for the Data Node due to UNKNOWN Operation " in { - embeddedElastic.deleteIndices() - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_UNKNOWN))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) - val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") - data should be(null) - } - - "Composite Search Indexer" should " sync the External Dialcode Data " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_CREATE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) - val data = elasticUtil.getDocumentAsString("X8R3W4") - data.isEmpty should be(false) - data.contains("X8R3W4") should be(true) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successDialcodeExternalEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dialcodeExternalEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedDialcodeExternalEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " update the External Dialcode Data " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_UPDATE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) - val data = elasticUtil.getDocumentAsString("X8R3W4") - data.isEmpty should be(false) - data.contains("X8R3W4") should be(true) - data.contains("channelTest Updated") should be(true) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successDialcodeExternalEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dialcodeExternalEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedDialcodeExternalEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " create and delete the External Dialcode Data " in { - embeddedElastic.deleteIndices() - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_CREATE, EventFixture.DIALCODE_EXTERNAL_DELETE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) - val data = elasticUtil.getDocumentAsString("X8R3W4") - data should be(null) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successDialcodeExternalEventCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dialcodeExternalEventCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedDialcodeExternalEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " do nothing for the External Dialcode Data due to UNKNOWN Operation " in { - embeddedElastic.deleteIndices() - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_UNKNOWN))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) - val data = elasticUtil.getDocumentAsString("X8R3W4") - data should be(null) - } - - "Composite Search Indexer" should " sync the Dialcode Metrics Data " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_CREATE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) - val data = elasticUtil.getDocumentAsString("QR1234") - data.isEmpty should be(false) - data.contains("QR1234") should be(true) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successDialcodeMetricEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dialcodeMetricEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedDialcodeMetricEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " update the Dialcode Metrics Data " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_UPDATE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) - val data = elasticUtil.getDocumentAsString("QR1234") - data.isEmpty should be(false) - data.contains("QR1234") should be(true) - data.contains("total_dial_scans_global") should be(true) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successDialcodeMetricEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dialcodeMetricEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedDialcodeMetricEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " create and delete the Dialcode Metrics Data " in { - embeddedElastic.deleteIndices() - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_CREATE, EventFixture.DIALCODE_METRIC_DELETE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) - val data = elasticUtil.getDocumentAsString("QR1234") - data should be(null) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successDialcodeMetricEventCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.dialcodeMetricEventCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedDialcodeMetricEventCount}").getValue() should be(0) - } - - "Composite Search Indexer" should " do nothing for the Dialcode Metrics Data due to UNKNOWN Operation " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_UNKNOWN))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) - val data = elasticUtil.getDocumentAsString("QR1234") - data should be(null) - } - - "Composite Search Indexer" should " do nothing due to UNKNOWN Node Type " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.UNKNOWN_NODE_TYPE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - } - - "Composite Search Indexer" should " do nothing due to FALSE value of INDEX of the Data " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.INDEX_FALSE))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - } - - "Composite Search Indexer" should " give error for the External Dialcode Data due to UNKNOWN objectType " in { - embeddedElastic.deleteIndices() - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_FAILED))) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) - intercept[Exception] { - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - } - CompositeSearchFailedEventSink.values.forEach(value => println(value)) - } - - "Search Indexer" should " ignore the event with restricted ObjectTypes " in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_IGNORE))) - new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - } - - def getEvent(event: String, nodeGraphId: Int): Event = { - val eventMap = ScalaJsonUtil.deserialize[util.Map[String, Any]](event) - eventMap.put("nodeGraphId", nodeGraphId) - new Event(eventMap,0, 15) - } - -} - -private class CompositeSearchEventSource(events: List[String]) extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - events.foreach(event => { - ctx.collect(getEvent(event, 509674)) - }) - } - - override def cancel() = {} - - def getEvent(event: String, nodeGraphId: Int): Event = { - val eventMap = ScalaJsonUtil.deserialize[util.Map[String, Any]](event) - eventMap.put("nodeGraphId", nodeGraphId) - new Event(eventMap,0, 16) - } -} - -class CompositeSearchFailedEventSink extends SinkFunction[String] { - - override def invoke(value: String): Unit = { - synchronized { - CompositeSearchFailedEventSink.values.add(value) - } - } -} - -object CompositeSearchFailedEventSink { - val values: util.List[String] = new util.ArrayList() -} diff --git a/transaction-event-processor/README.md b/transaction-event-processor/README.md deleted file mode 100644 index 9ac836144..000000000 --- a/transaction-event-processor/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Transaction Event Processor - -Transaction Event Processor job generates an audit event for every transaction in Graph updation for content data modification. The job utilizes the neo4j mutation data to create an index of transactions for audit purposes. For each neo4j transaction, the job will audit the old and new values of the updated object. - -## Getting Started - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. -### Prerequisites - -1. Download flink-1.13.6-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.13.6/flink-1.13.6-bin-scala_2.12.tgz). -2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. -3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. -4. Docker installed. -5. A running yarn cluster or a kubernetes cluster. - -### Build - -mvn clean install - -## Deployment - -### Yarn - -Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. - -``` -./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /transaction-event-processor/target/transaction-event-processor-0.0.1.jar -``` - -### Kubernetes - -``` -# Create a single node cluster -k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 -# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. -export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" - -# Only for Mac OSX -# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl -rm /usr/local/bin/kubectl -brew link --overwrite kubernetes-cli - -# Create a configmap using the flink-configuration-configmap.yaml -kubectl create -f knowledge-platform-job/kubernetes/flink-configuration-configmap.yaml - -# Create pods for jobmanager-service, job-manager and task-manager using the yaml files -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-service.yaml -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-deployment.yaml -kubectl create -f knowledge-platform-job/kubernetes/taskmanager-deployment.yaml - -# Create a port-forwarding for accessing the job-manager UI on localhost:8081 -kubectl port-forward deployment/flink-jobmanager 8081:8081 - -# Submit the job to the Kubernetes single node cluster flink-cluster -./bin/flink run -m localhost:8081 /transaction-event-processor/target/transaction-event-processor-0.0.1.jar - -# Commands to delete the pods created in the cluster -kubectl delete deployment/flink-jobmanager -kubectl delete deployment/flink-taskmanager -kubectl delete service/flink-jobmanager -kubectl delete configmaps/flink-config - -# Command to stop the single-node cluster -k3d stop --name="flink-cluster" -``` \ No newline at end of file diff --git a/transaction-event-processor/pom.xml b/transaction-event-processor/pom.xml deleted file mode 100644 index 63beee330..000000000 --- a/transaction-event-processor/pom.xml +++ /dev/null @@ -1,249 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - transaction-event-processor - 1.0.0 - jar - transaction-event-processor - - Transaction Event Processor Flink Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - - - - - com.google.code.gson - gson - 2.10.1 - - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - org.sunbird - platform-common - 1.0-beta - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-api - - - - - org.sunbird - platform-telemetry - 1.0-beta - - - com.squareup.okhttp3 - mockwebserver - 4.9.1 - test - - - - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.transaction.task.TransactionEventProcessorStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - audit-event-generator-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - diff --git a/transaction-event-processor/src/main/resources/log4j.properties b/transaction-event-processor/src/main/resources/log4j.properties deleted file mode 100644 index 387a4a8b1..000000000 --- a/transaction-event-processor/src/main/resources/log4j.properties +++ /dev/null @@ -1,13 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=audit-event-generator.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file - - - diff --git a/transaction-event-processor/src/main/resources/transaction-event-processor.conf b/transaction-event-processor/src/main/resources/transaction-event-processor.conf deleted file mode 100644 index cad7eab9c..000000000 --- a/transaction-event-processor/src/main/resources/transaction-event-processor.conf +++ /dev/null @@ -1,27 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - output.audit.topic = "sunbirddev.telemetry.raw" - output.obsrv.topic = "output.topic" - groupId = "sunbirddev-transaction-event-processor-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - producer.parallelism = 1 - window.time = 60 -} - -schema { - basePath = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" -} - -channel.default = "org.sunbird" - -job { - audit-event-generator = false - audit-history-indexer = false - obsrv-metadata-generator = true -} diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/CaseClasses.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/CaseClasses.scala deleted file mode 100644 index e6dbfa930..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/CaseClasses.scala +++ /dev/null @@ -1,7 +0,0 @@ -package org.sunbird.job.transaction.domain - -import java.io.Serializable -import java.util.Date - -@SerialVersionUID(-5779950964487302125L) -case class AuditHistoryRecord(var objectId: String, objectType: String, label: String, graphId: String, var userId: String, requestId: String, logRecord: String, operation: String, createdOn: Date) extends Serializable \ No newline at end of file diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/Event.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/Event.scala deleted file mode 100644 index ab53d987a..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/Event.scala +++ /dev/null @@ -1,60 +0,0 @@ - -package org.sunbird.job.transaction.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest - -import java.text.{DateFormat, SimpleDateFormat} -import java.util -import java.util.Date - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val df: DateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - - private val IMAGE_SUFFIX = ".img" - - private val jobName = "TransactionEventProcessor" - - def id: String = readOrDefault("nodeUniqueId", "") - - def operationType: String = readOrDefault("operationType", null) - - def nodeUniqueId: String = readOrDefault("nodeUniqueId", "") - - def createdOn: String = readOrDefault("createdOn", "") - - def channelId(channel: String): String = readOrDefault("channel", channel) - - def objectId: String = if (null != nodeUniqueId) nodeUniqueId.replaceAll(IMAGE_SUFFIX, "") else nodeUniqueId - - def objectType: String = readOrDefault[String]("objectType", null) - - def userId: String = readOrDefault[String]("userId", "") - - def transactionData: Map[String, AnyRef] = { - readOrDefault("transactionData", new util.HashMap[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - } - - def nodeType: String = readOrDefault("nodeType", "") - - def ets: Long = readOrDefault("ets", 0L) - - def label: String = readOrDefault("label", "") - - def graphId: String = readOrDefault("graphId", "") - - def requestId: String = readOrDefault("requestId", "") - - def syncMessage: String = readOrDefault("syncMessage", null) - - def createdOnDate: Date = if (createdOn.isEmpty) new Date else df.parse(createdOn) - - def audit: Boolean = readOrDefault("audit", true) - - def isValid: Boolean = { - StringUtils.isNotBlank(objectType) || - operationType != null && null == syncMessage && audit - } - -} diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/ObsrvEvent.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/ObsrvEvent.scala deleted file mode 100644 index 87a9d1810..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/domain/ObsrvEvent.scala +++ /dev/null @@ -1,30 +0,0 @@ -package org.sunbird.job.transaction.domain - -import org.slf4j.LoggerFactory -import org.sunbird.job.domain.reader.JobRequest -import org.sunbird.telemetry.dto.Telemetry -import scala.collection.JavaConverters._ -import java.util.UUID - - -class ObsrvEvent(eventMap: java.util.Map[String, Any], override val partition: Int, override val offset: Long) extends JobRequest(eventMap, partition, offset) { - - def eventsList: List[Map[String, Any]] = readOrDefault[List[Map[String, Any]]]("events", List()) - - def dataset = readOrDefault("dataset", "sb-knowledge-master") - - def msgid = readOrDefault("mid", UUID.randomUUID().toString) - - def syncts = readOrDefault("syncts", System.currentTimeMillis()) - - def updatedList = Map("events" -> List(eventMap.asScala.toMap).foldRight(eventsList)(_ :: _)) - - def updateEvent = { - updatedList ++ Map( - "dataset" -> dataset, - "mid" -> msgid, - "syncts" -> syncts - ) - } -} - diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditEventGenerator.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditEventGenerator.scala deleted file mode 100644 index 4455c6deb..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditEventGenerator.scala +++ /dev/null @@ -1,46 +0,0 @@ -package org.sunbird.job.transaction.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.transaction.domain.Event -import org.sunbird.job.transaction.service.TransactionEventProcessorService -import org.sunbird.job.transaction.task.TransactionEventProcessorConfig -import org.sunbird.job.{BaseProcessFunction, Metrics} - -import java.util - -class AuditEventGenerator(config: TransactionEventProcessorConfig) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with TransactionEventProcessorService { - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount, config.emptySchemaEventCount, config.emptyPropsEventCount - , config.totalAuditEventsCount, config.failedAuditEventsCount, config.auditEventSuccessCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - try { - metrics.incCounter(config.totalAuditEventsCount) - processAuditEvent(event, context, metrics)(config) - } catch { - case ex: Exception => - metrics.incCounter(config.failedAuditEventsCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } -} diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala deleted file mode 100644 index 56378bd22..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.sunbird.job.transaction.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.transaction.domain.Event -import org.sunbird.job.transaction.service.TransactionEventProcessorService -import org.sunbird.job.transaction.task.TransactionEventProcessorConfig -import org.sunbird.job.util.ElasticSearchUtil -import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} - -import java.util - -class AuditHistoryIndexer(config: TransactionEventProcessorConfig, var esUtil: ElasticSearchUtil) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessKeyedFunction[String, Event, String](config) with TransactionEventProcessorService { - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.esFailedEventCount, config.skippedEventCount, - config.totalAuditHistoryEventsCount, config.failedAuditHistoryEventsCount, config.auditHistoryEventSuccessCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - if (esUtil == null) { - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.auditHistoryIndex, config.auditHistoryIndexType) - } - } - - override def close(): Unit = { - esUtil.close() - super.close() - } - - override def processElement(event: Event, - context: KeyedProcessFunction[String, Event, String]#Context, - metrics: Metrics): Unit = { - metrics.incCounter(config.totalAuditHistoryEventsCount) - processAuditHistoryEvent(event, metrics)(esUtil, config) - } -} diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/ObsrvMetaDataGenerator.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/ObsrvMetaDataGenerator.scala deleted file mode 100644 index 8c2fcc4f3..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/ObsrvMetaDataGenerator.scala +++ /dev/null @@ -1,45 +0,0 @@ -package org.sunbird.job.transaction.functions - -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.{BaseProcessFunction, Metrics} -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.transaction.domain.{Event, ObsrvEvent} -import org.sunbird.job.transaction.service.TransactionEventProcessorService -import org.sunbird.job.transaction.task.TransactionEventProcessorConfig - -import java.util - -class ObsrvMetaDataGenerator(config: TransactionEventProcessorConfig) - extends BaseProcessFunction[Event, String](config) with TransactionEventProcessorService { - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.esFailedEventCount, config.skippedEventCount, - config.totalObsrvMetaDataGeneratorEventsCount, config.failedObsrvMetaDataGeneratorEventsCount, config.obsrvMetaDataGeneratorEventsSuccessCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, metrics: Metrics): Unit = { - try { - metrics.incCounter(config.totalObsrvMetaDataGeneratorEventsCount) - processEvent(event, context, metrics)(config) - } catch { - case ex: Exception => - metrics.incCounter(config.failedObsrvMetaDataGeneratorEventsCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } -} - - diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/TransactionEventRouter.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/TransactionEventRouter.scala deleted file mode 100644 index aac34c703..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/TransactionEventRouter.scala +++ /dev/null @@ -1,51 +0,0 @@ -package org.sunbird.job.transaction.functions - -import java.util -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.transaction.domain.Event -import org.sunbird.job.transaction.service.TransactionEventProcessorService -import org.sunbird.job.transaction.task.TransactionEventProcessorConfig -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.{BaseProcessFunction, Metrics} - -class TransactionEventRouter(config: TransactionEventProcessorConfig) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessFunction[Event, String](config) with TransactionEventProcessorService { - - private[this] lazy val logger = LoggerFactory.getLogger(classOf[TransactionEventRouter]) - - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.successEventCount, config.failedEventCount, config.skippedEventCount, config.emptySchemaEventCount, config.emptyPropsEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - } - - override def close(): Unit = { - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: ProcessFunction[Event, String]#Context, - metrics: Metrics): Unit = { - try { - metrics.incCounter(config.totalEventsCount) - if (event.isValid) { - logger.info("Valid event -> " + event.nodeUniqueId) - context.output(config.outputTag,event) - }else metrics.incCounter(config.skippedEventCount) - } catch { - case ex: Exception => - metrics.incCounter(config.failedEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } -} - diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/service/TransactionEventProcessorService.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/service/TransactionEventProcessorService.scala deleted file mode 100644 index 6bce5b6c8..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/service/TransactionEventProcessorService.scala +++ /dev/null @@ -1,279 +0,0 @@ -package org.sunbird.job.transaction.service - -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.apache.commons.lang3.StringUtils -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.util.{ElasticSearchUtil, JSONUtil} -import org.sunbird.telemetry.TelemetryGenerator -import org.sunbird.telemetry.TelemetryParams -import org.sunbird.job.transaction.domain.{AuditHistoryRecord, Event, ObsrvEvent} -import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition} -import com.google.gson.Gson -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.transaction.task.TransactionEventProcessorConfig -import scala.collection.JavaConverters._ -import java.io.IOException -import java.util -import java.text.SimpleDateFormat -import java.util.{Calendar, Date, TimeZone, UUID} -import scala.collection.convert.ImplicitConversions.`map AsScala` - -trait TransactionEventProcessorService { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[TransactionEventProcessorService]) - private val OBJECT_TYPE_IMAGE_SUFFIX = "Image" - private val SKIP_AUDIT = """{"object": {"type":null}}""" - private lazy val definitionCache = new DefinitionCache - private lazy val gson = new Gson - - private val systemPropsList = List("IL_SYS_NODE_TYPE", "IL_FUNC_OBJECT_TYPE", "IL_UNIQUE_ID", "IL_TAG_NAME", "IL_ATTRIBUTE_NAME", "IL_INDEXABLE_METADATA_KEY", "IL_NON_INDEXABLE_METADATA_KEY", - "IL_IN_RELATIONS_KEY", "IL_OUT_RELATIONS_KEY", "IL_REQUIRED_PROPERTIES", "IL_SYSTEM_TAGS_KEY", "IL_SEQUENCE_INDEX", "SYS_INTERNAL_LAST_UPDATED_ON", "lastUpdatedOn", "versionKey", "lastStatusChangedOn") - - private def getContext(channelId: String, env: String): Map[String, String] = { - val context = Map( - TelemetryParams.ACTOR.name -> "org.ekstep.learning.platform", - TelemetryParams.CHANNEL.name -> channelId, - TelemetryParams.ENV.name -> env - ) - context - } - - @throws(classOf[InvalidEventException]) - def processAuditEvent(message: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics)(implicit config: TransactionEventProcessorConfig): Unit = { - logger.info("AUDIT Event::" + JSONUtil.serialize(message)) - logger.info("Input Message Received for : [" + message.nodeUniqueId + "], Txn Event createdOn:" + message.createdOn + ", Operation Type:" + message.operationType) - try { - val (auditEventStr, objectType) = getAuditMessage(message)(config, metrics) - if (StringUtils.isNotBlank(objectType)) { - context.output(config.auditOutputTag, auditEventStr) - logger.info("Telemetry Audit Message Successfully Sent for : " + message.objectId + " :: event ::" + auditEventStr) - metrics.incCounter(config.auditEventSuccessCount) - } - else { - logger.info("Skipped event as the objectype is not available, event =" + auditEventStr) - metrics.incCounter(config.emptyPropsEventCount) - } - } catch { - case e: Exception => - logger.error("Failed to process message :: " + JSONUtil.serialize(message), e) - throw e - } - } - - def processEvent(message: Event, context: ProcessFunction[Event, String]#Context, metrics: Metrics)(implicit config: TransactionEventProcessorConfig): Unit = { - val inputEvent = JSONUtil.serialize(message) - logger.info("Input Event :" + inputEvent) - logger.info("Input Message Received for : [" + message.nodeUniqueId + "], Txn Event createdOn:" + message.createdOn + ", Operation Type:" + message.operationType) - try { - val (eventStr, objectType) = getAuditMessage(message)(config, metrics) - if (StringUtils.isNotBlank(objectType)) { - val propertyMap = message.transactionData("properties").asInstanceOf[Map[String, AnyRef]] - val nvValues: Map[String, AnyRef] = propertyMap.collect { - case (key, value) if (value.isInstanceOf[Map[_, _]]) => - val nestedMap = value.asInstanceOf[Map[String, AnyRef]] - val nvValue = nestedMap.get("nv").collect { case s: String => s } - key -> nvValue.getOrElse("") - } - - message.getMap().remove("transactionData") - message.getMap().putAll(nvValues.asJava) - logger.info("Modified event: " + message) - - val obsrvEvent = new ObsrvEvent(message.getMap(), message.partition, message.offset) - val updatedEvent = obsrvEvent.updateEvent - val outputEvent = JSONUtil.serialize(updatedEvent) - - context.output(config.obsrvAuditOutputTag, outputEvent) - logger.info("Telemetry Audit Message Successfully Sent for : " + message.objectId + " :: event ::" + eventStr) - metrics.incCounter(config.obsrvMetaDataGeneratorEventsSuccessCount) - } - else { - metrics.incCounter(config.emptyPropsEventCount) - } - } catch { - case e: Exception => - logger.error("Failed to process message :: " + JSONUtil.serialize(message), e) - throw e - } - } - - - def getDefinition(objectType: String)(implicit config: TransactionEventProcessorConfig, metrics: Metrics): ObjectDefinition = { - try { - definitionCache.getDefinition(objectType, config.configVersion, config.basePath) - } catch { - case ex: Exception => { - metrics.incCounter(config.emptySchemaEventCount) - new ObjectDefinition(objectType, config.configVersion, Map[String, AnyRef](), Map[String, AnyRef]()) - } - } - } - - - def getAuditMessage(message: Event)(implicit config: TransactionEventProcessorConfig, metrics: Metrics): (String, String) = { - var auditMap: String = null - var objectType = message.objectType - val env = if (null != objectType) objectType.toLowerCase.replace("image", "") else "system" - - val definitionNode: ObjectDefinition = getDefinition(objectType) - - val propertyMap = message.transactionData("properties").asInstanceOf[Map[String, AnyRef]] - val statusMap = propertyMap.getOrElse("status", null).asInstanceOf[Map[String, AnyRef]] - val lastStatusChangedOn = propertyMap.getOrElse("lastStatusChangedOn", null).asInstanceOf[Map[String, AnyRef]] - val addedRelations = message.transactionData.getOrElse("addedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - val removedRelations = message.transactionData.getOrElse("removedRelations", List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - - var pkgVersion = "" - var prevStatus = "" - var currStatus = "" - var duration = "" - val pkgVerMap = propertyMap.getOrElse("pkgVersion", null).asInstanceOf[Map[String, AnyRef]] - if (null != pkgVerMap) pkgVersion = s"${pkgVerMap.get("nv")}" - - if (null != statusMap) { - prevStatus = statusMap.getOrElse("ov", null).asInstanceOf[String] - currStatus = statusMap.getOrElse("nv", null).asInstanceOf[String] - // Compute Duration for Status Change - if (StringUtils.isNotBlank(currStatus) && StringUtils.isNotBlank(prevStatus) && null != lastStatusChangedOn) { - var ov = lastStatusChangedOn.getOrElse("ov", null).asInstanceOf[String] - val nv = lastStatusChangedOn.getOrElse("nv", null).asInstanceOf[String] - if (null == ov) ov = propertyMap.getOrElse("lastUpdatedOn", null).asInstanceOf[Map[String, AnyRef]].getOrElse("ov", null).asInstanceOf[String] - if (null != ov && null != nv) duration = String.valueOf(computeDuration(ov, nv)) - } - } - - var props: List[String] = propertyMap.keys.toList - props ++= getRelationProps(addedRelations, definitionNode) - props ++= getRelationProps(removedRelations, definitionNode) - val propsExceptSystemProps = props.filter(prop => !systemPropsList.contains(prop)) - val cdata = getCData(addedRelations, removedRelations, propertyMap) - - var context: Map[String, String] = getContext(message.channelId(config.defaultChannel), env) - - objectType = if (null != objectType) objectType.replaceAll(OBJECT_TYPE_IMAGE_SUFFIX, "") else objectType - context ++= Map("objectId" -> message.objectId, "objectType" -> objectType) - - if (StringUtils.isNotBlank(duration)) context ++= Map("duration" -> duration) - if (StringUtils.isNotBlank(pkgVersion)) context ++= Map("pkgVersion" -> pkgVersion) - if (StringUtils.isNotBlank(message.userId)) context ++= Map(TelemetryParams.ACTOR.name -> message.userId) - - if (propsExceptSystemProps.nonEmpty) { - val cdataList = gson.fromJson(JSONUtil.serialize(cdata), classOf[java.util.List[java.util.Map[String, Object]]]) - - TelemetryGenerator.setComponent("audit-event-generator") - auditMap = TelemetryGenerator.audit( - JSONUtil.deserialize[util.Map[String, String]](JSONUtil.serialize(context)), - JSONUtil.deserialize[util.List[String]](JSONUtil.serialize(propsExceptSystemProps)), - currStatus, - prevStatus, - cdataList) - logger.info("Audit Message for Content Id [" + message.objectId + "] : " + auditMap); - - (auditMap, message.objectType) - } - else { - logger.info("Skipping Audit log as props is null or empty") - (SKIP_AUDIT, "") - } - } - - /** - * @param addedRelations - * @param removedRelations - * @param propertyMap - * @return - */ - private def getCData(addedRelations: List[Map[String, AnyRef]], removedRelations: List[Map[String, AnyRef]], propertyMap: Map[String, AnyRef]): List[Map[String, AnyRef]] = { - var cdata = List[Map[String, AnyRef]]() - if (null != propertyMap && propertyMap.nonEmpty && propertyMap.contains("dialcodes")) { - val dialcodeMap = propertyMap("dialcodes").asInstanceOf[Map[String, AnyRef]] - val dialcodes = dialcodeMap("nv").asInstanceOf[List[String]] - if (null != dialcodes) { - cdata :+= Map[String, AnyRef]("id" -> dialcodes, "type" -> "DialCode") - } - } - if (null != addedRelations && addedRelations.nonEmpty) cdata ++= prepareCMap(addedRelations) - if (null != removedRelations && removedRelations.nonEmpty) cdata ++= prepareCMap(removedRelations) - cdata - } - - /** - * @param relations - */ - private def prepareCMap(relations: List[Map[String, AnyRef]]): List[Map[String, AnyRef]] = { - relations.map(relation => { - Map[String, AnyRef]("id" -> relation("id"), "type" -> relation("type")) - }) - } - - /** - * - * @param relations - */ - private def getRelationProps(relations: List[Map[String, AnyRef]], objectDefinition: ObjectDefinition)(implicit config: TransactionEventProcessorConfig): List[String] = { - var relationProps = List[String]() - if (relations.nonEmpty) { - relations.foreach(rel => { - val direction = rel.getOrElse("dir", "").asInstanceOf[String] - val relationType = rel.getOrElse("rel", "").asInstanceOf[String] - val targetObjType = rel.getOrElse("type", "").asInstanceOf[String] - val relationProp = objectDefinition.relationLabel(targetObjType, direction, relationType) - if (relationProp.nonEmpty) { - relationProps :+= relationProp.get - } - }) - } - relationProps - } - - /** - * @param oldDate - * @param newDate - * @return - */ - def computeDuration(oldDate: String, newDate: String): Long = { - val sdf: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - val od = sdf.parse(oldDate) - val nd = sdf.parse(newDate) - val diff = nd.getTime - od.getTime - val diffSeconds = diff / 1000 - diffSeconds - } - - @throws(classOf[InvalidEventException]) - def processAuditHistoryEvent(event: Event, metrics: Metrics)(implicit esUtil: ElasticSearchUtil, config: TransactionEventProcessorConfig): Unit = { - if (event.isValid) { - val identifier = event.nodeUniqueId - logger.info("Audit learning event received : " + identifier) - try { - val record = getAuditHistory(event) - val document = JSONUtil.serialize(record) - val indexName = getIndexName(event.ets) - esUtil.addDocumentWithIndex(document, indexName) - logger.info("Audit record created for " + identifier) - metrics.incCounter(config.auditHistoryEventSuccessCount) - } catch { - case ex: IOException => - logger.error("Error while indexing message :: " + event.getJson + " :: " + ex.getMessage) - ex.printStackTrace() - metrics.incCounter(config.esFailedEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - case ex: Exception => - logger.error("Error while processing message :: " + event.getJson + " :: ", ex) - metrics.incCounter(config.failedAuditHistoryEventsCount) - } - } - else logger.info("Learning event not qualified for audit") - } - - private def getIndexName(ets: Long)(implicit config: TransactionEventProcessorConfig): String = { - val cal = Calendar.getInstance(TimeZone.getTimeZone(config.timeZone)) - cal.setTime(new Date(ets)) - config.auditHistoryIndex + "_" + cal.get(Calendar.YEAR) + "_" + cal.get(Calendar.WEEK_OF_YEAR) - } - - def getAuditHistory(transactionDataMap: Event): AuditHistoryRecord = { - val nodeUniqueId = StringUtils.replace(transactionDataMap.nodeUniqueId, ".img", "") - AuditHistoryRecord(nodeUniqueId, transactionDataMap.objectType, transactionDataMap.label, transactionDataMap.graphId, transactionDataMap.userId, transactionDataMap.requestId, JSONUtil.serialize(transactionDataMap.transactionData), transactionDataMap.operationType, transactionDataMap.createdOnDate) - } -} \ No newline at end of file diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/task/TransactionEventProcessorConfig.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/task/TransactionEventProcessorConfig.scala deleted file mode 100644 index 67022e435..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/task/TransactionEventProcessorConfig.scala +++ /dev/null @@ -1,82 +0,0 @@ -package org.sunbird.job.transaction.task - -import java.util -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.scala.OutputTag -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.transaction.domain.Event - -class TransactionEventProcessorConfig(override val config: Config) extends BaseJobConfig(config, "transaction-event-processor") { - - private val serialVersionUID = 2905979434303791379L - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val kafkaAuditOutputTopic: String = config.getString("kafka.output.audit.topic") - val kafkaObsrvOutputTopic: String = config.getString("kafka.output.obsrv.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - - //Flink Jobs Configuration - val kafkaProducerParallelism: Int = config.getInt("task.producer.parallelism") - val auditEventGenerator: Boolean = if (config.hasPath("job.audit-event-generator")) config.getBoolean("job.audit-event-generator") else true - val auditHistoryIndexer: Boolean = if (config.hasPath("job.audit-history-indexer")) config.getBoolean("job.audit-history-indexer") else false - val obsrvMetadataGenerator: Boolean = if (config.hasPath("job.obsrv-metadata-generator")) config.getBoolean("job.obsrv-metadata-generator") else false - - val outputTag: OutputTag[Event] = OutputTag[Event]("output-tag") - val auditOutputTag: OutputTag[String] = OutputTag[String]("audit-event-tag") - val obsrvAuditOutputTag: OutputTag[String] = OutputTag[String]("obsrv-metadata-tag") - - val defaultChannel: String = config.getString("channel.default") - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val skippedEventCount = "skipped-events-count" - val emptySchemaEventCount = "empty-schema-events-count" - val emptyPropsEventCount = "empty-props-events-count" - val esFailedEventCount = "elasticsearch-error-events-count" - - //Audit Event Generator Metric List - val totalAuditEventsCount = "total-audit-events-count" - val failedAuditEventsCount = "failed-audit-events-count" - val auditEventSuccessCount = "audit-event-success-count" - - //Audit Event Generator Metric List - val totalAuditHistoryEventsCount = "total-audit-history-events-count" - val failedAuditHistoryEventsCount = "failed-audit-history-events-count" - val auditHistoryEventSuccessCount = "audit-history-event-success-count" - - //Obsrv MetaData Generator Metric List - val totalObsrvMetaDataGeneratorEventsCount = "total-obsrv-metadata-events-count" - val failedObsrvMetaDataGeneratorEventsCount = "failed-obsrv-metadata-events-count" - val obsrvMetaDataGeneratorEventsSuccessCount = "audit-obsrv-metadata-success-count" - - - // Consumers - val transactionEventConsumer = "transaction-event-processor-consumer" - val auditEventGeneratorFunction = "audit-event-generator-function" - val auditHistoryIndexerFunction = "audit-history-indexer-function" - val obsrvMetaDataGeneratorFunction = "obsrv-metadata-generator-function" - val transactionEventRouterFunction = "transaction-event-router-function" - val auditEventProducer = "audit-event-generator-producer" - val obsrvEventProducer = "obsrv-metadata-generator-producer" - - - val basePath = config.getString("schema.basePath") - val configVersion = "1.0" - - // ES Configs - val esConnectionInfo = config.getString("es.basePath") - - val timeZone = if (config.hasPath("timezone")) config.getString("timezone") else "IST" - val auditHistoryIndex = "kp_audit_log" - val auditHistoryIndexType = "ah" -} diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/task/TransactionEventProcessorStreamTask.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/task/TransactionEventProcessorStreamTask.scala deleted file mode 100644 index e6d8c390f..000000000 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/task/TransactionEventProcessorStreamTask.scala +++ /dev/null @@ -1,84 +0,0 @@ -package org.sunbird.job.transaction.task - -import java.io.File -import java.util -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.functions.KeySelector -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.slf4j.LoggerFactory -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.transaction.domain.Event -import org.sunbird.job.transaction.functions.{AuditEventGenerator, AuditHistoryIndexer, ObsrvMetaDataGenerator, TransactionEventRouter} -import org.sunbird.job.util.{ElasticSearchUtil, FlinkUtil} - - -class TransactionEventProcessorStreamTask(config: TransactionEventProcessorConfig, kafkaConnector: FlinkKafkaConnector, esUtil: ElasticSearchUtil) { - - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - // implicit val env: StreamExecutionEnvironment = StreamExecutionEnvironment.createLocalEnvironment() - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val mapEsTypeInfo: TypeInformation[util.Map[String, Any]] = TypeExtractor.getForClass(classOf[util.Map[String, Any]]) - - val inputStream = env.addSource(kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic)).name(config.transactionEventConsumer) - .uid(config.transactionEventConsumer).setParallelism(config.kafkaConsumerParallelism) - - val processorStreamTask = inputStream.rebalance - .process(new TransactionEventRouter(config)) - .name(config.transactionEventRouterFunction) - .uid(config.transactionEventRouterFunction) - .setParallelism(config.parallelism) - - val sideOutput = processorStreamTask.getSideOutput(config.outputTag) - - if (config.auditEventGenerator) { - val auditStream = sideOutput.process(new AuditEventGenerator(config)).name(config.auditEventGeneratorFunction) - .uid(config.auditEventGeneratorFunction).setParallelism(config.parallelism) - - auditStream.getSideOutput(config.auditOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaAuditOutputTopic)) - .name(config.auditEventProducer).uid(config.auditEventProducer).setParallelism(config.kafkaProducerParallelism) - } - - if (config.obsrvMetadataGenerator) { - val obsrvStream = sideOutput.process(new ObsrvMetaDataGenerator(config)).name(config.obsrvMetaDataGeneratorFunction) - .uid(config.obsrvMetaDataGeneratorFunction).setParallelism(config.parallelism) - - obsrvStream.getSideOutput(config.obsrvAuditOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaObsrvOutputTopic)) - .name(config.obsrvEventProducer).uid(config.obsrvEventProducer).setParallelism(config.kafkaProducerParallelism) - } - - if (config.auditHistoryIndexer) { - sideOutput.keyBy(new TransactionEventKeySelector).process(new AuditHistoryIndexer(config, esUtil)).name(config.auditHistoryIndexerFunction) - .uid(config.auditHistoryIndexerFunction).setParallelism(config.parallelism) - } - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object TransactionEventProcessorStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("transaction-event-processor.conf").withFallback(ConfigFactory.systemEnvironment())) - val transactionEventProcessorConfig = new TransactionEventProcessorConfig(config) - val kafkaUtil = new FlinkKafkaConnector(transactionEventProcessorConfig) - val esUtil: ElasticSearchUtil = null - val task = new TransactionEventProcessorStreamTask(transactionEventProcessorConfig, kafkaUtil, esUtil) - task.process() - } -} - -// $COVERAGE-ON$ - -class TransactionEventKeySelector extends KeySelector[Event, String] { - override def getKey(in: Event): String = in.id.replace(".img", "") -} diff --git a/transaction-event-processor/src/test/resources/logback-test.xml b/transaction-event-processor/src/test/resources/logback-test.xml deleted file mode 100644 index aebe06a10..000000000 --- a/transaction-event-processor/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/transaction-event-processor/src/test/resources/test.conf b/transaction-event-processor/src/test/resources/test.conf deleted file mode 100644 index 7b3dd3fc8..000000000 --- a/transaction-event-processor/src/test/resources/test.conf +++ /dev/null @@ -1,27 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.learning.graph.events" - output.audit.topic = "sunbirddev.telemetry.raw" - output.obsrv.topic = "output.topic" - groupId = "sunbirddev-transaction-event-processor-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - producer.parallelism = 1 - window.time = 60 -} - -schema { - basePath = "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/schemas/local" -} - -channel.default = "org.sunbird" - -job { - audit-event-generator = false - audit-history-indexer = false - obsrv-metadata-generator = false -} diff --git a/transaction-event-processor/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/transaction-event-processor/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index ea4e9f2d3..000000000 --- a/transaction-event-processor/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,74 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"ets":1518517878987,"nodeUniqueId":"do_11243969846440755213","requestId":null,"transactionData":{"properties":{"code":{"ov":null,"nv":"test_code"},"keywords":{"ov":null,"nv":["colors","games"]},"channel":{"ov":null,"nv":"in.ekstep"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/pdf"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentDisposition":{"ov":null,"nv":"inline"},"contentEncoding":{"ov":null,"nv":"identity"},"lastUpdatedOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentType":{"ov":null,"nv":"Story"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"versionKey":{"ov":null,"nv":"1518517878947"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"framework":{"ov":null,"nv":"NCF"},"compatibilityLevel":{"ov":null,"nv":1.0},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"Untitled Resource"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_11243969846440755213"},"status":{"ov":null,"nv":"Draft"},"resourceType":{"ov":null,"nv":"Story"}}},"operationType":"CREATE","nodeGraphId":113603,"label":"Untitled Resource","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2018-02-13T16:01:18.987+0530","objectType":"Content"} - |""".stripMargin - - val EVENT_2: String = - """ - |{"ets":1552464504681,"channel":"in.ekstep","transactionData":{"properties":{"lastStatusChangedOn":{"ov":"2019-03-13T13:25:43.129+0530","nv":"2019-03-13T13:38:24.358+0530"},"lastSubmittedOn":{"ov":null,"nv":"2019-03-13T13:38:21.901+0530"},"lastUpdatedOn":{"ov":"2019-03-13T13:36:20.093+0530","nv":"2019-03-13T13:38:24.399+0530"},"status":{"ov":"Draft","nv":"Review"},"versionKey":{"ov":"1552464380093","nv":"1552464504399"}}},"label":"Resource Content 1","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2019-03-13T13:38:24.680+0530","objectType":"Content","nodeUniqueId":"do_11271778298376192013","requestId":null,"operationType":"UPDATE","nodeGraphId":590883,"graphId":"domain"} - |""".stripMargin - - val EVENT_3: String = - """ - |{"ets":1552464380225,"channel":"in.ekstep","transactionData":{"properties":{"s3Key":{"ov":null,"nv":"content/do_11271778298376192013/artifact/pdf_1552464372724.pdf"},"size":{"ov":null,"nv":433994.0},"artifactUrl":{"ov":null,"nv":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11271778298376192013/artifact/pdf_1552464372724.pdf"},"lastUpdatedOn":{"ov":"2019-03-13T13:25:43.129+0530","nv":"2019-03-13T13:36:20.093+0530"},"versionKey":{"ov":"1552463743129","nv":"1552464380093"}}},"label":"Resource Content 1","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2019-03-13T13:36:20.223+0530","objectType":"Content","nodeUniqueId":"do_11271778298376192013","requestId":null,"operationType":"UPDATE","nodeGraphId":590883,"graphId":"domain"} - |""".stripMargin - - val EVENT_4: String = - """ - |{"ets":1552645516180,"channel":"in.ekstep","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"test.res.1"},"channel":{"ov":null,"nv":"in.ekstep"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/pdf"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2019-03-15T15:55:16.071+0530"},"contentDisposition":{"ov":null,"nv":"inline"},"lastUpdatedOn":{"ov":null,"nv":"2019-03-15T15:55:16.071+0530"},"contentEncoding":{"ov":null,"nv":"identity"},"dialcodeRequired":{"ov":null,"nv":"No"},"contentType":{"ov":null,"nv":"Resource"},"lastStatusChangedOn":{"ov":null,"nv":"2019-03-15T15:55:16.071+0530"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"visibility":{"ov":null,"nv":"Default"},"os":{"ov":null,"nv":["All"]},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"versionKey":{"ov":null,"nv":"1552645516071"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"framework":{"ov":null,"nv":"NCF"},"compatibilityLevel":{"ov":null,"nv":1.0},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"Resource Content 1"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_11271927206783385611"},"status":{"ov":null,"nv":"Draft"}}},"mid":"9ea9ae7a-9cc1-493d-aac3-3c66cd9ff01b","label":"Resource Content 1","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2019-03-15T15:55:16.178+0530","objectType":"Content","nodeUniqueId":"do_11271927206783385611","requestId":null,"operationType":"CREATE","nodeGraphId":590921,"graphId":"domain"} - |""".stripMargin - - val EVENT_5: String = - """ - |{"ets":1.614227716965E12,"transactionData":{"removedTags":[],"addedRelations":[{"rel":"hasSequenceMember","id":"do_11322375329257062411","label":"event 4.1: 11-1","dir":"OUT","type":"Event","relMetadata":{"IL_SEQUENCE_INDEX":1.0}}],"removedRelations":[],"addedTags":[],"properties":{}},"mid":"fc0bb006-7269-4b10-96c3-10672fca53a0","label":"eventset 4","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-02-25T04:35:16.965+0000","objectType":null,"partition":1,"nodeUniqueId":"do_11322375344215654413","operationType":"UPDATE","nodeGraphId":509461.0,"graphId":"domain"} - |""".stripMargin - - val EVENT_6: String = - """ - |{"ets":1502102183388,"nodeUniqueId":"do_112276071067320320114","requestId":null,"transactionData":{"addedRelations":[{"rel":"hasSequenceMember","id":"do_1123032073439723521148","label":"Test unit 11","dir":"IN","type":"Content"}],"removedRelations":[],"properties":{"name":{"nv":"","ov":""}}},"operationType":"CREATE","nodeGraphId":105631,"label":"collaborator test","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-08-07T10:36:23.388+0000","objectType":"Content"} - |""".stripMargin - - val EVENT_7: String = - """ - |{"ets":1614776853781,"channel":"b00bc992ef25f1a9a8d63291e20efc8d","transactionData":{"properties":{"dialcodes":{"ov":null,"nv":["K1W6L6"]}}},"mid":"5b5633a2-3c18-49a6-8822-6d7b85338104","label":"Test Again","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-03-03T13:07:33.781+0000","objectType":"Collection","nodeUniqueId":"do_1132282511204024321262","requestId":null,"operationType":"UPDATE","nodeGraphId":510086,"graphId":"domain"} - |""".stripMargin - - val EVENT_8: String = - """ - |{"ets":1.614227716965E12,"transactionData":{"removedTags":[],"addedRelations":[{"rel":"hasSequenceMember","id":"do_11322375329257062411","label":"event 4.1: 11-1","dir":"OUT","type":"Event","relMetadata":{"IL_SEQUENCE_INDEX":1.0}}],"removedRelations":[],"addedTags":[],"properties":{}},"mid":"fc0bb006-7269-4b10-96c3-10672fca53a0","label":"eventset 4","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-02-25T04:35:16.965+0000","objectType":"RandomSchemaType","partition":1,"nodeUniqueId":"do_11322375344215654413","operationType":"UPDATE","nodeGraphId":509461.0,"graphId":"domain"} - |""".stripMargin - - val EVENT_9: String = - """ - |{"ets":1518517878987,"nodeUniqueId":"do_11243969846440755213","requestId":null,"transactionData":{"properties":{"code":{"ov":null,"nv":"test_code"},"keywords":{"ov":null,"nv":["colors","games"]},"channel":{"ov":null,"nv":"in.ekstep"},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/pdf"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentDisposition":{"ov":null,"nv":"inline"},"contentEncoding":{"ov":null,"nv":"identity"},"lastUpdatedOn":{"ov":null,"nv":"2018-02-13T16:01:18.947+0530"},"contentType":{"ov":null,"nv":"Story"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"os":{"ov":null,"nv":["All"]},"visibility":{"ov":null,"nv":"Default"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"versionKey":{"ov":null,"nv":"1518517878947"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"framework":{"ov":null,"nv":"NCF"},"compatibilityLevel":{"ov":null,"nv":1.0},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"Untitled Resource"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_11243969846440755213"},"status":{"ov":null,"nv":"Draft"},"resourceType":{"ov":null,"nv":"Story"}}},"operationType":"CREATE","nodeGraphId":113603,"label":"Untitled Resource","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2018-02-13T16:01:18.987+0530","objectType":"Content"} - |""".stripMargin - - val EVENT_10: String = - """ - |{"ets":1502102183388,"nodeUniqueId":"do_112276071067320320114","requestId":null,"transactionData":{"addedRelations":[{"rel":"hasSequenceMember","id":"do_1123032073439723521148","label":"Test unit 11","dir":"IN","type":"Content"}],"removedRelations":[],"properties":{"name":{"nv":"","ov":""}}},"operationType":"CREATE","nodeGraphId":105631,"label":"collaborator test","graphId":"domain","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-08-07T10:36:23.388+0000","objectType":"Content"} - |""".stripMargin - - val EVENT_11: String = - """ - |{"ets":1615191835547,"channel":"01309282781705830427","transactionData":{"removedTags":[],"addedRelations":[],"removedRelations":[{"rel":"associatedTo","id":"do_113198273083662336127","label":"qq\n","dir":"OUT","type":"AssessmentItem","relMetadata":{}}],"addedTags":[],"properties":{}},"mid":"98145983-63dc-4d55-866c-248d49306ad8","label":"ECML_CHANGES","nodeType":"DATA_NODE","userId":"5a587cc1-e018-4859-a0a8-e842650b9d64","createdOn":"2021-03-08T08:23:55.547+0000","objectType":"Content","nodeUniqueId":"do_1132316371218268161118","requestId":null,"operationType":"UPDATE","nodeGraphId":510477,"graphId":"domain"} - |""".stripMargin - - val EVENT_12: String = - """ - |{"ets":1500888709490,"requestId":null,"transactionData":{"properties":{"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"morphology":{"ov":null,"nv":false},"consumerId":{"ov":null,"nv":"a6654129-b58d-4dd8-9cf2-f8f3c2f458bc"},"channel":{"ov":null,"nv":"in.ekstep"},"lemma":{"ov":null,"nv":"ವಿಶ್ಲೇಷಣೆ"},"createdOn":{"ov":null,"nv":"2017-07-24T09:32:18.130+0000"},"versionKey":{"ov":null,"nv":"1500888738130"},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Word"},"ekstepWordnet":{"ov":null,"nv":false},"lastUpdatedOn":{"ov":null,"nv":"2017-07-24T09:32:18.130+0000"},"isPhrase":{"ov":null,"nv":false},"IL_UNIQUE_ID":{"ov":null,"nv":"ka_11229528054276096015"},"status":{"ov":null,"nv":"Draft"}}},"nodeGraphId":433342,"label":"ವಿಶ್ಲೇಷಣೆ","graphId":"ka","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2017-07-24T09:31:49.490+0000","objectType":"Word"} - |""".stripMargin - - val EVENT_13: String = - """ - |{"ets":1614776853781,"channel":"b00bc992ef25f1a9a8d63291e20efc8d","transactionData":{"properties":{"dialcodes":{"ov":null,"nv":["K1W6L6"]}}},"mid":"5b5633a2-3c18-49a6-8822-6d7b85338104","label":"Test Again","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-03-03T13:07:33.781+0000","nodeUniqueId":"do_1132282511204024321262","requestId":null,"operationType":"UPDATE","nodeGraphId":510086,"graphId":"domain"} - |""".stripMargin - - val EVENT_14: String = - """ - |{"ets":1552464380225,"channel":"in.ekstep","transactionData":{"properties":{"s3Key":{"ov":null,"nv":"content/do_11271778298376192013/artifact/pdf_1552464372724.pdf"},"size":{"ov":null,"nv":433994.0},"artifactUrl":{"ov":null,"nv":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11271778298376192013/artifact/pdf_1552464372724.pdf"},"lastUpdatedOn":{"ov":"2019-03-13T13:25:43.129+0530","nv":"2019-03-13T13:36:20.093+0530"},"versionKey":{"ov":"1552463743129","nv":"1552464380093"}}},"label":"Resource Content 1","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2019-03-13T13:36:20.223+0530","objectType":" ","nodeUniqueId":"do_11271778298376192013","requestId":null,"operationType":"UPDATE","nodeGraphId":590883,"graphId":"domain"} - |""".stripMargin -} \ No newline at end of file diff --git a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala b/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala deleted file mode 100644 index 4e3a98231..000000000 --- a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala +++ /dev/null @@ -1,309 +0,0 @@ -package org.sunbird.job.spec - -import java.util -import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} -import okhttp3.mockwebserver.{MockResponse, MockWebServer} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.client.JobExecutionException -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.sink.SinkFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.mockito.ArgumentMatchers.any -import org.sunbird.job.util.{ElasticSearchUtil, JSONUtil} -import org.mockito.Mockito -import org.mockito.Mockito.when -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.transaction.domain.Event -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.transaction.task.{TransactionEventProcessorConfig, TransactionEventProcessorStreamTask} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -class TransactionEventProcessorTaskTestSpec extends BaseTestSpec { - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig: TransactionEventProcessorConfig = new TransactionEventProcessorConfig(config) - val esUtil: ElasticSearchUtil = null - val server = new MockWebServer() - - var currentMilliSecond = 1605816926271L - - override protected def beforeAll(): Unit = { - BaseMetricsReporter.gaugeMetrics.clear() - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - flinkCluster.after() - super.afterAll() - } - - "TransactionEventProcessorStreamTask" should "handle invalid events and increase metric count" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new failedEventMapSource) - try { - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(0) - throw new InvalidEventException(any[String]) - } - } - - "TransactionEventProcessorStreamTask" should "skip events and increase metric count" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new skippedEventMapSource) - try { - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - throw new InvalidEventException(any[String]) - } - } - - "TransactionEventProcessorStreamTask" should "generate audit event" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditEventMapSource) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaAuditOutputTopic)).thenReturn(new AuditEventSink) - val setBoolean = config.withValue("job.audit-event-generator", ConfigValueFactory.fromAnyRef(true)) - val newConfig: TransactionEventProcessorConfig = new TransactionEventProcessorConfig(setBoolean) - if (newConfig.auditEventGenerator) { - new TransactionEventProcessorStreamTask(newConfig, mockKafkaUtil, esUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.totalAuditEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.auditEventSuccessCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.failedAuditEventsCount}").getValue() should be(0) - - AuditEventSink.values.size() should be(1) - AuditEventSink.values.forEach(event => { - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](event) - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - }) - } - } - - "TransactionEventProcessorStreamTask" should "not generate audit event" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditEventMapSource) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaAuditOutputTopic)).thenReturn(new AuditEventSink) - - if (jobConfig.auditEventGenerator) { - - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalAuditEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.auditEventSuccessCount}").getValue() should be(0) - - } - } - - "TransactionEventProcessorStreamTask" should "increase metric for unknown schema" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new RandomObjectTypeAuditEventGeneratorMapSource) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaAuditOutputTopic)).thenReturn(new AuditEventSink) - if (jobConfig.auditEventGenerator) { - - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalAuditEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.emptySchemaEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.emptyPropsEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.auditEventSuccessCount}").getValue() should be(0) - } - } - - "TransactionEventProcessorStreamTask" should "not generate audit history indexer event" in { - server.start(9200) - server.enqueue(new MockResponse().setHeader( - "Content-Type", "application/json" - ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryMapSource) - if (jobConfig.auditHistoryIndexer) { - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalAuditHistoryEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.auditHistoryEventSuccessCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedAuditHistoryEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.esFailedEventCount}").getValue() should be(1) - } - } - - "TransactionEventProcessorStreamTask" should "generate audit history indexer event" in { - server.enqueue(new MockResponse().setHeader( - "Content-Type", "application/json" - ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryMapSource) - val setBoolean = config.withValue("job.audit-history-indexer", ConfigValueFactory.fromAnyRef(true)) - val newConfig: TransactionEventProcessorConfig = new TransactionEventProcessorConfig(setBoolean) - if (newConfig.auditHistoryIndexer) { - new TransactionEventProcessorStreamTask(newConfig, mockKafkaUtil, esUtil).process() - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.totalAuditHistoryEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.auditHistoryEventSuccessCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.failedAuditHistoryEventsCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.esFailedEventCount}").getValue() should be(0) - server.close() - } - } - - "TransactionEventProcessorStreamTask" should "throw exception and increase es error count" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryMapSource) - - try { - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalAuditHistoryEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.auditHistoryEventSuccessCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedAuditHistoryEventsCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.esFailedEventCount}").getValue() should be(1) - } - } - - "TransactionEventProcessorStreamTask" should "not generate obsrv event" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditEventMapSource) - when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaObsrvOutputTopic)).thenReturn(new AuditEventSink) - if (jobConfig.obsrvMetadataGenerator) { - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalObsrvMetaDataGeneratorEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.obsrvMetaDataGeneratorEventsSuccessCount}").getValue() should be(0) - } - } - - "TransactionEventProcessorStreamTask" should "generate obsrv event" in { - val setBoolean = config.withValue("job.obsrv-metadata-generator", ConfigValueFactory.fromAnyRef(true)) - val newConfig: TransactionEventProcessorConfig = new TransactionEventProcessorConfig(setBoolean) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](newConfig.kafkaInputTopic)).thenReturn(new EventMapSource) - when(mockKafkaUtil.kafkaStringSink(newConfig.kafkaObsrvOutputTopic)).thenReturn(new AuditEventSink) - - if (newConfig.obsrvMetadataGenerator) { - new TransactionEventProcessorStreamTask(newConfig, mockKafkaUtil, esUtil).process() - - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.totalObsrvMetaDataGeneratorEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.obsrvMetaDataGeneratorEventsSuccessCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.failedObsrvMetaDataGeneratorEventsCount}").getValue() should be(0) - } - } - - "TransactionEventProcessorStreamTask" should "increase metrics and throw exception for invalid event" in { - val setBoolean = config.withValue("job.obsrv-metadata-generator", ConfigValueFactory.fromAnyRef(true)) - val newConfig: TransactionEventProcessorConfig = new TransactionEventProcessorConfig(setBoolean) - - when(mockKafkaUtil.kafkaJobRequestSource[Event](newConfig.kafkaInputTopic)).thenReturn(new EventMapSource) - when(mockKafkaUtil.kafkaStringSink(newConfig.kafkaObsrvOutputTopic)).thenReturn(new AuditEventSink) - - try { - new TransactionEventProcessorStreamTask(newConfig, mockKafkaUtil, esUtil).process() - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.totalObsrvMetaDataGeneratorEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.obsrvMetaDataGeneratorEventsSuccessCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${newConfig.jobName}.${newConfig.failedObsrvMetaDataGeneratorEventsCount}").getValue() should be(1) - } - } - - "TransactionEventProcessorStreamTask" should "throw exception in TransactionEventRouter" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new failedEventMapSource) - - try { - new TransactionEventProcessorStreamTask(jobConfig, mockKafkaUtil, esUtil).process() - } catch { - case ex: JobExecutionException => - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(0) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.failedEventCount}").getValue() should be(1) - } - } -} - -class AuditEventMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1), 0, 10)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_5), 0, 11)) - } - - override def cancel(): Unit = {} -} - -class AuditHistoryMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_9), 0, 10)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_12), 0, 11)) - } - - override def cancel(): Unit = {} -} - -class EventMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2), 0, 10)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3), 0, 10)) - } - - override def cancel(): Unit = {} -} - -class failedEventMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_14), 0, 10)) - } - - override def cancel(): Unit = {} -} - -class skippedEventMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_13), 0, 10)) - } - - override def cancel(): Unit = {} -} - -class RandomObjectTypeAuditEventGeneratorMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_8), 0, 10)) - } - - override def cancel(): Unit = {} -} - -class AuditEventSink extends SinkFunction[String] { - - override def invoke(value: String): Unit = { - synchronized { - AuditEventSink.values.add(value) - } - } -} - -object AuditEventSink { - val values: util.List[String] = new util.ArrayList() -} \ No newline at end of file diff --git a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/service/TransactionEventProcessorServiceTestSpec.scala b/transaction-event-processor/src/test/scala/org/sunbird/job/spec/service/TransactionEventProcessorServiceTestSpec.scala deleted file mode 100644 index cbc31d757..000000000 --- a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/service/TransactionEventProcessorServiceTestSpec.scala +++ /dev/null @@ -1,283 +0,0 @@ -package org.sunbird.job.spec.service - -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.commons.lang3.StringUtils -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.streaming.api.functions.ProcessFunction -import org.junit.Assert.assertFalse -import org.mockito.ArgumentMatchers.anyString -import org.mockito.Mockito -import org.mockito.Mockito.{verify, when} -import org.slf4j.Logger -import org.sunbird.job.Metrics -import org.sunbird.job.transaction.domain.{AuditHistoryRecord, Event} -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.transaction.functions.{AuditEventGenerator, AuditHistoryIndexer, ObsrvMetaDataGenerator, TransactionEventRouter} -import org.sunbird.job.transaction.service.TransactionEventProcessorService -import org.sunbird.job.transaction.task.TransactionEventProcessorConfig -import org.sunbird.job.util.{ElasticSearchUtil, JSONUtil} -import org.sunbird.spec.BaseTestSpec - -import java.util -import java.util.Date - -class TransactionEventProcessorServiceTestSpec extends BaseTestSpec with TransactionEventProcessorService { - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - implicit val mpTypeInfo: TypeInformation[util.Map[String, Any]] = TypeExtractor.getForClass(classOf[util.Map[String, Any]]) - implicit val strTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - - val config: Config = ConfigFactory.load("test.conf") - lazy val jobConfig: TransactionEventProcessorConfig = new TransactionEventProcessorConfig(config) - lazy val mockMetrics = mock[Metrics](Mockito.withSettings().serializable()) - lazy val auditEventGenerator: AuditEventGenerator = new AuditEventGenerator(jobConfig) - lazy val mockElasticUtil: ElasticSearchUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - lazy val auditHistoryIndexer: AuditHistoryIndexer = new AuditHistoryIndexer(jobConfig, mockElasticUtil) - lazy val obsrvMetaDataGenerator: ObsrvMetaDataGenerator = new ObsrvMetaDataGenerator(jobConfig) - val loggerMock = mock[Logger](Mockito.withSettings().serializable()) - val mockContext = mock[ProcessFunction[Event, String]#Context] - lazy val transactionEventRouter: TransactionEventRouter = new TransactionEventRouter(jobConfig) - - override protected def beforeAll(): Unit = { - super.beforeAll() - } - - override protected def afterAll(): Unit = { - super.afterAll() - } - - "TransactionEventProcessorService" should "generate audit event" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - } - - "TransactionEventProcessorService" should "throw exception while processing audit event" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_14) - val message: Event = new Event(inputEvent, 0, 10) - try { - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(message)(jobConfig, mockMetrics) - auditEventGenerator.processAuditEvent(message, mockContext, mockMetrics)(jobConfig) - } catch { - case exception: Exception => - verify(loggerMock).error("Failed to process message :: " + anyString() + exception) - assertThrows[Exception](auditEventGenerator.processAuditEvent(message, mockContext, mockMetrics)(jobConfig)) - } - } - - "TransactionEventProcessorService" should "add duration of status change" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]]("duration").asInstanceOf[Int] - duration should be(761) - } - - "TransactionEventProcessorService" should "add Duration as null" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_3) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]].getOrElse("duration", null) - duration should be(null) - } - - "TransactionEventProcessorService" should "generate audit for content creation" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_4) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]].getOrElse("duration", null) - duration should be(null) - } - - "TransactionEventProcessorService" should "skip audit for objectType is null" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_5) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - - eventStr should be("{\"object\": {\"type\":null}}") - } - - "TransactionEventProcessorService" should "skip audit when ObjectType is not available" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_13) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - } - - "TransactionEventProcessorService" should "event for addedRelations" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_6) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata").asInstanceOf[Map[String, AnyRef]]("props").asInstanceOf[List[String]] should contain("name") - eventMap("edata").asInstanceOf[Map[String, AnyRef]]("props").asInstanceOf[List[String]] should contain("collections") - val duration = eventMap("edata").asInstanceOf[Map[String, AnyRef]].getOrElse("duration", null) - duration should be(null) - } - - "TransactionEventProcessorService" should "generate audit for update dialcode" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_7) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("edata").asInstanceOf[Map[String, AnyRef]]("props").asInstanceOf[List[String]] should contain("dialcodes") - val cdata = eventMap("cdata").asInstanceOf[List[Map[String, AnyRef]]] - cdata.head("id").asInstanceOf[List[String]] should contain("K1W6L6") - cdata.head("type") should be("DialCode") - } - - "TransactionEventProcessorService" should "compute duration" in { - val ov = "2019-03-13T13:25:43.129+0530" - val nv = "2019-03-13T13:38:24.358+0530" - val duration = auditEventGenerator.computeDuration(ov, nv) - duration should be(761) - } - - "TransactionEventProcessorService" should "generate es log" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_9), 0, 10) - - val auditHistoryRec: AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent) - - auditHistoryRec.objectId should be(inputEvent.nodeUniqueId) - auditHistoryRec.objectType should be(inputEvent.objectType) - auditHistoryRec.logRecord should be("""{"properties":{"mediaType":{"nv":"content"},"name":{"nv":"Untitled Resource"},"createdOn":{"nv":"2018-02-13T16:01:18.947+0530"},"channel":{"nv":"in.ekstep"},"lastUpdatedOn":{"nv":"2018-02-13T16:01:18.947+0530"},"IL_FUNC_OBJECT_TYPE":{"nv":"Content"},"resourceType":{"nv":"Story"},"compatibilityLevel":{"nv":1.0},"audience":{"nv":["Learner"]},"os":{"nv":["All"]},"IL_SYS_NODE_TYPE":{"nv":"DATA_NODE"},"framework":{"nv":"NCF"},"versionKey":{"nv":"1518517878947"},"mimeType":{"nv":"application/pdf"},"code":{"nv":"test_code"},"contentType":{"nv":"Story"},"language":{"nv":["English"]},"status":{"nv":"Draft"},"keywords":{"nv":["colors","games"]},"idealScreenSize":{"nv":"normal"},"contentEncoding":{"nv":"identity"},"osId":{"nv":"org.ekstep.quiz.app"},"IL_UNIQUE_ID":{"nv":"do_11243969846440755213"},"contentDisposition":{"nv":"inline"},"visibility":{"nv":"Default"},"idealScreenDensity":{"nv":"hdpi"}}}""") - } - - "TransactionEventProcessorService" should "generate with added relations" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_10), 0, 11) - - val auditHistoryRec: AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent) - - auditHistoryRec.objectId should be(inputEvent.nodeUniqueId) - auditHistoryRec.objectType should be(inputEvent.objectType) - auditHistoryRec.logRecord should be("""{"addedRelations":[{"label":"Test unit 11","rel":"hasSequenceMember","dir":"IN","id":"do_1123032073439723521148","type":"Content"}],"removedRelations":[],"properties":{"name":{"nv":"","ov":""}}}""") - } - - "TransactionEventProcessorService" should "generate with removed relations" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_11), 0, 12) - - val auditHistoryRec: AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent) - - auditHistoryRec.objectId should be(inputEvent.nodeUniqueId) - auditHistoryRec.objectType should be(inputEvent.objectType) - auditHistoryRec.logRecord should be("""{"addedTags":[],"addedRelations":[],"properties":{},"removedRelations":[{"label":"qq\n","rel":"associatedTo","dir":"OUT","id":"do_113198273083662336127","relMetadata":{},"type":"AssessmentItem"}],"removedTags":[]}""") - } - - "TransactionEventProcessorService" should "generate obsrv event for valid events" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2) - - val (eventStr, objectType) = auditEventGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - val eventMap = JSONUtil.deserialize[Map[String, AnyRef]](eventStr) - - eventMap("eid") should be("AUDIT") - eventMap("ver") should be("3.0") - eventMap("edata") shouldNot be(null) - } - - "TransactionEventProcessorService" should "skip obsrv event when ObjectType is null" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_5) - - val (eventStr, objectType) = obsrvMetaDataGenerator.getAuditMessage(new Event(inputEvent, 0, 10))(jobConfig, mockMetrics) - - eventStr should be("{\"object\": {\"type\":null}}") - } - - "TransactionEventProcessorService" should "throw exception while processing obsrv event" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_14) - val message: Event = new Event(inputEvent, 0, 10) - try { - val (eventStr, objectType) = obsrvMetaDataGenerator.getAuditMessage(message)(jobConfig, mockMetrics) - obsrvMetaDataGenerator.processAuditEvent(message, mockContext, mockMetrics)(jobConfig) - } catch { - case exception: Exception => - verify(loggerMock).error("Failed to process message :: " + anyString() + exception) - assertThrows[Exception](obsrvMetaDataGenerator.processAuditEvent(message, mockContext, mockMetrics)(jobConfig)) - } - } - - "TransactionEventProcessorService" should "throw exception while processing event for AuditHistoryIndexer" in { - val inputEvent: util.Map[String, Any] = JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_14) - val message: Event = new Event(inputEvent, 0, 10) - try { - val rec = auditHistoryIndexer.getAuditHistory(message) - auditHistoryIndexer.processAuditHistoryEvent(message, mockMetrics)(mockElasticUtil, jobConfig) - } catch { - case exception: Exception => - verify(loggerMock).error("Error while processing message :: " + message.getJson() + "::" + exception) - assertThrows[Exception](auditHistoryIndexer.processAuditHistoryEvent(message, mockMetrics)(mockElasticUtil, jobConfig)) - } - } - - "TransactionEventProcessorService" should "create audit record with provided values" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_9), 0, 10) - - val auditHistoryRec: AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent) - - val auditRec = AuditHistoryRecord( - objectId = auditHistoryRec.objectId, - objectType = auditHistoryRec.objectType, - label = auditHistoryRec.label, - graphId = auditHistoryRec.graphId, - userId = auditHistoryRec.userId, - requestId = auditHistoryRec.requestId, - logRecord = auditHistoryRec.logRecord, - operation = auditHistoryRec.operation, - createdOn = auditHistoryRec.createdOn - ) - - assert(auditHistoryRec == auditRec) - } - - "TransactionEventProcessorService" should "update userId correctly" in { - val inputEvent: Event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_9), 0, 10) - - val auditHistoryRec: AuditHistoryRecord = auditHistoryIndexer.getAuditHistory(inputEvent) - - val auditRec = AuditHistoryRecord( - objectId = auditHistoryRec.objectId, - objectType = auditHistoryRec.objectType, - label = auditHistoryRec.label, - graphId = auditHistoryRec.graphId, - userId = auditHistoryRec.userId, - requestId = auditHistoryRec.requestId, - logRecord = auditHistoryRec.logRecord, - operation = auditHistoryRec.operation, - createdOn = auditHistoryRec.createdOn - ) - - assert(auditRec.userId == auditHistoryRec.userId) - - auditRec.userId = "user456" - - assert(auditRec.userId == "user456") - } - -} diff --git a/video-stream-generator/README.md b/video-stream-generator/README.md deleted file mode 100644 index ba5a320d5..000000000 --- a/video-stream-generator/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Video Stream Generator - -Video Stream Generator job is to generate the stream url for uploaded video (mp4/webm) files - -## Getting Started - -These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a yarn or kubernetes. -### Prerequisites - -1. Download flink-1.13.6-scala_2.12 from [apache-flink-downloads](https://www.apache.org/dyn/closer.lua/flink/flink-1.13.6/flink-1.13.6-bin-scala_2.12.tgz). -2. Download [hadoop dependencies](https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar) (only for running on Yarn). Copy the hadoop dependency jar under lib folder of the flink download. -3. export HADOOP_CLASSPATH=`/hadoop classpath` either in .bashrc or current execution shell. -4. Docker installed. -5. A running yarn cluster or a kubernetes cluster. - -### Build - -mvn clean install - -## Deployment - -### Yarn - -Flink requires memory to be allocated for both job-manager and task manager. -yjm parameter assigns job-manager memory and -ytm assigns task-manager memory. - -``` -./bin/flink run -m yarn-cluster -p 2 -yjm 1024m -ytm 1024m /video-stream-generator/target/video-stream-generator-0.0.1.jar -``` - -### Kubernetes - -``` -# Create a single node cluster -k3d create --server-arg --no-deploy --server-arg traefik --name flink-cluster --image rancher/k3s:v1.0.0 -# Export the single node cluster into KUBECONFIG in the current shell or in ~/.bashrc. -export KUBECONFIG="$(k3d get-kubeconfig --name='flink-cluster')" - -# Only for Mac OSX -# /usr/local/bin/kubectl -> /Applications/Docker.app/Contents/Resources/bin/kubectl -rm /usr/local/bin/kubectl -brew link --overwrite kubernetes-cli - -# Create a configmap using the flink-configuration-configmap.yaml -kubectl create -f knowledge-platform-job/kubernetes/flink-configuration-configmap.yaml - -# Create pods for jobmanager-service, job-manager and task-manager using the yaml files -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-service.yaml -kubectl create -f knowledge-platform-job/kubernetes/jobmanager-deployment.yaml -kubectl create -f knowledge-platform-job/kubernetes/taskmanager-deployment.yaml - -# Create a port-forwarding for accessing the job-manager UI on localhost:8081 -kubectl port-forward deployment/flink-jobmanager 8081:8081 - -# Submit the job to the Kubernetes single node cluster flink-cluster -./bin/flink run -m localhost:8081 /video-stream-generator/target/video-stream-generator-0.0.1.jar - -# Commands to delete the pods created in the cluster -kubectl delete deployment/flink-jobmanager -kubectl delete deployment/flink-taskmanager -kubectl delete service/flink-jobmanager -kubectl delete configmaps/flink-config - -# Command to stop the single-node cluster -k3d stop --name="flink-cluster" -``` diff --git a/video-stream-generator/pom.xml b/video-stream-generator/pom.xml deleted file mode 100644 index 700c4c51f..000000000 --- a/video-stream-generator/pom.xml +++ /dev/null @@ -1,257 +0,0 @@ - - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - video-stream-generator - 1.0.0 - jar - video-stream-generator - - Video stream URL generator Flink Job - - - - UTF-8 - 1.4.0 - - - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - com.fasterxml.jackson.module - jackson-module-scala_${scala.version} - 2.14.2 - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - com.oracle.oci.sdk - oci-java-sdk-common - 3.5.0 - - - com.oracle.oci.sdk - oci-java-sdk-mediaservices - 3.5.0 - - - com.oracle.oci.sdk - oci-java-sdk-common-httpclient-jersey - 3.5.0 - - - com.googlecode.json-simple - json-simple - 1.1 - - - - - src/main/scala - src/test/scala - - - org.codehaus.mojo - build-helper-maven-plugin - 3.2.0 - - - generate-sources - - add-source - - - - src/main/java - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.videostream.task.VideoStreamGeneratorStreamTask - - - - reference.conf - - - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - maven-surefire-plugin - 2.22.2 - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - video-stream-generator-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - - \ No newline at end of file diff --git a/video-stream-generator/src/main/java/org.sunbird.job.videostream.helpers/MediaServiceHelper.java b/video-stream-generator/src/main/java/org.sunbird.job.videostream.helpers/MediaServiceHelper.java deleted file mode 100644 index e2050cd4e..000000000 --- a/video-stream-generator/src/main/java/org.sunbird.job.videostream.helpers/MediaServiceHelper.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.sunbird.job.videostream.helpers; - -import java.util.List; - -import com.oracle.bmc.ConfigFileReader; -import com.oracle.bmc.Region; -import com.oracle.bmc.auth.*; -import com.oracle.bmc.mediaservices.model.*; -import com.oracle.bmc.mediaservices.requests.*; -import com.oracle.bmc.mediaservices.responses.*; -import org.json.simple.JSONObject; - -import com.oracle.bmc.mediaservices.MediaServicesClient; -import org.json.simple.parser.JSONParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class MediaServiceHelper { - - public static Logger logger = LoggerFactory.getLogger(MediaServiceHelper.class); - - public JSONObject createJSONObject(String jsonString) { - JSONObject jsonObject = new JSONObject(); - JSONParser jsonParser = new JSONParser(); - if ((jsonString != null) && !(jsonString.isEmpty())) { - try { - jsonObject = (JSONObject) jsonParser.parse(jsonString); - } catch (org.json.simple.parser.ParseException e) { - e.printStackTrace(); - } - } - return jsonObject; - } - - public MediaWorkflowJob submitJob(String compartment_id, String mediaWorkFlowId, - JSONObject mediaFlowJobParameters) { - MediaServicesClient mediaClient = connectMediaService(); - logger.debug("mediaFlowJobParameters...." + mediaFlowJobParameters + " mediaWorkFlowId..." + mediaWorkFlowId - + " compartment_id..." + compartment_id); - CreateMediaWorkflowJobRequest request = CreateMediaWorkflowJobRequest.builder() - .createMediaWorkflowJobDetails(CreateMediaWorkflowJobByIdDetails.builder() - .displayName("media-flow-job-diksha").compartmentId(compartment_id) - .mediaWorkflowId(mediaWorkFlowId).parameters(mediaFlowJobParameters).build()) - .build(); - CreateMediaWorkflowJobResponse response = mediaClient.createMediaWorkflowJob(request); - logger.debug("JOB_ID::{}", response.getMediaWorkflowJob().getId()); - closeMediaClient(mediaClient); - return response.getMediaWorkflowJob(); - } - - public String getStreamingPaths(String mediaWorkflowId, String gatewayDomain) { - String streamingURL = null; - MediaWorkflowJob job = getWorkflowJob(mediaWorkflowId); - List outputList = job.getOutputs(); - String mediaAssetId = null; - logger.debug("getStreamingPaths() gatewayDomain::{}", mediaWorkflowId, gatewayDomain); - for (JobOutput output : outputList) { - if (output.getObjectName().contains("master.m3u8")) { - mediaAssetId = output.getId(); - break; - } - } - streamingURL = "https://" + gatewayDomain + "/" + mediaAssetId + "/master.m3u8"; - logger.debug("getStreamingPaths() streamingURL::{}", streamingURL); - return streamingURL; - } - - // Get Media Workflow Job - public MediaWorkflowJob getWorkflowJob(String mediaWorkflowJobId) { - logger.debug(" <<<< Entering getWorkflowJob() >>>> mediaWorkflowId::" + mediaWorkflowJobId); - GetMediaWorkflowJobRequest request = GetMediaWorkflowJobRequest.builder().mediaWorkflowJobId(mediaWorkflowJobId) - .build(); - MediaServicesClient mediaClient = connectMediaService(); - GetMediaWorkflowJobResponse response = mediaClient.getMediaWorkflowJob(request); - MediaWorkflowJob mediaWorkflowJob = response.getMediaWorkflowJob(); - logger.debug("mediaWorkflowJob::::::::::::::" + mediaWorkflowJob); - closeMediaClient(mediaClient); - logger.debug(" <<<< Exiting getWorkflowJob() >>>> mediaWorkflowId::" + mediaWorkflowJobId); - return mediaWorkflowJob; - } - - public MediaServicesClient connectMediaService() { - logger.debug("<<<< Entering connectMediaService() >>>>"); - MediaServicesClient mediaClient = null; - try { -// final ConfigFileReader.ConfigFile configFile = ConfigFileReader.parse("~/.oci/config", "SUNBIRD"); -// AuthenticationDetailsProvider authenticationDetailsProvider = new ConfigFileAuthenticationDetailsProvider(configFile); - - InstancePrincipalsAuthenticationDetailsProvider authenticationDetailsProvider = InstancePrincipalsAuthenticationDetailsProvider - .builder().build(); - - mediaClient = MediaServicesClient.builder().build(authenticationDetailsProvider); - logger.debug("mediaClient::::::::::::::" + mediaClient); - } catch (Exception myThrowableObject) { - myThrowableObject.printStackTrace(); - } - logger.debug("<<<< Exiting connectMediaService() >>>>"); - return mediaClient; - } - - // Close Media Service Client - private void closeMediaClient(MediaServicesClient mc) { - mc.close(); - } -} - diff --git a/video-stream-generator/src/main/resources/log4j.properties b/video-stream-generator/src/main/resources/log4j.properties deleted file mode 100644 index 1e78a6488..000000000 --- a/video-stream-generator/src/main/resources/log4j.properties +++ /dev/null @@ -1,11 +0,0 @@ -# log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.file=video-stream-generator.log -log4j.appender.file.append=true -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.MaxFileSize=256KB -log4j.appender.file.MaxBackupIndex=4 -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n - -# Suppress the irrelevant (wrong) warnings from the Netty channel handler -log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/video-stream-generator/src/main/resources/video-stream-generator.conf b/video-stream-generator/src/main/resources/video-stream-generator.conf deleted file mode 100644 index ae95f603c..000000000 --- a/video-stream-generator/src/main/resources/video-stream-generator.conf +++ /dev/null @@ -1,112 +0,0 @@ -include "base-config.conf" - -kafka { - input.topic = "sunbirddev.content.postpublish.request" - groupId = "sunbirddev-video-stream-generator-group" -} - -task { - consumer.parallelism = 1 - parallelism = 1 - timer.duration = 60 - max.retries = 10 -} - -lms-cassandra { - keyspace = "sunbirddev_platform_db" - table = "job_request" -} - -service { - content { - basePath = "http://11.2.6.6/content" - } -} - -# Azure Media Service Config -azure { - location = "centralindia" - tenant = "tenant" - subscription_id = "subscription id " - - login { - endpoint="https://login.microsoftonline.com" - } - - api { - endpoint="https://management.azure.com" - version = "2018-07-01" - } - - account_name = "account name" - resource_group_name = "group name" - - transform { - default = "media_transform_default" - hls = "media_transform_hls" - } - - stream { - base_url = "https://sunbirdspikemedia-inct.streaming.media.azure.net" - endpoint_name = "default" - protocol = "Hls" - policy_name = "Predefined_ClearStreamingOnly" - } - - token { - client_key = "client key" - client_secret = "client secret" - } -} - -azure_tenant="tenant" -azure_subscription_id="subscription id" -azure_account_name="account name" -azure_resource_group_name="group name" -azure_token_client_key="client key" -azure_token_client_secret="client secret" - -# CSP Name. e.g: aws or azure -media_service_type="oci" - -#AWS Elemental Media Convert Config -aws { - region="ap-south-1" - content_bucket_name="awsmedia-spike" - token { - access_key="access key" - access_secret="access secret" - } - api { - endpoint="API Endpoint for media convert" - version="2017-08-29" - } - service { - name="mediaconvert" - queue="Media Convert Queue Id" - role="Media Convert Role Name" - } - stream { - protocol="Hls" - } -} - -# OCI Elemental Media Convert Config -# values for unit test (if applicable) -oci { - region="your-oci-home-region" - compartment_id="your-oci-compartment-ocid" - namespace="your-oci-tenant-namespace" - bucket { - content_bucket_name="src_bucket" - processed_bucket_name="dest_bucket" - } - stream { - prefix_input="sunbird" - work_flow_id ="your-media-workflow-ocid" - distribution_channel_id="your-distribution-channel-ocid" - stream_package_config_id="your-media-package-config-ocid" - } -} - -media_service_job_success_status=["FINISHED", "COMPLETE", "SUCCEEDED"] diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/domain/Event.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/domain/Event.scala deleted file mode 100644 index 048b2f6c1..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/domain/Event.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.sunbird.job.videostream.domain - -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.domain.reader.JobRequest - -import java.util - -class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { - - private val jobName = "VideoStreamGenerator" - - def action: String = readOrDefault[String]("edata.action", "") - - def mimeType: String = readOrDefault[String]("edata.mimeType", "") - - def channel: String = readOrDefault[String]("context.channel", "") - - def eid: String = readOrDefault[String]("eid", "") - - def artifactUrl: String = readOrDefault[String]("edata.artifactUrl", "") - - def identifier: String = readOrDefault[String]("edata.identifier", "") - - def eData: Map[String, AnyRef] = readOrDefault("edata", new util.HashMap[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] - - def isValid: Boolean = { - StringUtils.isNotBlank(artifactUrl) && - StringUtils.isNotBlank(mimeType) && - ( - StringUtils.equalsIgnoreCase(mimeType, "video/mp4") || - StringUtils.equalsIgnoreCase(mimeType, "video/webm") - ) && - StringUtils.isNotBlank(identifier) && - StringUtils.isNotBlank(channel) - } - -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/exception/MediaServiceException.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/exception/MediaServiceException.scala deleted file mode 100644 index 3d2287288..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/exception/MediaServiceException.scala +++ /dev/null @@ -1,3 +0,0 @@ -package org.sunbird.job.videostream.exception - -class MediaServiceException(var errorCode: String = null, msg: String, ex: Exception = null) extends Exception(msg, ex) diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/functions/VideoStreamGenerator.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/functions/VideoStreamGenerator.scala deleted file mode 100644 index cba819a58..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/functions/VideoStreamGenerator.scala +++ /dev/null @@ -1,86 +0,0 @@ -package org.sunbird.job.videostream.functions - -import java.util -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration -import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.slf4j.LoggerFactory -import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.videostream.domain.Event -import org.sunbird.job.videostream.service.VideoStreamService -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} - -class VideoStreamGenerator(config: VideoStreamGeneratorConfig, httpUtil:HttpUtil) - (implicit mapTypeInfo: TypeInformation[util.Map[String, AnyRef]], - stringTypeInfo: TypeInformation[String]) - extends BaseProcessKeyedFunction[String, Event, Event](config) { - - implicit lazy val videoStreamConfig: VideoStreamGeneratorConfig = config - private[this] lazy val logger = LoggerFactory.getLogger(classOf[VideoStreamGenerator]) - private var videoStreamService:VideoStreamService = _ - lazy val timerDurationInMS: Long = config.timerDuration * 1000 - private var nextTimerTimestamp = 0l - - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.skippedEventCount, config.successEventCount, config.failedEventCount, config.retryEventCount) - } - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - videoStreamService = new VideoStreamService()(config, httpUtil); - } - - override def close(): Unit = { - videoStreamService.closeConnection() - super.close() - } - - @throws(classOf[InvalidEventException]) - override def processElement(event: Event, - context: KeyedProcessFunction[String, Event, Event]#Context, - metrics: Metrics): Unit = { - try { - metrics.incCounter(config.totalEventsCount) - if (event.isValid) { - val eData = event.eData ++ Map("channel"-> event.channel) - videoStreamService.submitJobRequest(eData) - logger.info("Streaming job submitted for " + event.identifier + " with url: " + event.artifactUrl) - registerTimer(context) - } else metrics.incCounter(config.skippedEventCount) - } catch { - case ex: Exception => - metrics.incCounter(config.failedEventCount) - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } - } - - override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, Event, Event]#OnTimerContext, metrics: Metrics): Unit = { - unregisterTimer() - val processing = videoStreamService.readFromDB(Map("status" -> "PROCESSING")) ++ videoStreamService.readFromDB(Map("status" -> "FAILED", "iteration" -> Map("type" -> "lte", "value" -> 10))) - if (processing.nonEmpty) { - logger.info("Requests in queue to validate update the status: " + processing.size) - videoStreamService.processJobRequest(metrics) - registerTimer(ctx) - } else { - logger.info("There are no video streaming requests in queue to validate update the status.") - } - } - - private def registerTimer(context: KeyedProcessFunction[String, Event, Event]#Context): Unit = { - if (nextTimerTimestamp == 0l) { - val nextTrigger = context.timestamp() + timerDurationInMS - context.timerService().registerProcessingTimeTimer(nextTrigger) - nextTimerTimestamp = nextTrigger - logger.info("Timer registered to execute at " + nextTimerTimestamp) - } else { - logger.info("Timer already exists at: " + nextTimerTimestamp) - } - } - - private def unregisterTimer(): Unit = { - nextTimerTimestamp = 0l - } - -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsRequestBody.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsRequestBody.scala deleted file mode 100644 index 823142c06..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsRequestBody.scala +++ /dev/null @@ -1,6 +0,0 @@ -package org.sunbird.job.videostream.helpers - -object AwsRequestBody { - - val submit_hls_job = "{\"Queue\": \"queueId\",\"role\":\"mediaRole\",\"Settings\": {\"OutputGroups\": [{\"CustomName\": \"hls\",\"Name\": \"Apple HLS\",\"Outputs\": [{\"ContainerSettings\": {\"Container\": \"M3U8\",\"M3u8Settings\": {\"AudioFramesPerPes\": 4,\"PcrControl\": \"PCR_EVERY_PES_PACKET\",\"PmtPid\": 480,\"PrivateMetadataPid\": 503,\"ProgramNumber\": 1,\"PatInterval\": 0,\"PmtInterval\": 0,\"Scte35Source\": \"NONE\",\"NielsenId3\": \"NONE\",\"TimedMetadata\": \"NONE\",\"VideoPid\": 481,\"AudioPids\": [482,483,484,485,486,487,488,489,490,491,492]}},\"VideoDescription\": {\"Width\": 256,\"ScalingBehavior\": \"DEFAULT\",\"Height\": 144,\"TimecodeInsertion\": \"DISABLED\",\"AntiAlias\": \"ENABLED\",\"Sharpness\": 50,\"CodecSettings\": {\"Codec\": \"H_264\",\"H264Settings\": {\"InterlaceMode\": \"PROGRESSIVE\",\"NumberReferenceFrames\": 3,\"Syntax\": \"DEFAULT\",\"Softness\": 0,\"GopClosedCadence\": 1,\"GopSize\": 90,\"Slices\": 1,\"GopBReference\": \"DISABLED\",\"SlowPal\": \"DISABLED\",\"SpatialAdaptiveQuantization\": \"ENABLED\",\"TemporalAdaptiveQuantization\": \"ENABLED\",\"FlickerAdaptiveQuantization\": \"DISABLED\",\"EntropyEncoding\": \"CABAC\",\"Bitrate\": 109000,\"FramerateControl\": \"INITIALIZE_FROM_SOURCE\",\"RateControlMode\": \"CBR\",\"CodecProfile\": \"MAIN\",\"Telecine\": \"NONE\",\"MinIInterval\": 0,\"AdaptiveQuantization\": \"HIGH\",\"CodecLevel\": \"AUTO\",\"FieldEncoding\": \"PAFF\",\"SceneChangeDetect\": \"ENABLED\",\"QualityTuningLevel\": \"SINGLE_PASS\",\"FramerateConversionAlgorithm\": \"DUPLICATE_DROP\",\"UnregisteredSeiTimecode\": \"DISABLED\",\"GopSizeUnits\": \"FRAMES\",\"ParControl\": \"INITIALIZE_FROM_SOURCE\",\"NumberBFramesBetweenReferenceFrames\": 2,\"RepeatPps\": \"DISABLED\",\"DynamicSubGop\": \"STATIC\"}},\"AfdSignaling\": \"NONE\",\"DropFrameTimecode\": \"ENABLED\",\"RespondToAfd\": \"NONE\",\"ColorMetadata\": \"INSERT\"},\"AudioDescriptions\": [{\"AudioTypeControl\": \"FOLLOW_INPUT\",\"CodecSettings\": {\"Codec\": \"AAC\",\"AacSettings\": {\"AudioDescriptionBroadcasterMix\": \"NORMAL\",\"Bitrate\": 96000,\"RateControlMode\": \"CBR\",\"CodecProfile\": \"LC\",\"CodingMode\": \"CODING_MODE_2_0\",\"RawFormat\": \"NONE\",\"SampleRate\": 48000,\"Specification\": \"MPEG4\"}},\"LanguageCodeControl\": \"FOLLOW_INPUT\"}],\"OutputSettings\": {\"HlsSettings\": {\"AudioGroupId\": \"program_audio\",\"SegmentModifier\": \"$dt$\",\"IFrameOnlyManifest\": \"EXCLUDE\"}},\"NameModifier\": \"_144\"},{\"ContainerSettings\": {\"Container\": \"M3U8\",\"M3u8Settings\": {\"AudioFramesPerPes\": 4,\"PcrControl\": \"PCR_EVERY_PES_PACKET\",\"PmtPid\": 480,\"PrivateMetadataPid\": 503,\"ProgramNumber\": 1,\"PatInterval\": 0,\"PmtInterval\": 0,\"Scte35Source\": \"NONE\",\"Scte35Pid\": 500,\"NielsenId3\": \"NONE\",\"TimedMetadata\": \"NONE\",\"TimedMetadataPid\": 502,\"VideoPid\": 481,\"AudioPids\": [482,483,484,485,486,487,488,489,490,491,492]}},\"VideoDescription\": {\"Width\": 640,\"ScalingBehavior\": \"DEFAULT\",\"Height\": 360,\"TimecodeInsertion\": \"DISABLED\",\"AntiAlias\": \"ENABLED\",\"Sharpness\": 50,\"CodecSettings\": {\"Codec\": \"H_264\",\"H264Settings\": {\"InterlaceMode\": \"PROGRESSIVE\",\"NumberReferenceFrames\": 3,\"Syntax\": \"DEFAULT\",\"Softness\": 0,\"GopClosedCadence\": 1,\"GopSize\": 90,\"Slices\": 1,\"GopBReference\": \"DISABLED\",\"SlowPal\": \"DISABLED\",\"SpatialAdaptiveQuantization\": \"ENABLED\",\"TemporalAdaptiveQuantization\": \"ENABLED\",\"FlickerAdaptiveQuantization\": \"DISABLED\",\"EntropyEncoding\": \"CABAC\",\"Bitrate\": 525000,\"FramerateControl\": \"INITIALIZE_FROM_SOURCE\",\"RateControlMode\": \"CBR\",\"CodecProfile\": \"MAIN\",\"Telecine\": \"NONE\",\"MinIInterval\": 0,\"AdaptiveQuantization\": \"HIGH\",\"CodecLevel\": \"AUTO\",\"FieldEncoding\": \"PAFF\",\"SceneChangeDetect\": \"ENABLED\",\"QualityTuningLevel\": \"SINGLE_PASS\",\"FramerateConversionAlgorithm\": \"DUPLICATE_DROP\",\"UnregisteredSeiTimecode\": \"DISABLED\",\"GopSizeUnits\": \"FRAMES\",\"ParControl\": \"INITIALIZE_FROM_SOURCE\",\"NumberBFramesBetweenReferenceFrames\": 2,\"RepeatPps\": \"DISABLED\",\"DynamicSubGop\": \"STATIC\"}},\"AfdSignaling\": \"NONE\",\"DropFrameTimecode\": \"ENABLED\",\"RespondToAfd\": \"NONE\",\"ColorMetadata\": \"INSERT\"},\"AudioDescriptions\": [{\"AudioTypeControl\": \"FOLLOW_INPUT\",\"CodecSettings\": {\"Codec\": \"AAC\",\"AacSettings\": {\"AudioDescriptionBroadcasterMix\": \"NORMAL\",\"Bitrate\": 96000,\"RateControlMode\": \"CBR\",\"CodecProfile\": \"LC\",\"CodingMode\": \"CODING_MODE_2_0\",\"RawFormat\": \"NONE\",\"SampleRate\": 48000,\"Specification\": \"MPEG4\"}},\"LanguageCodeControl\": \"FOLLOW_INPUT\"}],\"OutputSettings\": {\"HlsSettings\": {\"AudioGroupId\": \"program_audio\",\"SegmentModifier\": \"$dt$\",\"IFrameOnlyManifest\": \"EXCLUDE\"}},\"NameModifier\": \"_360\"},{\"ContainerSettings\": {\"Container\": \"M3U8\",\"M3u8Settings\": {\"AudioFramesPerPes\": 4,\"PcrControl\": \"PCR_EVERY_PES_PACKET\",\"PmtPid\": 480,\"PrivateMetadataPid\": 503,\"ProgramNumber\": 1,\"PatInterval\": 0,\"PmtInterval\": 0,\"Scte35Source\": \"NONE\",\"Scte35Pid\": 500,\"NielsenId3\": \"NONE\",\"TimedMetadata\": \"NONE\",\"TimedMetadataPid\": 502,\"VideoPid\": 481,\"AudioPids\": [482,483,484,485,486,487,488,489,490,491,492]}},\"VideoDescription\": {\"Width\": 1280,\"ScalingBehavior\": \"DEFAULT\",\"Height\": 720,\"TimecodeInsertion\": \"DISABLED\",\"AntiAlias\": \"ENABLED\",\"Sharpness\": 50,\"CodecSettings\": {\"Codec\": \"H_264\",\"H264Settings\": {\"InterlaceMode\": \"PROGRESSIVE\",\"NumberReferenceFrames\": 3,\"Syntax\": \"DEFAULT\",\"Softness\": 0,\"GopClosedCadence\": 1,\"GopSize\": 90,\"Slices\": 1,\"GopBReference\": \"DISABLED\",\"SlowPal\": \"DISABLED\",\"SpatialAdaptiveQuantization\": \"ENABLED\",\"TemporalAdaptiveQuantization\": \"ENABLED\",\"FlickerAdaptiveQuantization\": \"DISABLED\",\"EntropyEncoding\": \"CABAC\",\"Bitrate\": 1378000,\"FramerateControl\": \"INITIALIZE_FROM_SOURCE\",\"RateControlMode\": \"CBR\",\"CodecProfile\": \"MAIN\",\"Telecine\": \"NONE\",\"MinIInterval\": 0,\"AdaptiveQuantization\": \"HIGH\",\"CodecLevel\": \"AUTO\",\"FieldEncoding\": \"PAFF\",\"SceneChangeDetect\": \"ENABLED\",\"QualityTuningLevel\": \"SINGLE_PASS\",\"FramerateConversionAlgorithm\": \"DUPLICATE_DROP\",\"UnregisteredSeiTimecode\": \"DISABLED\",\"GopSizeUnits\": \"FRAMES\",\"ParControl\": \"INITIALIZE_FROM_SOURCE\",\"NumberBFramesBetweenReferenceFrames\": 2,\"RepeatPps\": \"DISABLED\",\"DynamicSubGop\": \"STATIC\"}},\"AfdSignaling\": \"NONE\",\"DropFrameTimecode\": \"ENABLED\",\"RespondToAfd\": \"NONE\",\"ColorMetadata\": \"INSERT\"},\"AudioDescriptions\": [{\"AudioTypeControl\": \"FOLLOW_INPUT\",\"CodecSettings\": {\"Codec\": \"AAC\",\"AacSettings\": {\"AudioDescriptionBroadcasterMix\": \"NORMAL\",\"Bitrate\": 96000,\"RateControlMode\": \"CBR\",\"CodecProfile\": \"LC\",\"CodingMode\": \"CODING_MODE_2_0\",\"RawFormat\": \"NONE\",\"SampleRate\": 48000,\"Specification\": \"MPEG4\"}},\"LanguageCodeControl\": \"FOLLOW_INPUT\"}],\"OutputSettings\": {\"HlsSettings\": {\"AudioGroupId\": \"program_audio\",\"IFrameOnlyManifest\": \"EXCLUDE\"}},\"NameModifier\": \"_720\"}],\"OutputGroupSettings\": {\"Type\": \"HLS_GROUP_SETTINGS\",\"HlsGroupSettings\": {\"ManifestDurationFormat\": \"INTEGER\",\"SegmentLength\": 10,\"TimedMetadataId3Period\": 10,\"CaptionLanguageSetting\": \"OMIT\",\"TimedMetadataId3Frame\": \"PRIV\",\"CodecSpecification\": \"RFC_4281\",\"OutputSelection\": \"MANIFESTS_AND_SEGMENTS\",\"ProgramDateTimePeriod\": 600,\"MinSegmentLength\": 0,\"MinFinalSegmentLength\": 0,\"DirectoryStructure\": \"SINGLE_DIRECTORY\",\"ProgramDateTime\": \"EXCLUDE\",\"SegmentControl\": \"SEGMENTED_FILES\",\"ManifestCompression\": \"NONE\",\"ClientCache\": \"ENABLED\",\"StreamInfResolution\": \"INCLUDE\",\"Destination\": \"outputLocation\"}}}],\"AdAvailOffset\": 0,\"Inputs\": [{\"AudioSelectors\": {\"Audio Selector 1\": {\"Offset\": 0,\"DefaultSelection\": \"DEFAULT\",\"ProgramSelection\": 1}},\"VideoSelector\": {\"ColorSpace\": \"FOLLOW\"},\"FilterEnable\": \"AUTO\",\"PsiControl\": \"USE_PSI\",\"FilterStrength\": 0,\"DeblockFilter\": \"DISABLED\",\"DenoiseFilter\": \"DISABLED\",\"TimecodeSource\": \"EMBEDDED\",\"FileInput\": \"inputVideoFile\"}]}}" -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsResult.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsResult.scala deleted file mode 100644 index f78b6191f..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsResult.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.sunbird.job.videostream.helpers - -import java.text.SimpleDateFormat -import java.util.{Date, TimeZone} - -import org.apache.commons.lang3.StringUtils - -import scala.collection.immutable.HashMap - -object AwsResult extends Result { - - val formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") - formatter.setTimeZone(TimeZone.getTimeZone("UTC")) - - override def getSubmitJobResult(response: MediaResponse): Map[String, AnyRef] = { - val job: Map[String, AnyRef] = response.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]] - val timing: Map[String, AnyRef] = job.getOrElse("timing", Map).asInstanceOf[Map[String, AnyRef]] - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> job.getOrElse("id", "").toString, - "status" -> job.getOrElse("status", "").toString.toUpperCase(), - "submittedOn" -> formatter.format(new Date(timing.getOrElse("submitTime", "").toString.toLong*1000)) - ) - ) - } - - override def getJobResult(response: MediaResponse): Map[String, AnyRef] = { - val job: Map[String, AnyRef] = response.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]] - val timing: Map[String, AnyRef] = job.getOrElse("timing", Map).asInstanceOf[Map[String, AnyRef]] - - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> job.getOrElse("id", "").toString, - "status" -> job.getOrElse("status", "").toString.toUpperCase(), - "submittedOn" -> formatter.format(new Date(timing.getOrElse("submitTime", "").toString.toLong*1000)), - "lastModifiedOn" -> { if(StringUtils.isNotBlank(timing.getOrElse("finishTime", "").toString)){formatter.format(new Date(timing.getOrElse("finishTime", "").toString.toLong*1000))} else {""}}, - "error" -> { - if (StringUtils.equalsIgnoreCase(job.getOrElse("status", "").toString.toUpperCase(), "ERROR")) { - HashMap[String, String]( - "errorCode" -> job.getOrElse("errorCode", "").toString, - "errorMessage" -> job.getOrElse("errorMessage", "").toString - ) - }else{ - null - } - } - ) - ) - } - - override def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] = ??? - - override def getListJobResult(response: MediaResponse): Map[String, AnyRef] = ??? -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsSignUtils.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsSignUtils.scala deleted file mode 100644 index 5766b953b..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AwsSignUtils.scala +++ /dev/null @@ -1,133 +0,0 @@ -package org.sunbird.job.videostream.helpers - -import java.text.SimpleDateFormat -import java.util.{Date, TimeZone} - -import javax.crypto.Mac -import javax.crypto.spec.SecretKeySpec -import javax.xml.bind.DatatypeConverter -import org.apache.commons.codec.binary.Hex -import org.apache.commons.codec.digest.DigestUtils -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig - -object AwsSignUtils { - - val dateFormat = new SimpleDateFormat("yyyyMMdd") - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")) - - def getSiginingkey()(implicit config: VideoStreamGeneratorConfig): Array[Byte] = { - val date = dateFormat.format(new Date()).getBytes("UTF8") - val kSecret = ("AWS4" + config.getConfig("aws.token.access_secret")).getBytes("UTF8") - val kDate = HmacSHA256(date, kSecret) - val kRegion = HmacSHA256(config.getConfig("aws.region").getBytes("UTF8"), kDate) - val kService = HmacSHA256(config.getConfig("aws.service.name").getBytes("UTF8"), kRegion) - val kSigning = HmacSHA256("aws4_request".getBytes("UTF8"), kService) - kSigning - } - - def getStringToSign(httpMethod: String, url: String, headers: Map[String, String], payload: String)(implicit config: VideoStreamGeneratorConfig): String = { - val canonicalUri = getCanonicalUri(url) - val canonicalQueryString = getCanonicalQueryString(url) - val hashedPayload = getHashedPayload(payload) - val canonicalHeaders = getCanonicalHeaders(headers, hashedPayload) - val signedHeaders = getSignedHeaders(headers.keySet) - - val canonicalRequest = httpMethod + "\n" + canonicalUri + "\n" + canonicalQueryString + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + hashedPayload - - val timeStampISO8601Format = headers.get("x-amz-date").get - val scope = dateFormat.format(new Date()) + "/" + config.getConfig("aws.region") + "/" + config.getConfig("aws.service.name") + "/aws4_request" - - val stringToSign = "AWS4-HMAC-SHA256" + "\n" + timeStampISO8601Format + "\n" + scope + "\n" + sha256Hash(canonicalRequest) - - stringToSign - } - - def generateToken(httpMethod: String, url: String, headers: Map[String, String], payload: String)(implicit config: VideoStreamGeneratorConfig): String = { - val signature = new String(Hex.encodeHex(HmacSHA256(getStringToSign(httpMethod, url, headers, payload).getBytes("UTF-8"), getSiginingkey()))) - - "AWS4-HMAC-SHA256 Credential=" + config.getConfig("aws.token.access_key") + "/" + dateFormat.format(new Date()) + "/" + config.getConfig("aws.region") + "/" + config.getConfig("aws.service.name") + "/aws4_request,SignedHeaders=" + getSignedHeaders(headers.keySet) + ",Signature=" + signature - } - - - @throws[Exception] - def HmacSHA256(data: Array[Byte], key: Array[Byte]) = { - val algorithm: String = "HMacSha256" - val mac: Mac = Mac.getInstance(algorithm) - mac.init(new SecretKeySpec(key, algorithm)) - mac.doFinal(data) - } - - def uriEncode(input: Array[Char], encodeSlash: Boolean): String = { - val result: StringBuilder = new StringBuilder() - var i: Int = 0 - for (i <- 0 to input.length()) { - val ch = input.charAt(i) - if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' || ch == '~' || ch == '.') { - result.append(ch) - } else if (ch == '/') { - if (encodeSlash) result.append("%2F") else result.append(ch) - } else { - result.append(Integer.toHexString((ch.asInstanceOf[Integer]))) - } - } - result.toString() - } - - def sha256Hash(input: String): String = { - DigestUtils.sha256Hex(input) - } - - def getCanonicalUri(url: String)(implicit config: VideoStreamGeneratorConfig): String = { - val version = config.getConfig("aws.api.version") - var uri = url.split(version)(1) - if (StringUtils.isBlank(uri)) "" else { - uri = "/" + version + uri; uri - } - } - - def getCanonicalQueryString(url: String): String = { - var result: String = "" - if (url.split("\\?").length > 1) { - val queryString: String = url.split("\\?")(1) - if (StringUtils.isNotBlank(queryString)) { - for (param <- queryString.split("\\&")) { - if (param.split("=").length == 2) - result += uriEncode(param.split("=")(0).toCharArray, false) + "=" + uriEncode(param.split("=")(1).toCharArray, false) + "&" - else - result += uriEncode(param.split("=")(0).toCharArray, false) + "&" - } - result = result.substring(0, result.length - 2) - } - result - } else result - } - - def getCanonicalHeaders(headers: Map[String, String], hashedPayload: String): String = { - var result: String = "" - headers.foreach(header => { - result += header._1.toLowerCase + ":" + header._2.trim + "\n" - }) - result - } - - def getSignedHeaders(keySet: scala.collection.Set[String]): String = { - var result: String = "" - keySet.foreach(key => { - result += key.toLowerCase + ";" - }) - result.substring(0, result.length - 1) - } - - def getHashedPayload(payload: String): String = { - if (StringUtils.isNotBlank(payload)) { - sha256Hash(payload) - } else { - sha256Hash("") - } - } - - def stringtoHex(str: String): String = { - DatatypeConverter.printHexBinary(str.getBytes("UTF8")) - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AzureRequestBody.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AzureRequestBody.scala deleted file mode 100644 index d9158c867..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AzureRequestBody.scala +++ /dev/null @@ -1,8 +0,0 @@ -package org.sunbird.job.videostream.helpers - -object AzureRequestBody { - - val create_asset = " {\"properties\": {\"description\": \"assetDescription\",\"alternateId\" : \"assetId\"}}" - val submit_job = "{\"properties\": {\"input\": {\"@odata.type\": \"#Microsoft.Media.JobInputHttp\",\"baseUri\": \"baseInputUrl\",\"files\": [\"inputVideoFile\"]},\"outputs\": [{\"@odata.type\": \"#Microsoft.Media.JobOutputAsset\",\"assetName\": \"assetId\"}]}}" - val create_stream_locator="{\"properties\":{\"assetName\": \"assetId\",\"streamingPolicyName\": \"policyName\"}}" -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AzureResult.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AzureResult.scala deleted file mode 100644 index 77d05c7db..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/AzureResult.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.videostream.helpers - -import org.apache.commons.lang3.StringUtils - -import scala.collection.immutable.HashMap - -object AzureResult extends Result { - - override def getSubmitJobResult(response: MediaResponse): Map[String, AnyRef] = { - val result = response.result - val output: Map[String, AnyRef] = result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("outputs", List).asInstanceOf[List[Map[String, AnyRef]]].head - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> result.getOrElse("name", "").toString, - "status" -> output.getOrElse("state", "").toString.toUpperCase(), - "submittedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("created", "").toString, - "lastModifiedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("lastModified", "").toString - ) - ) - - } - - override def getJobResult(response: MediaResponse): Map[String, AnyRef] = { - val result = response.result - val output: Map[String, AnyRef] = result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("outputs", List).asInstanceOf[List[Map[String, AnyRef]]].head - - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> result.getOrElse("name", "").toString, - "status" -> output.getOrElse("state", "").toString.toUpperCase(), - "submittedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("created", "").toString, - "lastModifiedOn" -> result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("lastModified", "").toString, - "error" -> { - if (StringUtils.equalsIgnoreCase(output.getOrElse("state", "").toString.toUpperCase(),"ERROR")) { - val errorMap: Map[String, AnyRef] = output.getOrElse("error", Map).asInstanceOf[Map[String, AnyRef]] - Map[String, String]( - "errorCode" -> errorMap.getOrElse("code", "").toString, - "errorMessage" -> errorMap.getOrElse("details", List).asInstanceOf[List[Map[String, AnyRef]]].head.getOrElse("message", "").toString - ) - } else { - null - } - } - ) - ) - } - - override def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - - override def getListJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/CaseClasses.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/CaseClasses.scala deleted file mode 100644 index ad1f3406a..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/CaseClasses.scala +++ /dev/null @@ -1,24 +0,0 @@ -package org.sunbird.job.videostream.helpers - -import org.joda.time.DateTime -import scala.collection.immutable.HashMap - - -case class MediaRequest(id: String, params: Map[String, AnyRef] = new HashMap[String, AnyRef], request: Map[String, AnyRef] = new HashMap[String, AnyRef]); - -case class MediaResponse(id: String, ts: String, params: Map[String, Any] = new HashMap[String, AnyRef], responseCode: String, result: Map[String, AnyRef] = new HashMap[String, AnyRef]) - -object ResponseCode extends Enumeration { - type Code = Value - val OK = Value(200) - val CLIENT_ERROR = Value(400) - val SERVER_ERROR = Value(500) - val RESOURCE_NOT_FOUND = Value(404) -} - -trait AlgoOutput extends AnyRef - -case class JobRequest(client_key: String, request_id: String, job_id: Option[String], status: String, request_data: String, iteration: Int, dt_job_submitted: Option[DateTime] = None, location: Option[String] = None, dt_file_created: Option[DateTime] = None, dt_first_event: Option[DateTime] = None, dt_last_event: Option[DateTime] = None, dt_expiration: Option[DateTime] = None, dt_job_processing: Option[DateTime] = None, dt_job_completed: Option[DateTime] = None, input_events: Option[Int] = None, output_events: Option[Int] = None, file_size: Option[Long] = None, latency: Option[Int] = None, execution_time: Option[Long] = None, err_message: Option[String] = None, stage: Option[String] = None, stage_status: Option[String] = None, job_name: Option[String] = None) extends AlgoOutput - -case class JobStage(request_id: String, client_key: String, stage: String, stage_status: String, status: String, err_message: String = "", dt_job_processing: Option[DateTime] = Option(new DateTime())) -case class StreamingStage(request_id: String, client_key: String, job_id: String, stage: String, stage_status: String, status: String, iteration: Int, err_message: String = "") \ No newline at end of file diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/JsonUtilility.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/JsonUtilility.scala deleted file mode 100644 index c6a9fde22..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/JsonUtilility.scala +++ /dev/null @@ -1,25 +0,0 @@ -package org.sunbird.job.videostream.helpers - -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule - -object JsonUtility { - val mapper = new ObjectMapper() with ScalaObjectMapper - mapper.registerModule(DefaultScalaModule) - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - - def toJson(value: Map[Symbol, Any]): String = { - toJson(value map { case (k,v) => k.name -> v}) - } - - def toJson(value: Any): String = { - mapper.writeValueAsString(value) - } - - def toMap[V](json:String)(implicit m: Manifest[V]) = fromJson[Map[String,V]](json) - - def fromJson[T](json: String)(implicit m : Manifest[T]): T = { - mapper.readValue[T](json) - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/OCIRequestBody.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/OCIRequestBody.scala deleted file mode 100644 index 6b2564ce5..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/OCIRequestBody.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.sunbird.job.videostream.helpers - -object OCIRequestBody { - val get_object_parameters = "{\"taskParameters\": [{\"storageType\": \"objectStorage\",\"target\": \"getFiles/${/input/objectName}\",\"namespaceName\": \"${/input/namespaceName}\",\"bucketName\": \"${/input/bucketName}\",\"objectName\": \"${/input/objectName}\"}]}" - val transcode_parameters = "{ \"transcodeType\": \"standardTranscode\", \"standardTranscode\": { \"input\": \"${/getFiles/taskParameters/0/target}\", \"outputPrefix\": \"${/transcode/outputPrefix}\", \"videoCodec\": \"h264\", \"audioCodec\": \"aac\", \"packaging\": { \"packageType\": \"hls\", \"segmentLength\": 6 }, \"ladder\": [ { \"size\": { \"height\": 1080, \"resizeMethod\": \"scale\" } }, { \"size\": { \"height\": 720, \"resizeMethod\": \"scale\" } }, { \"size\": { \"height\": 480, \"resizeMethod\": \"scale\" } }, { \"size\": { \"height\": 360, \"resizeMethod\": \"scale\" } } ] } }" - val thumbnail_parameters = "{ \"thumbnails\": { \"input\": \"${/getFiles/taskParameters/0/target}\", \"frameSelectors\": [ { \"namePrefix\": \"thumb\", \"format\": \"jpg\", \"sizes\": [ { \"width\": 390 }, { \"width\": 196 } ], \"clipImagePicker\": { \"percentList\": { \"pickList\": [ 10, 20, 30 ] } } } ] } }" - // val transcribe_parameters = "{ \"inputVideo\": \"${/getFiles/taskParameters/0/target}\", \"outputAudio\": \"${/output/objectNamePath}${/transcode/outputPrefix}.wav\", \"outputBucketName\": \"${/output/bucketName}\", \"outputNamespaceName\": \"${/output/namespaceName}\", \"outputTranscriptionPrefix\": \"${/output/objectNamePath}${/transcode/outputPrefix}\", \"transcriptionJobCompartment\": \"${/output/assetCompartmentId}\", \"waitForCompletion\" : true }" - val final_task_parameters = "{ \"taskParameters\": [ { \"namespaceName\": \"${/output/namespaceName}\", \"bucketName\": \"${/output/bucketName}\", \"source\": \"${/transcode/outputPrefix}*.m3u8\", \"objectName\": \"${/output/objectNamePath}${/transcode/outputPrefix}\", \"assetCompartmentId\": \"${/output/assetCompartmentId}\", \"registerMetadata\": true }, { \"namespaceName\": \"${/output/namespaceName}\", \"bucketName\": \"${/output/bucketName}\", \"source\": \"master.m3u8\", \"objectName\": \"${/output/objectNamePath}${/transcode/outputPrefix}-master.m3u8\", \"assetCompartmentId\": \"${/output/assetCompartmentId}\", \"registerMetadata\": true }, { \"namespaceName\": \"${/output/namespaceName}\", \"bucketName\": \"${/output/bucketName}\", \"source\": \"*.fmp4\", \"objectName\": \"${/output/objectNamePath}\", \"assetCompartmentId\": \"${/output/assetCompartmentId}\", \"registerMetadata\": true }, { \"namespaceName\": \"${/output/namespaceName}\", \"bucketName\": \"${/output/bucketName}\", \"source\": \"*.${/thumbnail/thumbnails/frameSelectors/0/format}\", \"objectName\": \"${/output/objectNamePath}${/transcode/outputPrefix}-\", \"assetCompartmentId\": \"${/output/assetCompartmentId}\", \"registerMetadata\": true } ] }" - val ingest_parameters = "{\"distributionChannelId\": \"distribution_channel_id\",\"masterPlaylistName\": \"master.m3u8\"}" - - val media_workflow_parameters = "{ \"input\": { \"objectName\": \"${/video/srcVideo}\", \"bucketName\": \"${/video/srcBucket}\", \"namespaceName\": \"${/video/namespace}\" }, \"output\": { \"bucketName\": \"${/video/dstBucket}\", \"namespaceName\": \"${/video/namespace}\", \"assetCompartmentId\": \"${/video/compartmentID}\", \"objectNamePath\": \"output/${/input/objectName}/\" }, \"transcode\": { \"outputPrefix\": \"${/video/outputPrefixName}\" } }" - val media_workflow_job_parameters = "{ \"video\": { \"srcBucket\": \"src_bucket\", \"dstBucket\": \"dst_bucket\", \"namespace\": \"namespace\", \"compartmentID\": \"compartment_id\", \"srcVideo\": \"src_video\", \"outputPrefixName\" : \"prefix_input\" } }" -} - diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/OCIResult.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/OCIResult.scala deleted file mode 100644 index 871ba1338..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/OCIResult.scala +++ /dev/null @@ -1,52 +0,0 @@ -package org.sunbird.job.videostream.helpers - -import org.apache.commons.lang3.StringUtils - -import scala.collection.immutable.HashMap - -object OCIResult extends Result { - - override def getSubmitJobResult(response: MediaResponse): Map[String, AnyRef] = { - val result = response.result - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> result.getOrElse("id", "").toString, - "status" -> result.getOrElse("lifecycleState", "").toString.toUpperCase(), - "submittedOn" -> result.getOrElse("timeCreated", "").toString, - "lastModifiedOn" -> result.getOrElse("timeUpdated", "").toString - ) - ) - } - - override def getJobResult(response: MediaResponse): Map[String, AnyRef] = { - val result = response.result - - HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> result.getOrElse("id", "").toString, - "status" -> result.getOrElse("lifecycleState", "").toString.toUpperCase(), - "submittedOn" -> result.getOrElse("timeCreated", "").toString, - "lastModifiedOn" -> result.getOrElse("timeUpdated", "").toString, - "error" -> { - if (StringUtils.equalsIgnoreCase(result.getOrElse("lifecycleState", "").toString.toUpperCase(),"FAILED")) { - Map[String, String]( - "errorCode" -> result.getOrElse("lifecycleState", "").toString, - "errorMessage" -> result.getOrElse("lifecycleDetails", "").toString - ) - } else { - null - } - } - ) - ) - } - - override def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - - override def getListJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/Response.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/Response.scala deleted file mode 100644 index 2207f433f..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/Response.scala +++ /dev/null @@ -1,66 +0,0 @@ -package org.sunbird.job.videostream.helpers - - -import java.util.UUID -import scala.collection.immutable.HashMap -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.util.{HTTPResponse, JSONUtil} - - -object Response { - - lazy val MEDIA_SERVICE_TYPE = "" -// val MEDIA_SERVICE_TYPE = AppConfig.getConfig("media_service_type") - - def getResponse(response: HTTPResponse): MediaResponse = { - var result: Map[String, AnyRef] = new HashMap[String, AnyRef] - - try { - val body = response.body - if (StringUtils.isNotBlank(body)) - result = JSONUtil.deserialize[Map[String, AnyRef]](body) - } catch { - case e: UnsupportedOperationException => e.printStackTrace() - case e: Exception => e.printStackTrace() - } - - response.status match { - case 200 => getSuccessResponse(result) - case 201 => getSuccessResponse(result) - case 400 => getFailureResponse(result, "BAD_REQUEST", "Please Provide Correct Request Data.") - case 401 => getFailureResponse(result, "SERVER_ERROR", "Access Token Expired.") - case 404 => getFailureResponse(result, "RESOURCE_NOT_FOUND", "Resource Not Found.") - case 405 => getFailureResponse(result, "METHOD_NOT_ALLOWED", "Requested Operation Not Allowed.") - case 500 => getFailureResponse(result, "SERVER_ERROR", "Internal Server Error. Please Try Again Later!") - case _ => getFailureResponse(result, "SERVER_ERROR", "Internal Server Error. Please Try Again Later!") - } - - } - - def getSuccessResponse(result: Map[String, AnyRef]): MediaResponse = { - MediaResponse(UUID.randomUUID().toString, System.currentTimeMillis().toString, new HashMap[String, AnyRef], - ResponseCode.OK.toString, result) - } - - def getFailureResponse(result: Map[String, AnyRef], errorCode: String, errorMessage: String): MediaResponse = { - val respCode: String = errorCode match { - case "BAD_REQUEST" => ResponseCode.CLIENT_ERROR.toString - case "RESOURCE_NOT_FOUND" => ResponseCode.RESOURCE_NOT_FOUND.toString - case "METHOD_NOT_ALLOWED" => ResponseCode.CLIENT_ERROR.toString - case "SERVER_ERROR" => ResponseCode.SERVER_ERROR.toString - } - val params = HashMap[String, String]( - "err" -> errorCode, - "errMsg" -> errorMessage - ) - MediaResponse(UUID.randomUUID().toString, System.currentTimeMillis().toString, params, respCode, result) - } - - def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } - - def getListJobResult(response: MediaResponse): Map[String, AnyRef] = { - null - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/Result.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/Result.scala deleted file mode 100644 index 7d7da8092..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/helpers/Result.scala +++ /dev/null @@ -1,13 +0,0 @@ -package org.sunbird.job.videostream.helpers - - -trait Result { - - def getSubmitJobResult(response: MediaResponse): Map[String, AnyRef] - - def getJobResult(response: MediaResponse): Map[String, AnyRef] - - def getCancelJobResult(response: MediaResponse): Map[String, AnyRef] - - def getListJobResult(response: MediaResponse): Map[String, AnyRef] -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/AwsMediaService.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/AwsMediaService.scala deleted file mode 100644 index ba6e2107a..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/AwsMediaService.scala +++ /dev/null @@ -1,81 +0,0 @@ -package org.sunbird.job.videostream.service - -import java.text.SimpleDateFormat -import java.util.{Date, TimeZone} - -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.videostream.exception.MediaServiceException -import org.sunbird.job.videostream.helpers.{AwsRequestBody, AwsSignUtils, MediaResponse, Response} -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig - -import scala.collection.immutable.HashMap -import scala.reflect.io.File - -abstract class AwsMediaService extends IMediaService { - - protected def getApiUrl(apiName: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val host: String = config.getConfig("aws.api.endpoint") - val apiVersion: String = config.getConfig("aws.api.version") - val baseUrl: String = host + File.separator + apiVersion - apiName.toLowerCase() match { - case "job" => baseUrl + "/jobs" - case _ => throw new MediaServiceException("ERR_INVALID_API_NAME", "Please Provide Valid AWS Media Service API Name") - } - } - - protected def getJobDetails(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("job") + "/" + jobId - val header = getDefaultHeader("GET", url, null) - Response.getResponse(httpUtil.get(url, header)) - } - - protected def prepareJobRequestBody(jobRequest: Map[String, AnyRef])(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val queue = config.getConfig("aws.service.queue") - val role = config.getConfig("aws.service.role") - val streamType = config.getConfig("aws.stream.protocol").toLowerCase() - val artifactUrl = jobRequest.get("artifactUrl").mkString - val contentId = jobRequest.get("identifier").mkString - val pkgVersion = jobRequest.getOrElse("pkgVersion", "").toString - val inputFile = prepareInputUrl(artifactUrl) - val output = prepareOutputUrl(contentId, streamType, pkgVersion) - AwsRequestBody.submit_hls_job - .replace("queueId", queue) - .replace("mediaRole", role) - .replace("inputVideoFile", inputFile) - .replace("outputLocation", output) - } - - protected def prepareInputUrl(url: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val temp = url.split("content") - val bucket = config.getConfig("aws.content_bucket_name") - val separator = File.separator; - "s3:" + separator + separator + bucket + separator + "content" + temp(1) - } - - protected def prepareOutputUrl(contentId: String, streamType: String, pkgVersion: String)(implicit config: VideoStreamGeneratorConfig): String = { - val bucket = config.getConfig("aws.content_bucket_name") - val output = streamType.toLowerCase + "_" + pkgVersion - val separator = File.separator; - "s3:" + separator + separator + bucket + separator + "content" + separator + contentId + separator + output + separator - } - - protected def getSignatureHeader()(implicit config: VideoStreamGeneratorConfig): Map[String, String] = { - val formatter = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'") - formatter.setTimeZone(TimeZone.getTimeZone("UTC")) - val date = formatter.format(new Date()) - val host: String = config.getConfig("aws.api.endpoint").replace("https://", "") - Map[String, String]("Content-Type" -> "application/json", "host" -> host, "x-amz-date" -> date) - } - - protected def getDefaultHeader(httpMethod: String, url: String, payload: String)(implicit config: VideoStreamGeneratorConfig): Map[String, String] = { - val signHeader = getSignatureHeader - val authToken = AwsSignUtils.generateToken(httpMethod, url, signHeader, payload) - val host: String = config.getConfig("aws.api.endpoint").replace("https://", "") - HashMap[String, String]( - "Content-Type" -> "application/json", - "host" -> host, - "x-amz-date" -> signHeader.get("x-amz-date").mkString, - "Authorization" -> authToken - ) - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/AzureMediaService.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/AzureMediaService.scala deleted file mode 100644 index b1d2b1f76..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/AzureMediaService.scala +++ /dev/null @@ -1,149 +0,0 @@ -package org.sunbird.job.videostream.service - -import java.io.File -import org.apache.commons.lang3.StringUtils -import org.sunbird.job.videostream.exception.MediaServiceException -import org.sunbird.job.util.{HttpUtil, JSONUtil} -import org.sunbird.job.videostream.helpers.{AzureRequestBody, MediaResponse, Response} -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig - -import scala.collection.immutable.HashMap - - -abstract class AzureMediaService extends IMediaService { - - private var API_ACCESS_TOKEN: String = "" - - private def getToken()(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val tenant = config.getSystemConfig("azure.tenant") - val clientKey = config.getSystemConfig("azure.token.client_key") - val clientSecret = config.getSystemConfig("azure.token.client_secret") - val loginUrl = config.getConfig("azure.login.endpoint") + "/" + tenant + "/oauth2/token" - - val data = Map[String, String]( - "grant_type" -> "client_credentials", - "client_id" -> clientKey, - "client_secret" -> clientSecret, - "resource" -> "https://management.core.windows.net/" - ) - - val header = Map[String, String]( - "Content-Type" -> "application/x-www-form-urlencoded", - "Keep-Alive" -> "true" - ) - - val response:MediaResponse = Response.getResponse(httpUtil.post_map(loginUrl, data, header)) - if(response.responseCode == "OK"){ - response.result("access_token").asInstanceOf[String] - } else { - throw new Exception("Error while getting the azure access token::"+JSONUtil.serialize(response)) - } - } - - protected def getJobDetails(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("job").replace("jobIdentifier", jobId) - val response:MediaResponse = Response.getResponse(httpUtil.get(url, getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while getting the job detail::"+JSONUtil.serialize(response)) - } - } - - protected def createAsset(assetId: String, jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("asset").replace("assetId", assetId) - val requestBody = AzureRequestBody.create_asset.replace("assetId", assetId) - .replace("assetDescription", "Output Asset for " + jobId) - val response:MediaResponse = Response.getResponse(httpUtil.put(url, requestBody, getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while creating asset::(assetId->"+assetId+", jobId->"+jobId+")::"+JSONUtil.serialize(response)) - } - } - - protected def createStreamingLocator(streamingLocatorName: String, assetName: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("stream_locator").replace("streamingLocatorName", streamingLocatorName) - val streamingPolicyName = config.getConfig("azure.stream.policy_name") - val reqBody = AzureRequestBody.create_stream_locator.replace("assetId", assetName).replace("policyName", streamingPolicyName) - Response.getResponse(httpUtil.put(url, reqBody, getDefaultHeader())) - } - - protected def getStreamingLocator(streamingLocatorName: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("stream_locator").replace("streamingLocatorName", streamingLocatorName) - val response:MediaResponse = Response.getResponse(httpUtil.get(url, getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while getStreamingLocator::(streamingLocatorName->" + streamingLocatorName + ")::"+JSONUtil.serialize(response)) - } - } - - protected def getStreamUrls(streamingLocatorName: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("list_paths").replace("streamingLocatorName", streamingLocatorName) - val response:MediaResponse = Response.getResponse(httpUtil.post(url, "{}", getDefaultHeader())) - if(response.responseCode == "OK"){ - response - } else { - throw new Exception("Error while getStreamUrls::(streamingLocatorName->" + streamingLocatorName + ")::"+JSONUtil.serialize(response)) - } - } - - protected def getApiUrl(apiName: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): String = { - val subscriptionId: String = config.getSystemConfig("azure.subscription_id") - val resourceGroupName: String = config.getSystemConfig("azure.resource_group_name") - val accountName: String = config.getSystemConfig("azure.account_name") - val apiVersion: String = config.getConfig("azure.api.version") - val transformName: String = config.getConfig("azure.transform.default") - - val baseUrl: String = new StringBuilder().append(config.getConfig("azure.api.endpoint")+"/subscriptions/") - .append(subscriptionId) - .append("/resourceGroups/") - .append(resourceGroupName) - .append("/providers/Microsoft.Media/mediaServices/") - .append(accountName).mkString - - - apiName.toLowerCase() match { - case "asset" => baseUrl + "/assets/assetId?api-version=" + apiVersion - case "job" => baseUrl + "/transforms/" + transformName + "/jobs/jobIdentifier?api-version=" + apiVersion - case "stream_locator" => baseUrl + "/streamingLocators/streamingLocatorName?api-version=" + apiVersion - case "list_paths" => baseUrl + "/streamingLocators/streamingLocatorName/listPaths?api-version=" + apiVersion - case _ => throw new MediaServiceException("ERR_INVALID_API_NAME", "Please Provide Valid Media Service API Name") - } - } - - protected def getDefaultHeader()(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): Map[String, String] = { - val accessToken = if (StringUtils.isNotBlank(API_ACCESS_TOKEN)) API_ACCESS_TOKEN else getToken() - val authToken = "Bearer " + accessToken - HashMap[String, String]( - "Content-Type" -> "application/json", - "Accept" -> "application/json", - "Authorization" -> authToken - ) - } - - protected def prepareStreamingUrl(streamLocatorName: String, jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): Map[String, AnyRef] = { - val streamType = config.getConfig("azure.stream.protocol") - val streamHost = config.getConfig("azure.stream.base_url") - var url = "" - val listPathResponse = getStreamUrls(streamLocatorName) - if (listPathResponse.responseCode.equalsIgnoreCase("OK")) { - val urlList: List[Map[String, AnyRef]] = listPathResponse.result.getOrElse("streamingPaths", List).asInstanceOf[List[Map[String, AnyRef]]] - urlList.foreach(streamMap => { - if (StringUtils.equalsIgnoreCase(streamMap.getOrElse("streamingProtocol", null).toString, streamType)) { - url = streamMap("paths").asInstanceOf[List[String]].head - } - }) - val streamUrl = streamHost + url.replace("aapl", "aapl-v3") - HashMap[String, AnyRef]("streamUrl" -> streamUrl) - } else { - val getResponse: MediaResponse = getJobDetails(jobId) - val fileName: String = getResponse.result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("input", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("files", List).asInstanceOf[List[AnyRef]].head.toString - val getStreamResponse = getStreamingLocator(streamLocatorName); - val locatorId = getStreamResponse.result.getOrElse("properties", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("streamingLocatorId", "").toString - val streamUrl = streamHost + File.separator + locatorId + File.separator + fileName.replace(".mp4", ".ism") + "/manifest(format=m3u8-aapl-v3)" - HashMap[String, AnyRef]("streamUrl" -> streamUrl) - } - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/IMediaService.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/IMediaService.scala deleted file mode 100644 index c3973167e..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/IMediaService.scala +++ /dev/null @@ -1,20 +0,0 @@ -package org.sunbird.job.videostream.service - -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.videostream.helpers.{MediaRequest, MediaResponse} - - -trait IMediaService { - - def submitJob(request: MediaRequest)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse - - def getJob(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse - - def getStreamingPaths(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse - - def listJobs(listJobsRequest: MediaRequest): MediaResponse - - def cancelJob(cancelJobRequest: MediaRequest): MediaResponse - -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/VideoStreamService.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/VideoStreamService.scala deleted file mode 100644 index 3dabfba85..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/VideoStreamService.scala +++ /dev/null @@ -1,215 +0,0 @@ -package org.sunbird.job.videostream.service - -import java.util.UUID - -import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} -import org.joda.time.DateTime -import org.slf4j.LoggerFactory -import org.sunbird.job.Metrics -import org.sunbird.job.videostream.service.impl.MediaServiceFactory -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.job.videostream.helpers.{JobRequest, MediaRequest, MediaResponse, StreamingStage} - -import scala.collection.JavaConverters._ - -class VideoStreamService(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil) { - private[this] lazy val logger = LoggerFactory.getLogger(classOf[VideoStreamService]) - private lazy val mediaService = MediaServiceFactory.getMediaService(config) - private lazy val dbKeyspace:String = config.dbKeyspace - private lazy val dbTable:String = config.dbTable - lazy val cassandraUtil:CassandraUtil = new CassandraUtil(config.lmsDbHost, config.lmsDbPort, config) - private lazy val clientKey:String = "SYSTEM_LP" - private lazy val SUBMITTED:String = "SUBMITTED" - private lazy val VIDEO_STREAMING:String = "VIDEO_STREAMING" - - def submitJobRequest(eData: Map[String, AnyRef]): Unit = { - val stageName = "STREAMING_JOB_SUBMISSION"; - val jobSubmitted = DateTime.now() - val requestId = UUID.randomUUID().toString - val jobRequest = JobRequest(clientKey, requestId, None, SUBMITTED, JSONUtil.serialize(eData), 0, Option(jobSubmitted), - Option(eData.getOrElse("artifactUrl", "").asInstanceOf[String]), None, None, None, None, None, - None, None, None, None, None, None, None, Option(stageName), Option(SUBMITTED), Option(VIDEO_STREAMING)) - - saveJobRequest(jobRequest) - submitStreamJob(jobRequest) - } - - def processJobRequest(metrics: Metrics): Unit = { - updateProcessingRequest(metrics) - resubmitFailedJob() - } - - def updateProcessingRequest(metrics: Metrics): Unit = { - val processingJobRequests = readFromDB(Map("status" -> "PROCESSING")) - val stageName = "STREAMING_JOB_COMPLETE" - - for (jobRequest <- processingJobRequests) { - val iteration = jobRequest.iteration - val streamStage = if (jobRequest.job_id != None) { - val mediaResponse:MediaResponse = mediaService.getJob(jobRequest.job_id.get) - logger.info("Get job details while saving: " + JSONUtil.serialize(mediaResponse.result)) - logger.info("updateProcessingRequest mediaResponse.responseCode: " + mediaResponse.responseCode) - if(mediaResponse.responseCode.contentEquals("OK")) { - val job = mediaResponse.result.getOrElse("job", Map()).asInstanceOf[Map[String, AnyRef]] - val jobStatus = job.getOrElse("status","").asInstanceOf[String] - val workFlowJobId = job.getOrElse("id","").asInstanceOf[String] - logger.info("updateProcessingRequest job: " + job) - logger.info("updateProcessingRequest workFlowJobId: " + workFlowJobId) - if(config.jobStatus.contains(jobStatus)) { - val streamingUrl = mediaService.getStreamingPaths(workFlowJobId).result.getOrElse("streamUrl","").asInstanceOf[String] - val requestData = JSONUtil.deserialize[Map[String, AnyRef]](jobRequest.request_data) - val contentId = requestData.getOrElse("identifier", "").asInstanceOf[String] - val channel = requestData.getOrElse("channel", "").asInstanceOf[String] - logger.info("Before calling updatePreviewUrl : contentId ::" + contentId+" streamingUrl ::"+streamingUrl+" channel ::"+channel) - if(updatePreviewUrl(contentId, streamingUrl, channel)) { - logger.info("updatePreviewUrl COMPLETE : contentId ::" + contentId) - StreamingStage(jobRequest.request_id, jobRequest.client_key, jobRequest.job_id.get, stageName, jobStatus, "FINISHED", iteration + 1); - } else { - // Set job status to FAILED - val errMessage = job.getOrElse("error", Map()).asInstanceOf[Map[String, AnyRef]].getOrElse("errorMessage", "Error updating streamingUrl").asInstanceOf[String] - StreamingStage(jobRequest.request_id, jobRequest.client_key, jobRequest.job_id.get, stageName, jobStatus, "FAILED", iteration + 1, errMessage) - } - } else if(jobStatus.equalsIgnoreCase("ERROR") || jobStatus.equalsIgnoreCase("FAILED")){ - val errMessage = job.getOrElse("error", Map()).asInstanceOf[Map[String, AnyRef]].getOrElse("errorMessage", "No error message").asInstanceOf[String] - StreamingStage(jobRequest.request_id, jobRequest.client_key, jobRequest.job_id.get, stageName, jobStatus, "FAILED", iteration + 1, errMessage) - } else { - null - } - } else { - val errorMsg = mediaResponse.result.toString - StreamingStage(jobRequest.request_id, jobRequest.client_key, null, stageName, "FAILED", "FAILED", iteration + 1, errorMsg); - } - } else { - StreamingStage(jobRequest.request_id, jobRequest.client_key, null, stageName, "FAILED", "FAILED", iteration + 1, jobRequest.err_message.getOrElse("")); - } - - if (streamStage != null) { - val counter = if (streamStage.status.equals("FINISHED")) config.successEventCount else { - if (streamStage.iteration <= config.maxRetries) config.retryEventCount else config.failedEventCount - } - metrics.incCounter(counter) - updateJobRequestStage(streamStage) - } - } - } - - def resubmitFailedJob(): Unit = { - val failedJobRequests = readFromDB(Map("status" -> "FAILED", "iteration" -> Map("type"-> "lte", "value" -> config.maxRetries))).toArray - failedJobRequests.foreach { jobRequest => - submitStreamJob(jobRequest) - } - } - - def submitStreamJob(jobRequest: JobRequest): Unit = { - - val requestData = JSONUtil.deserialize[Map[String, AnyRef]](jobRequest.request_data) - val mediaRequest = MediaRequest(UUID.randomUUID().toString, null, requestData) - val response:MediaResponse = mediaService.submitJob(mediaRequest) - val stageName = "STREAMING_JOB_SUBMISSION" - var streamStage:Option[StreamingStage] = None - - if (response.responseCode.equals("OK")) { - val jobId = response.result.getOrElse("job", Map()).asInstanceOf[Map[String, AnyRef]].getOrElse("id","").asInstanceOf[String]; - val jobStatus = response.result.getOrElse("job", Map()).asInstanceOf[Map[String, AnyRef]].getOrElse("status","").asInstanceOf[String]; - streamStage = Option(StreamingStage(jobRequest.request_id, jobRequest.client_key, jobId, stageName, jobStatus, "PROCESSING", jobRequest.iteration + 1)) - } else { - val errorMsg = response.result.toString - - streamStage = Option(StreamingStage(jobRequest.request_id, jobRequest.client_key, null, stageName, "FAILED", "FAILED", jobRequest.iteration + 1, errorMsg)); - } - - updateJobRequestStage(streamStage.get); - } - - private def updatePreviewUrl(contentId: String, streamingUrl: String, channel: String): Boolean = { - if(streamingUrl.nonEmpty && contentId.nonEmpty) { - val requestBody = "{\"request\": {\"content\": {\"streamingUrl\":\""+ streamingUrl +"\"}}}" - val url = config.lpURL + config.contentV4Update + contentId - val headers = Map[String, String]("X-Channel-Id" -> channel, "Content-Type"->"application/json") - logger.info("Before updating streamingUrl : url ::" + url+" requestBody ::"+requestBody) - val response:HTTPResponse = httpUtil.patch(url, requestBody, headers) - logger.info("updatePreviewUrl() response.status ::" + response.status) - if(response.status == 200){ - logger.info("StreamingUrl updated successfully. url ::" + url+" contentId ::"+contentId) - true - } else { - logger.error("Error while updating previewUrl for content : " + contentId + " :: "+response.body) -// throw new Exception("Error while updating previewUrl for content : " + contentId + " :: "+response.body) - false - } - } else { - false - } - } - - def readFromDB(columns: Map[String, AnyRef]): List[JobRequest] = { - val selectWhere: Select.Where = QueryBuilder.select().all() - .from(dbKeyspace, dbTable) - .allowFiltering() - .where() - - columns.map(col => { - col._2 match { - case value: List[Any] => - selectWhere.and(QueryBuilder.in(col._1, value.asJava)) - case value: Map[String, AnyRef] => - if (value("type") == "lte") { - selectWhere.and(QueryBuilder.lte(col._1, value("value"))) - } else { - selectWhere.and(QueryBuilder.gte(col._1, value("value"))) - } - case _ => - selectWhere.and(QueryBuilder.eq(col._1, col._2)) - } - }) - - selectWhere.and(QueryBuilder.eq("job_name", VIDEO_STREAMING)) - logger.info("readFromDB : Query ::" + selectWhere.toString) - val result = cassandraUtil.find(selectWhere.toString).asScala.toList.map { jr => - JobRequest(jr.getString("client_key"), jr.getString("request_id"), Option(jr.getString("job_id")), jr.getString("status"), jr.getString("request_data"), jr.getInt("iteration"), stage=Option(jr.getString("stage")), stage_status=Option(jr.getString("stage_status")),job_name=Option(jr.getString("job_name"))) - } - result - } - - def saveJobRequest(jobRequest: JobRequest): Boolean = { - val query = QueryBuilder.insertInto(dbKeyspace, dbTable) - .value("client_key", jobRequest.client_key) - .value("request_id", jobRequest.request_id) - .value("job_id", jobRequest.job_id.getOrElse("")) - .value("status", jobRequest.status) - .value("request_data", jobRequest.request_data) - .value("iteration", jobRequest.iteration) - .value("dt_job_submitted", setDateColumn(jobRequest.dt_job_submitted).get) - .value("location", jobRequest.location.get) - .value("stage", jobRequest.stage.get) - .value("stage_status", jobRequest.stage_status.get) - .value("job_name", jobRequest.job_name.get) - - val result = cassandraUtil.session.execute(query) - result.wasApplied() - } - - def updateJobRequestStage(streamStage: StreamingStage): Boolean = { - val query = QueryBuilder.update(dbKeyspace, dbTable) - .`with`(QueryBuilder.set("job_id", streamStage.job_id)) - .and(QueryBuilder.set("stage", streamStage.stage)) - .and(QueryBuilder.set("stage_status", streamStage.stage_status)) - .and(QueryBuilder.set("status", streamStage.status)) - .and(QueryBuilder.set("iteration", streamStage.iteration)) - .and(QueryBuilder.set("err_message", streamStage.err_message)) - .where(QueryBuilder.eq("request_id", streamStage.request_id)) - .and(QueryBuilder.eq("client_key", streamStage.client_key)) - - cassandraUtil.upsert(query.toString) - } - - def setDateColumn(date: Option[DateTime]): Option[Long] = { - val timestamp = date.orNull - if (null == timestamp) None else Option(timestamp.getMillis) - } - - def closeConnection(): Unit = { - cassandraUtil.close() - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/AwsMediaServiceImpl.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/AwsMediaServiceImpl.scala deleted file mode 100644 index 9d3ce78c1..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/AwsMediaServiceImpl.scala +++ /dev/null @@ -1,46 +0,0 @@ -package org.sunbird.job.videostream.service.impl - -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.videostream.helpers.{AwsResult, MediaRequest, MediaResponse, Response} -import org.sunbird.job.videostream.service.AwsMediaService -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig - -import scala.collection.immutable.HashMap - -object AwsMediaServiceImpl extends AwsMediaService { - - override def submitJob(request: MediaRequest)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val url = getApiUrl("job") - val reqBody = prepareJobRequestBody(request.request) - val header = getDefaultHeader("POST", url, reqBody) - val response:MediaResponse = Response.getResponse(httpUtil.post(url, reqBody, header)) - if (response.responseCode == "OK") Response.getSuccessResponse(AwsResult.getSubmitJobResult(response)) else response - } - - override def getJob(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val response = getJobDetails(jobId) - if (response.responseCode == "OK") Response.getSuccessResponse(AwsResult.getJobResult(response)) else response - } - - override def getStreamingPaths(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val region = config.getConfig("aws.region"); - val getResponse = getJobDetails(jobId) - val inputs: List[Map[String, AnyRef]] = getResponse.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("settings", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("inputs", List).asInstanceOf[List[Map[String, AnyRef]]] - val input: String = inputs.head.getOrElse("fileInput", "").toString - val outputGroups: List[Map[String, AnyRef]] = getResponse.result.getOrElse("job", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("settings", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("outputGroups", List).asInstanceOf[List[Map[String, AnyRef]]] - val outputGroupSettings = outputGroups.head.getOrElse("outputGroupSettings", Map).asInstanceOf[Map[String, AnyRef]] - val destination = outputGroupSettings.getOrElse("hlsGroupSettings", Map).asInstanceOf[Map[String, AnyRef]].getOrElse("destination", "").asInstanceOf[String] - val temp = destination.split("_") - val output = config.getConfig("aws.stream.protocol").toLowerCase() + "_" + temp(temp.length-1).replace("/","").trim() - val host = "https://s3." + region + ".amazonaws.com" - val streamUrl: String = input.replace("s3:/", host) - .replace("artifact", output) - .replace(".mp4", ".m3u8") - .replace(".webm", ".m3u8") - Response.getSuccessResponse(HashMap[String, AnyRef]("streamUrl" -> streamUrl)) - } - - override def listJobs(listJobsRequest: MediaRequest): MediaResponse = ??? - - override def cancelJob(cancelJobRequest: MediaRequest): MediaResponse = ??? -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/AzureMediaServiceImpl.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/AzureMediaServiceImpl.scala deleted file mode 100644 index 17d40515c..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/AzureMediaServiceImpl.scala +++ /dev/null @@ -1,56 +0,0 @@ -package org.sunbird.job.videostream.service.impl - -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.videostream.helpers.{AzureRequestBody, AzureResult, MediaRequest, MediaResponse, Response} -import org.sunbird.job.videostream.service.AzureMediaService -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig - -import scala.collection.immutable.HashMap - - -object AzureMediaServiceImpl extends AzureMediaService { - - override def submitJob(request: MediaRequest)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val inputUrl = request.request.getOrElse("artifactUrl", "").toString - val contentId = request.request.get("identifier").mkString - val jobId = contentId + "_" + System.currentTimeMillis() - val temp = inputUrl.splitAt(inputUrl.lastIndexOf("/") + 1) - val assetId = "asset-" + jobId - - val createAssetResponse = createAsset(assetId, jobId) - - if (createAssetResponse.responseCode.equalsIgnoreCase("OK")) { - val apiUrl = getApiUrl("job").replace("jobIdentifier", jobId) - val reqBody = AzureRequestBody.submit_job.replace("assetId", assetId).replace("baseInputUrl", temp._1).replace("inputVideoFile", temp._2) - val response:MediaResponse = Response.getResponse(httpUtil.put(apiUrl, reqBody, getDefaultHeader())) - if (response.responseCode == "OK") Response.getSuccessResponse(AzureResult.getSubmitJobResult(response)) else response - } else { - Response.getFailureResponse(createAssetResponse.result, "SERVER_ERROR", "Output Asset [ " + assetId + " ] Creation Failed for Job : " + jobId) - } - } - - override def getJob(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val response = getJobDetails(jobId) - if (response.responseCode == "OK") Response.getSuccessResponse(AzureResult.getJobResult(response)) else response - } - - override def getStreamingPaths(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val streamLocatorName = "sl-" + jobId - val assetName = "asset-" + jobId - val locatorResponse = createStreamingLocator(streamLocatorName, assetName) - if (locatorResponse.responseCode == "OK" || locatorResponse.responseCode == "CLIENT_ERROR") { - Response.getSuccessResponse(prepareStreamingUrl(streamLocatorName, jobId)) - } else { - Response.getFailureResponse(new HashMap[String, AnyRef], "SERVER_ERROR", "Streaming Locator [" + streamLocatorName + "] Creation Failed for Job : " + jobId) - } - } - - override def listJobs(listJobsRequest: MediaRequest): MediaResponse = { - null - } - - override def cancelJob(cancelJobRequest: MediaRequest): MediaResponse = { - null - } - -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/MediaServiceFactory.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/MediaServiceFactory.scala deleted file mode 100644 index 23a242b38..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/MediaServiceFactory.scala +++ /dev/null @@ -1,18 +0,0 @@ -package org.sunbird.job.videostream.service.impl - -import org.sunbird.job.videostream.exception.MediaServiceException -import org.sunbird.job.videostream.service.IMediaService -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig - -object MediaServiceFactory { - - def getMediaService(config: VideoStreamGeneratorConfig): IMediaService = { - val serviceType: String = config.getConfig("media_service_type") - serviceType match { - case "azure" => AzureMediaServiceImpl - case "aws" => AwsMediaServiceImpl - case "oci" => OCIMediaServiceImpl - case _ => throw new MediaServiceException("ERR_INVALID_SERVICE_TYPE", "Please Provide Valid Media Service Name") - } - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/OCIMediaServiceImpl.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/OCIMediaServiceImpl.scala deleted file mode 100644 index e937a7b23..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/service/impl/OCIMediaServiceImpl.scala +++ /dev/null @@ -1,118 +0,0 @@ -package org.sunbird.job.videostream.service.impl - -import org.slf4j.LoggerFactory -import org.sunbird.job.util.HttpUtil -import org.sunbird.job.videostream.helpers.{MediaRequest, MediaResponse, MediaServiceHelper, OCIRequestBody, Response, ResponseCode} -import org.sunbird.job.videostream.service.IMediaService -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig -import scala.collection.immutable.HashMap -import com.google.gson.Gson -import java.util -import org.sunbird.job.util.JSONUtil - - -object OCIMediaServiceImpl extends IMediaService { - - private[this] val logger = LoggerFactory.getLogger("OCIMediaServiceImpl") - - override def submitJob(request: MediaRequest)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - - val inputUrl = request.request.getOrElse("artifactUrl", "").toString - logger.info("inputUrl...{}",inputUrl) - val contentId = request.request.get("identifier").mkString - val compartment_id = config.getConfig("oci.compartment_id") - logger.info("compartment_id...{}",compartment_id) - val src_bucket = inputUrl.split("/")(3); - logger.info("src_bucket...{}",src_bucket) - val dst_bucket = config.getConfig("oci.bucket.processed_bucket_name") - val namespace = config.getConfig("oci.namespace") - val temp = inputUrl.splitAt(inputUrl.lastIndexOf("/") + 1) - val src_video = inputUrl.substring(inputUrl.indexOf(inputUrl.split("/")(3))+inputUrl.split("/")(3).length+1, inputUrl.length) - logger.info("src_video...{}",src_video) - val prefix_input = config.getConfig("oci.stream.prefix_input") - val media_flow_id = config.getConfig("oci.stream.work_flow_id") - logger.info("media_flow_id...{}",media_flow_id) - val mediaServiceHelper = new MediaServiceHelper() - val mediaflowjobParameters = "{ \"video\": { \"srcBucket\": \"" + src_bucket + "\", \"dstBucket\": \""+ dst_bucket + "\", \"namespace\": \"" + namespace + "\", \"compartmentID\": \"" + compartment_id + "\", \"srcVideo\": \"" + src_video + "\", \"outputPrefixName\" : \"" + prefix_input + "\" } }" - val mediaFlowResp = mediaServiceHelper.submitJob(compartment_id, media_flow_id, mediaServiceHelper.createJSONObject(mediaflowjobParameters)) - logger.info("mediaFlowResp.getId...{}",mediaFlowResp.getId) - if (mediaFlowResp.getLifecycleState != "FAILED") { - MediaResponse(mediaFlowResp.getId, System.currentTimeMillis().toString, new HashMap[String, AnyRef], - ResponseCode.OK.toString, HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> mediaFlowResp.getId, - "status" -> mediaFlowResp.getLifecycleState.toString.toUpperCase, - "submittedOn" -> mediaFlowResp.getTimeCreated.toString, - "lastModifiedOn" -> mediaFlowResp.getTimeUpdated.toString - ) - )) - }else { - Response.getFailureResponse(HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> mediaFlowResp.getId, - "status" -> mediaFlowResp.getLifecycleState.toString.toUpperCase, - "submittedOn" -> mediaFlowResp.getTimeCreated.toString, - "lastModifiedOn" -> mediaFlowResp.getTimeUpdated.toString - ) - ), "SERVER_ERROR", "Output Asset [ " + contentId + " ] Creation Failed for Job : " + mediaFlowResp.getId) - } - } - def jsonToMap(json: String): Map[String, AnyRef] = { - val gson = new Gson() - gson.fromJson(json, new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[Map[String, AnyRef]] - } - - override def getJob(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val mediaServiceHelper = new MediaServiceHelper() - try - { - val mediaFlowResp = mediaServiceHelper.getWorkflowJob(jobId); - logger.info("LifecycleState...{}",mediaFlowResp.getLifecycleState) - if (mediaFlowResp.getLifecycleState != "FAILED") { - MediaResponse(mediaFlowResp.getId, System.currentTimeMillis().toString, new HashMap[String, AnyRef], - ResponseCode.OK.toString, HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> mediaFlowResp.getId, - "status" -> mediaFlowResp.getLifecycleState.toString.toUpperCase, - "submittedOn" -> mediaFlowResp.getTimeCreated.toString, - "lastModifiedOn" -> mediaFlowResp.getTimeUpdated.toString - ) - )) - } else { - Response.getFailureResponse(HashMap[String, AnyRef]( - "job" -> HashMap[String, AnyRef]( - "id" -> mediaFlowResp.getId, - "status" -> mediaFlowResp.getLifecycleState.toString.toUpperCase, - "submittedOn" -> mediaFlowResp.getTimeCreated.toString, - "lastModifiedOn" -> mediaFlowResp.getTimeUpdated.toString - ) - ), "SERVER_ERROR", "Get WorkFlowJob Failed for the Id : " + mediaFlowResp.getId) - } } catch { - case e: Exception => e.printStackTrace() - Response.getFailureResponse(new HashMap[String, AnyRef], "SERVER_ERROR", "Get WorkFlowJob Failed for Job : " + jobId) - } - } - - override def getStreamingPaths(jobId: String)(implicit config: VideoStreamGeneratorConfig, httpUtil: HttpUtil): MediaResponse = { - val mediaServiceHelper = new MediaServiceHelper() - val gatewayDomain = config.getConfig("oci.stream.gateway_domain") - val streamUrl = mediaServiceHelper.getStreamingPaths(jobId, gatewayDomain) - logger.info("streamUrl...{}",streamUrl) - if (streamUrl == null || streamUrl == None) { - Response.getFailureResponse(new HashMap[String, AnyRef], "SERVER_ERROR", "Streaming Locator Creation Failed for Job : " + jobId) - } - else - { - Response.getSuccessResponse(HashMap[String, AnyRef]("streamUrl" -> streamUrl)) - } - } - - override def listJobs(listJobsRequest: MediaRequest): MediaResponse = { - null - } - - override def cancelJob(cancelJobRequest: MediaRequest): MediaResponse = { - null - } - -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/task/VideoStreamGeneratorConfig.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/task/VideoStreamGeneratorConfig.scala deleted file mode 100644 index 196712dbc..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/task/VideoStreamGeneratorConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package org.sunbird.job.videostream.task - -import java.util - -import com.typesafe.config.Config -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.sunbird.job.BaseJobConfig -import org.sunbird.job.videostream.exception.MediaServiceException - -class VideoStreamGeneratorConfig(override val config: Config) extends BaseJobConfig(config, "video-stream-generator") { - - private val serialVersionUID = 2905979434303791379L - - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - // Kafka Topics Configuration - val kafkaInputTopic: String = config.getString("kafka.input.topic") - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - override val parallelism: Int = config.getInt("task.parallelism") - - val timerDuration = config.getInt("task.timer.duration") // Timer duration in sec. - val maxRetries = if (config.hasPath("task.max.retries")) config.getInt("task.max.retries") else 10 - - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val retryEventCount = "retry-events-count" - val skippedEventCount = "skipped-events-count" - - // Consumers - val videoStreamConsumer = "video-streaming-consumer" - val videoStreamGeneratorFunction = "manage-streaming-jobs" - - // Cassandra Configurations - val dbTable: String = config.getString("lms-cassandra.table") - val dbKeyspace: String = config.getString("lms-cassandra.keyspace") - val hierarchyPrimaryKey: List[String] = List("identifier") - - // LP Configurations - val lpURL: String = config.getString("service.content.basePath") - val contentV4Update = "/content/v4/system/update/" - - val jobStatus:util.List[String] = if(config.hasPath("media_service_job_success_status")) config.getStringList("media_service_job_success_status") else util.Arrays.asList("FINISHED", "COMPLETE", "SUCCEEDED") - - def getConfig(key: String): String = { - if (config.hasPath(key)) - config.getString(key) - else throw new MediaServiceException("CONFIG_NOT_FOUND", "Configuration for key [" + key + "] Not Found.") - } - - def getSystemConfig(key: String): String = { - val sysKey=key.replaceAll("\\.","_") - if (config.hasPath(sysKey)) - config.getString(sysKey) - else throw new MediaServiceException("CONFIG_NOT_FOUND", "Configuration for key [" + sysKey + "] Not Found.") - } -} diff --git a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/task/VideoStreamGeneratorStreamTask.scala b/video-stream-generator/src/main/scala/org/sunbird/job/videostream/task/VideoStreamGeneratorStreamTask.scala deleted file mode 100644 index a6db17dff..000000000 --- a/video-stream-generator/src/main/scala/org/sunbird/job/videostream/task/VideoStreamGeneratorStreamTask.scala +++ /dev/null @@ -1,53 +0,0 @@ -package org.sunbird.job.videostream.task - -import java.io.File -import java.util -import com.typesafe.config.ConfigFactory -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.api.java.utils.ParameterTool -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.videostream.domain.Event -import org.sunbird.job.videostream.functions.VideoStreamGenerator -import org.sunbird.job.util.{FlinkUtil, HttpUtil} - - -class VideoStreamGeneratorStreamTask(config: VideoStreamGeneratorConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { - def process(): Unit = { - implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) - implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) - implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) - env.addSource(source).name(config.videoStreamConsumer) - .uid(config.videoStreamConsumer).setParallelism(config.kafkaConsumerParallelism) - .rebalance - .keyBy(_.identifier) - .process(new VideoStreamGenerator(config, httpUtil)) - .name(config.videoStreamGeneratorFunction) - .uid(config.videoStreamGeneratorFunction) - .setParallelism(config.parallelism) - - env.execute(config.jobName) - } -} - -// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object VideoStreamGeneratorStreamTask { - - def main(args: Array[String]): Unit = { - val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) - val config = configFilePath.map { - path => ConfigFactory.parseFile(new File(path)).resolve() - }.getOrElse(ConfigFactory.load("video-stream-generator.conf").withFallback(ConfigFactory.systemEnvironment())) - val videoStreamConfig = new VideoStreamGeneratorConfig(config) - val kafkaUtil = new FlinkKafkaConnector(videoStreamConfig) - val httpUtil = new HttpUtil - val task = new VideoStreamGeneratorStreamTask(videoStreamConfig, kafkaUtil, httpUtil) - task.process() - } -} - -// $COVERAGE-ON$ diff --git a/video-stream-generator/src/test/resources/job_request.cql b/video-stream-generator/src/test/resources/job_request.cql deleted file mode 100644 index 820a372b7..000000000 --- a/video-stream-generator/src/test/resources/job_request.cql +++ /dev/null @@ -1,2 +0,0 @@ - -INSERT INTO local_platform_db.job_request (client_key,request_id,dt_expiration,dt_file_created,dt_first_event,dt_job_completed,dt_job_processing,dt_job_submitted,dt_last_event,err_message,execution_time,file_size,input_events,iteration,job_id,job_name,latency,location,output_events,request_data,stage,stage_status,status) values ('SYSTEM_LP', '32722dd03b737ed32014a01b8c1f7c83', null, null, null, null, '2019-01-02 22:30:08.033+0000', '2018-12-18 20:11:03.088+0000', null, '', 0, 0, 0, 1, 'do_3126597193576939521909_1605816926271', 'VIDEO_STREAMING', 0, 'https://testurl.com/test.mp4', 0, '{"content_id":"do_3126597193576939521909","channel":"01254290140407398431","artifactUrl":"https://testurl.com/test.mp4"}', 'STREAMING_JOB_SUBMISSION', 'PROCESSING', 'PROCESSING'); \ No newline at end of file diff --git a/video-stream-generator/src/test/resources/logback-test.xml b/video-stream-generator/src/test/resources/logback-test.xml deleted file mode 100644 index e81294323..000000000 --- a/video-stream-generator/src/test/resources/logback-test.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - - - \ No newline at end of file diff --git a/video-stream-generator/src/test/resources/test.conf b/video-stream-generator/src/test/resources/test.conf deleted file mode 100644 index d92b086ac..000000000 --- a/video-stream-generator/src/test/resources/test.conf +++ /dev/null @@ -1,74 +0,0 @@ -include "base-test.conf" - -kafka { - input.topic = "sunbirddev.content.postpublish.request" - groupId = "sunbirddev-video-stream-generator-group" -} - -task { - consumer.parallelism = 1 - timer.duration = 10 - max.retries = 10 -} - -lms-cassandra { - keyspace = "local_platform_db" - table = "job_request" -} - -service { - content { - basePath = "http://dev.sunbirded.org/content" - } -} - -threshold.batch.read.interval = 60 // In sec -threshold.batch.read.size = 1000 -threshold.batch.write.size = 4 - - azure { - location = "centralindia" - tenant = "tenant name" - subscription_id = "subscription id" - - login { - endpoint="https://login.microsoftonline.com" - } - - api { - endpoint="https://management.azure.com" - version = "2018-07-01" - } - - account_name = "account name" - resource_group_name = "Resource Group Name" - - transform { - default = "media_transform_default" - hls = "media_transform_hls" - } - - stream { - base_url = "https://sunbirdspikemedia-inct.streaming.media.azure.net" - endpoint_name = "default" - protocol = "Hls" - policy_name = "Predefined_ClearStreamingOnly" - } - - token { - client_key = "client key" - client_secret = "client secret" - } -} - -azure_tenant="test_tenant" -azure_subscription_id="test_id" -azure_account_name="test_account_name" -azure_resource_group_name="test_resource_group_name" -azure_token_client_key="test_client_key" -azure_token_client_secret="test_client_secret" -elasticsearch.service.endpoint="test_service_endpoint" -elasticsearch.index.compositesearch.name="test_compositesearch_name" - -media_service_type="azure" - diff --git a/video-stream-generator/src/test/resources/test.cql b/video-stream-generator/src/test/resources/test.cql deleted file mode 100644 index 6ed1eca28..000000000 --- a/video-stream-generator/src/test/resources/test.cql +++ /dev/null @@ -1,31 +0,0 @@ -CREATE KEYSPACE IF NOT EXISTS local_platform_db WITH replication = { - 'class': 'SimpleStrategy', - 'replication_factor': '1' -}; - -CREATE TABLE IF NOT EXISTS local_platform_db.job_request ( - client_key text, - request_id text, - job_id text, - status text, - request_data text, - location text, - dt_file_created timestamp, - dt_first_event timestamp, - dt_last_event timestamp, - dt_expiration timestamp, - iteration int, - dt_job_submitted timestamp, - dt_job_processing timestamp, - dt_job_completed timestamp, - input_events int, - output_events int, - file_size bigint, - latency int, - execution_time bigint, - err_message text, - stage text, - stage_status text, - job_name text, - PRIMARY KEY (client_key, request_id) -); \ No newline at end of file diff --git a/video-stream-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/video-stream-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala deleted file mode 100644 index c56e49461..000000000 --- a/video-stream-generator/src/test/scala/org/sunbird/job/fixture/EventFixture.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.sunbird.job.fixture - -object EventFixture { - - val EVENT_1: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1598956686981,"mid":"LP.1598956686981.a260af12-cd9b-4ffd-a525-1d944df47c61","actor":{"id":"Post Publish Processor","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"01254290140407398431","env":"sunbirddev"},"object":{"ver":"1587632475439","id":"do_3126597193576939521910"},"edata":{"action":"post-publish-process","iteration":1,"identifier":"do_3126597193576939521910","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_1126980548391075841140/ariel-view-of-earth.mp4","mimeType":"video/mp4","contentType":"Resource","pkgVersion":1,"status":"Live"}} - |""".stripMargin - - val EVENT_2: String = - """ - |{"eid":"BE_JOB_REQUEST","ets":1598956686981,"mid":"LP.1598956686981.a260af12-cd9b-4ffd-a525-1d944df47c61","actor":{"id":"Post Publish Processor","type":"System"},"context":{"pdata":{"ver":"1.0","id":"org.ekstep.platform"},"channel":"01254290140407398431","env":"sunbirddev"},"object":{"ver":"1587632475439","id":"do_3126597193576939521910"},"edata":{"action":"post-publish-process","iteration":1,"identifier":"do_3126597193576939521910","artifactUrl":"https://sunbirded.com/test.mp4","mimeType":"video","contentType":"Resource","pkgVersion":1,"status":"Live"}} - |""".stripMargin -} \ No newline at end of file diff --git a/video-stream-generator/src/test/scala/org/sunbird/job/spec/VideoStreamGeneratorTaskTestSpec.scala b/video-stream-generator/src/test/scala/org/sunbird/job/spec/VideoStreamGeneratorTaskTestSpec.scala deleted file mode 100644 index cff9e7e32..000000000 --- a/video-stream-generator/src/test/scala/org/sunbird/job/spec/VideoStreamGeneratorTaskTestSpec.scala +++ /dev/null @@ -1,130 +0,0 @@ -package org.sunbird.job.spec - -import java.util -import com.datastax.driver.core.Row -import com.typesafe.config.{Config, ConfigFactory} -import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.api.java.typeutils.TypeExtractor -import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration -import org.apache.flink.streaming.api.functions.source.SourceFunction -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext -import org.apache.flink.test.util.MiniClusterWithClientResource -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.mockito.ArgumentMatchers.{any, anyString, contains} -import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.videostream.domain.Event -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.videostream.service.IMediaService -import org.sunbird.job.videostream.task.{VideoStreamGeneratorConfig, VideoStreamGeneratorStreamTask} -import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} - -import scala.collection.JavaConverters._ - -class VideoStreamGeneratorTaskTestSpec extends BaseTestSpec { - - implicit val mapTypeInfo: TypeInformation[java.util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[java.util.Map[String, AnyRef]]) - - val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() - .setConfiguration(testConfiguration()) - .setNumberSlotsPerTaskManager(1) - .setNumberTaskManagers(1) - .build) - val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) - val mediaService: IMediaService = mock[IMediaService](Mockito.withSettings().serializable()) - val config: Config = ConfigFactory.load("test.conf") - val jobConfig: VideoStreamGeneratorConfig = new VideoStreamGeneratorConfig(config) - val mockHttpUtil:HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) -// val httpUtil:HttpUtil = new HttpUtil - var cassandraUtil: CassandraUtil = _ - var currentMilliSecond = 1605816926271L - - val accessTokenResp = """{"token_type":"Bearer","expires_in":"3599","ext_expires_in":"3599","expires_on":"1605789466","not_before":"1605785566","resource":"https://management.core.windows.net/","access_token":"testToken"}""" - val assetJson = """{"name":"asset-do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-aaaa-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/assets/asset-do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/assets","properties":{"assetId":"aaaaaaa-13bb-45c7-aaaa-32ac2e97cf12","created":"2020-11-19T20:16:54.463Z","lastModified":"2020-11-19T20:20:33.613Z","alternateId":"asset-do_3126597193576939521910_1605816926271","description":"Output Asset for do_3126597193576939521910_1605816926271","container":"asset-aaaaaaa-13bb-45c7-b186-32ac2e97cf12","storageAccountName":"sunbirddevmedia","storageEncryptionFormat":"None"}}""" - val submitJobJson = """{"name":"do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-8a14-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/transforms/media_transform_default/jobs/do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/transforms/jobs","properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"Queued","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getJobJson = """{"name":"do_3126597193576939521910_1605816926271","job":{"status":"Finished"},"properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"FINISHED","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getStreamUrlJson = """{"streamingPaths":[{"streamingProtocol":"Hls","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-aapl)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-cmaf)"]},{"streamingProtocol":"Dash","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-csf)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-cmaf)"]},{"streamingProtocol":"SmoothStreaming","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest"]}],"downloadPaths":[]}""" - val getStreamLocatorJson = """{"properties":{"streamingLocatorId":"adcacdd-13bb-45c7-aaaa-32ac2e97cf12"}}""" - - override protected def beforeAll(): Unit = { - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.lmsDbHost, jobConfig.lmsDbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - testCassandraUtil(cassandraUtil) - BaseMetricsReporter.gaugeMetrics.clear() - flinkCluster.before() - super.beforeAll() - } - - override protected def afterAll(): Unit = { - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - flinkCluster.after() - super.afterAll() - } - - override protected def afterEach():Unit = { - super.afterEach() - } - - ignore should "submit a job" in { - when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new VideoStreamGeneratorMapSource) - - when(mockHttpUtil.post_map(contains("/oauth2/token"), any[Map[String, AnyRef]](), any[Map[String, String]]())).thenReturn(HTTPResponse(200, accessTokenResp)) - when(mockHttpUtil.put(contains("/providers/Microsoft.Media/mediaServices/"+jobConfig.getSystemConfig("azure.account.name")+"/assets/asset-"), anyString(), any())).thenReturn(HTTPResponse(200, assetJson)) - when(mockHttpUtil.put(contains("transforms/media_transform_default/jobs"), anyString(), any())).thenReturn(HTTPResponse(200, submitJobJson)) - when(mockHttpUtil.get(contains("transforms/media_transform_default/jobs"), any())).thenReturn(HTTPResponse(200, getJobJson)) - - when(mockHttpUtil.post(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271/listPaths?api-version="), any(), any())).thenReturn(HTTPResponse(200, getStreamUrlJson)) - when(mockHttpUtil.put(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any(), any())).thenReturn(HTTPResponse(400, getJobJson)) - when(mockHttpUtil.get(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any())).thenReturn(HTTPResponse(200, getStreamLocatorJson)) - when(mockHttpUtil.patch(contains(jobConfig.contentV4Update), any(), any())).thenReturn(HTTPResponse(200, getJobJson)) - - new VideoStreamGeneratorStreamTask(jobConfig, mockKafkaUtil, mockHttpUtil).process() - val event1Progress = readFromCassandra(EventFixture.EVENT_1) - - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.skippedEventCount}").getValue() should be(1) - BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.successEventCount}").getValue() should be(1) - event1Progress.size() should be(1) - - event1Progress.forEach(col => { - col.getObject("status") should be("FINISHED") - }) - - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - } - - def readFromCassandra(event: String): util.List[Row] = { - val event1 = JSONUtil.deserialize[Map[String, Any]](event) - val contentId = event1("object").asInstanceOf[Map[String, AnyRef]]("id") - val query = s"select * from ${jobConfig.dbKeyspace}.${jobConfig.dbTable} where job_id='${contentId}_$currentMilliSecond' ALLOW FILTERING;" - cassandraUtil.find(query) - } - -} - -class VideoStreamGeneratorMapSource extends SourceFunction[Event] { - - override def run(ctx: SourceContext[Event]) { - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 10)) - ctx.collect(new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2),0, 11)) - - } - - override def cancel(): Unit = {} - -} \ No newline at end of file diff --git a/video-stream-generator/src/test/scala/org/sunbird/job/spec/service/VideoStreamServiceTestSpec.scala b/video-stream-generator/src/test/scala/org/sunbird/job/spec/service/VideoStreamServiceTestSpec.scala deleted file mode 100644 index e1d2998b9..000000000 --- a/video-stream-generator/src/test/scala/org/sunbird/job/spec/service/VideoStreamServiceTestSpec.scala +++ /dev/null @@ -1,96 +0,0 @@ -package org.sunbird.job.spec.service - -import java.util -import com.datastax.driver.core.Row -import com.typesafe.config.{Config, ConfigFactory} -import org.cassandraunit.CQLDataLoader -import org.cassandraunit.dataset.cql.FileCQLDataSet -import org.cassandraunit.utils.EmbeddedCassandraServerHelper -import org.joda.time.DateTimeUtils -import org.mockito.ArgumentMatchers.{any, anyString, contains} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.sunbird.job.fixture.EventFixture -import org.sunbird.job.videostream.service.VideoStreamService -import org.sunbird.job.videostream.task.VideoStreamGeneratorConfig -import org.sunbird.spec.BaseTestSpec -import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} -import org.sunbird.job.videostream.domain.Event -import org.sunbird.job.Metrics - -class VideoStreamServiceTestSpec extends BaseTestSpec { - var cassandraUtil: CassandraUtil = _ - val config: Config = ConfigFactory.load("test.conf") - lazy val jobConfig: VideoStreamGeneratorConfig = new VideoStreamGeneratorConfig(config) - val httpUtil: HttpUtil = new HttpUtil - val mockHttpUtil:HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) - val metricJson = s"""{"${jobConfig.totalEventsCount}": 0, "${jobConfig.skippedEventCount}": 0}""" - val mockMetrics = mock[Metrics](Mockito.withSettings().serializable()) - - val accessTokenResp = """{"token_type":"Bearer","expires_in":"3599","ext_expires_in":"3599","expires_on":"1605789466","not_before":"1605785566","resource":"https://management.core.windows.net/","access_token":"testToken"}""" - val assetJson = """{"name":"asset-do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-aaaa-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/assets/asset-do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/assets","properties":{"assetId":"aaaaaaa-13bb-45c7-aaaa-32ac2e97cf12","created":"2020-11-19T20:16:54.463Z","lastModified":"2020-11-19T20:20:33.613Z","alternateId":"asset-do_3126597193576939521910_1605816926271","description":"Output Asset for do_3126597193576939521910_1605816926271","container":"asset-aaaaaaa-13bb-45c7-b186-32ac2e97cf12","storageAccountName":"sunbirddevmedia","storageEncryptionFormat":"None"}}""" - val submitJobJson = """{"name":"do_3126597193576939521910_1605816926271","id":"/subscriptions/aaaaaaaa-6899-4ef6-8a14-5a185b3b7254/resourceGroups/sunbird-devnew-env/providers/Microsoft.Media/mediaservices/sunbirddevmedia/transforms/media_transform_default/jobs/do_3126597193576939521910_1605816926271","type":"Microsoft.Media/mediaservices/transforms/jobs","properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"Queued","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getJobJson = """{"name":"do_3126597193576939521910_1605816926271","job":{"status":"Finished"},"properties":{"created":"2020-11-19T20:26:49.7953248Z","state":"Scheduled","input":{"@odata.type":"#Microsoft.Media.JobInputHttp","files":["test.mp4"],"baseUri":"https://sunbirded.com/"},"lastModified":"2020-11-19T20:26:49.7953248Z","outputs":[{"@odata.type":"#Microsoft.Media.JobOutputAsset","state":"FINISHED","progress":0,"label":"BuiltInStandardEncoderPreset_0","assetName":"asset-do_3126597193576939521910_1605816926271"}],"priority":"Normal","correlationData":{}}}""" - val getStreamUrlJson = """{"streamingPaths":[{"streamingProtocol":"Hls","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-aapl)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=m3u8-cmaf)"]},{"streamingProtocol":"Dash","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-csf)","/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest(format=mpd-time-cmaf)"]},{"streamingProtocol":"SmoothStreaming","encryptionScheme":"NoEncryption","paths":["/4ddff5cd-6479-4572-bc95-ebad508b65ce/ariel-view-of-earth.ism/manifest"]}],"downloadPaths":[]}""" - val getStreamLocatorJson = """{"properties":{"streamingLocatorId":"adcacdd-13bb-45c7-aaaa-32ac2e97cf12"}}""" - - override protected def beforeAll(): Unit = { - DateTimeUtils.setCurrentMillisFixed(1605816926271L); - EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) - cassandraUtil = new CassandraUtil(jobConfig.lmsDbHost, jobConfig.lmsDbPort, jobConfig) - val session = cassandraUtil.session - val dataLoader = new CQLDataLoader(session); - dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)); - testCassandraUtil(cassandraUtil) - super.beforeAll() - } - - override protected def afterAll(): Unit = { - DateTimeUtils.setCurrentMillisSystem(); - try { - EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() - } catch { - case ex: Exception => { - } - } - super.afterAll() - } - - "VideoStreamService" should "submit job request" in { - when(mockHttpUtil.post_map(contains("/oauth2/token"), any[Map[String, AnyRef]](), any[Map[String, String]]())).thenReturn(HTTPResponse(200, accessTokenResp)) - when(mockHttpUtil.put(contains("/providers/Microsoft.Media/mediaServices/"+jobConfig.getSystemConfig("azure.account.name")+"/assets/asset-"), anyString(), any())).thenReturn(HTTPResponse(200, assetJson)) - when(mockHttpUtil.put(contains("transforms/media_transform_default/jobs"), anyString(), any())).thenReturn(HTTPResponse(200, submitJobJson)) - when(mockHttpUtil.get(contains("transforms/media_transform_default/jobs"), any())).thenReturn(HTTPResponse(200, getJobJson)) - - when(mockHttpUtil.post(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271/listPaths?api-version="), any(), any())).thenReturn(HTTPResponse(200, getStreamUrlJson)) - when(mockHttpUtil.put(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any(), any())).thenReturn(HTTPResponse(400, getJobJson)) - when(mockHttpUtil.get(contains("/streamingLocators/sl-do_3126597193576939521910_1605816926271?api-version="), any())).thenReturn(HTTPResponse(200, getStreamLocatorJson)) - when(mockHttpUtil.patch(contains(jobConfig.contentV4Update), any(), any())).thenReturn(HTTPResponse(200, getJobJson)) - doNothing().when(mockMetrics).incCounter(any()) - - val eventMap1 = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_1),0, 12) - - val videoStreamService = new VideoStreamService()(jobConfig, mockHttpUtil); - videoStreamService.submitJobRequest(eventMap1.eData) - videoStreamService.processJobRequest(mockMetrics) - - val event1Progress = readFromCassandra(EventFixture.EVENT_1) - event1Progress.size() should be(1) - - event1Progress.forEach(col => { - col.getObject("status") should be("FINISHED") - }) - - } - - def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { - cassandraUtil.reconnect() - } - - def readFromCassandra(event: String): util.List[Row] = { - val event1 = JSONUtil.deserialize[Map[String, Any]](event) - val contentId = event1("object").asInstanceOf[Map[String, AnyRef]]("id") - val query = s"select * from ${jobConfig.dbKeyspace}.${jobConfig.dbTable} where job_id='${contentId}_1605816926271' ALLOW FILTERING;" - cassandraUtil.find(query) - } -}