Issue #CO-748 merge: Merge pull request #8 from amitpriyadarshi/relea…

…se-7.0.0

CO-748: metrics-data-transformer job migration from knowlg to coKreat

cokreat-user-delete/src/test/resources/test.conf

@@ -23,6 +23,8 @@ source {
   baseUrl = "https://dev.sunbirded.org/api"
 }
 
+program-service-baseUrl = "http://localhost:9000"
+
 cloudstorage.metadata.replace_absolute_path=true
 cloudstorage.read_base_path="https://sunbirddev.blob.core.windows.net"
 cloudstorage.write_base_path=["https://sunbirddev.blob.core.windows.net","https://obj.dev.sunbird.org"]

cokreat-user-delete/src/test/scala/org/sunbird/job/userdelete/fixture/EventFixture.scala

@@ -9,7 +9,7 @@ object EventFixture {
 
   val EVENT_2: String =
     """
-      |{"eid":"BE_JOB_REQUEST","ets":1617174912468,"mid":"LP.1617174912468.0ef201f6-4a42-4053-9f5e-cdb89470b664","actor":{"id":"Auto Creator","type":"System"},"context":{"pdata":{"id":"org.sunbird.platform","ver":"1.0","env":"dev"},"channel":"01309282781705830427"},"object":{"id":"do_113244425048121344131","ver":"1616751174274"},"edata":{"action":"auto-create","originData":{},"iteration":1,"metadata":{"unitIdentifiers":["do_1132443997036380161925"],"organisationId":"13495698-a117-460b-920c-41007923c764","previewUrl":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/do_113244425048121344131_html_1616751462170.html","keywords":["added1"],"subject":["Mathematics"],"channel":"01309282781705830427","downloadUrl":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462043_do_113244425048121344131_1_SPINE.ecar","language":["English"],"mimeType":"application/vnd.ekstep.html-archive","showHints":"No","variants":{"spine":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462043_do_113244425048121344131_1_SPINE.ecar","online":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462123_do_113244425048121344131_1_ONLINE.ecar"},"objectType":"QuestionSet","gradeLevel":["Class 1"],"primaryCategory":"Practice Question Set","contentEncoding":"gzip","showSolutions":"No","identifier":"do_113244425048121344131","audience":["Teacher"],"visibility":"Default","showTimer":"Yes","author":"vaishali","consumerId":"fa13b438-8a3d-41b1-8278-33b0c50210e4","childNodes":["do_113244425563799552132"],"languageCode":["en"],"version":1,"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-03-26T09:37:42.028+0000","name":"LearningHTML","status":"Live","code":"9ae33d1e-a682-f30c-04b5-9bda236650ac","allowSkip":"Yes","containsUserData":"No","description":"added1","medium":["Hindi"],"createdOn":"2021-03-26T09:31:44.508+0000","pdfUrl":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/do_113244425048121344131_pdf_1616751462170.pdf","contentDisposition":"inline","additionalCategories":["Lesson Plan"],"lastUpdatedOn":"2021-03-26T09:37:42.575+0000","collectionId":"do_1132443997030891521922","allowAnonymousAccess":"Yes","lastStatusChangedOn":"2021-03-26T09:37:42.575+0000","creator":"lily21","requiresSubmit":"No","se_FWIds":["ekstep_ncert_k-12"],"setType":"materialised","pkgVersion":1,"versionKey":"1616751174274","showFeedback":"No","framework":"ekstep_ncert_k-12","depth":0,"createdBy":"d31960d0-613f-4f5d-803d-9354a7bc056d","compatibilityLevel":5,"navigationMode":"non-linear","timeLimits":{"maxTime":"3600"},"shuffle":true,"board":"CBSE","programId":"cb59d300-8e0e-11eb-8137-bd637a1d2ab0","contentType":"PreviousBoardExamPapers","lastPublishedBy":"5a587cc1-e018-4859-a0a8-e842650b9d64","source":"https://dock.sunbirded.org/api/questionset/v1/read/do_113244425048121344131","processId":"18350db4-a135-4f28-be64-086398980a3b"},"repository":"https://dock.sunbirded.org/api/questionset/v1/read/do_113244425048121344131","collection":[{"identifier":"do_11324642736155033614","unitId":"do_11324642761348710417"}],"objectType":"QuestionSet","stage":""}}
+      |{"eid":"BE_JOB_REQUEST","ets":1617174912468,"mid":"LP.1617174912468.0ef201f6-4a42-4053-9f5e-cdb89470b664","actor":{"id":"User Delete","type":"System"},"context":{"pdata":{"id":"org.sunbird.platform","ver":"1.0","env":"dev"},"channel":"01309282781705830427"},"object":{"id":"do_113244425048121344131","ver":"1616751174274"},"edata":{"action":"delete-user","userId": "delete-user", "originData":{},"iteration":1,"metadata":{"unitIdentifiers":["do_1132443997036380161925"],"organisationId":"13495698-a117-460b-920c-41007923c764","previewUrl":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/do_113244425048121344131_html_1616751462170.html","keywords":["added1"],"subject":["Mathematics"],"channel":"01309282781705830427","downloadUrl":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462043_do_113244425048121344131_1_SPINE.ecar","language":["English"],"mimeType":"application/vnd.ekstep.html-archive","showHints":"No","variants":{"spine":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462043_do_113244425048121344131_1_SPINE.ecar","online":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462123_do_113244425048121344131_1_ONLINE.ecar"},"objectType":"QuestionSet","gradeLevel":["Class 1"],"primaryCategory":"Practice Question Set","contentEncoding":"gzip","showSolutions":"No","identifier":"do_113244425048121344131","audience":["Teacher"],"visibility":"Default","showTimer":"Yes","author":"vaishali","consumerId":"fa13b438-8a3d-41b1-8278-33b0c50210e4","childNodes":["do_113244425563799552132"],"languageCode":["en"],"version":1,"license":"CC BY 4.0","prevState":"Draft","lastPublishedOn":"2021-03-26T09:37:42.028+0000","name":"LearningHTML","status":"Live","code":"9ae33d1e-a682-f30c-04b5-9bda236650ac","allowSkip":"Yes","containsUserData":"No","description":"added1","medium":["Hindi"],"createdOn":"2021-03-26T09:31:44.508+0000","pdfUrl":"https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/do_113244425048121344131_pdf_1616751462170.pdf","contentDisposition":"inline","additionalCategories":["Lesson Plan"],"lastUpdatedOn":"2021-03-26T09:37:42.575+0000","collectionId":"do_1132443997030891521922","allowAnonymousAccess":"Yes","lastStatusChangedOn":"2021-03-26T09:37:42.575+0000","creator":"lily21","requiresSubmit":"No","se_FWIds":["ekstep_ncert_k-12"],"setType":"materialised","pkgVersion":1,"versionKey":"1616751174274","showFeedback":"No","framework":"ekstep_ncert_k-12","depth":0,"createdBy":"d31960d0-613f-4f5d-803d-9354a7bc056d","compatibilityLevel":5,"navigationMode":"non-linear","timeLimits":{"maxTime":"3600"},"shuffle":true,"board":"CBSE","programId":"cb59d300-8e0e-11eb-8137-bd637a1d2ab0","contentType":"PreviousBoardExamPapers","lastPublishedBy":"5a587cc1-e018-4859-a0a8-e842650b9d64","source":"https://dock.sunbirded.org/api/questionset/v1/read/do_113244425048121344131","processId":"18350db4-a135-4f28-be64-086398980a3b"},"repository":"https://dock.sunbirded.org/api/questionset/v1/read/do_113244425048121344131","collection":[{"identifier":"do_11324642736155033614","unitId":"do_11324642761348710417"}],"objectType":"QuestionSet","stage":""}}
       |""".stripMargin
 
 }

cokreat-user-delete/src/test/scala/org/sunbird/job/userdelete/spec/UserDeleteTaskTestSpec.scala

@@ -48,10 +48,8 @@ class UserDeleteTaskTestSpec extends BaseTestSpec {
   "event" should " process the input map and return metadata " in {
     val event = new Event(JSONUtil.deserialize[util.Map[String, Any]](EventFixture.EVENT_2), 0, 1)
     event.isValid should be(true)
-    event.action should be("auto-create")
-    //event.objectId should be("do_113244425048121344131")
-    event.eData.size should be(8)
-    //event.metadata.size should be(63)
+    event.action should be("delete-user")
+    event.eData.size should be(9)
   }
 
    ignore should "generate event" in {

jobs-distribution/pom.xml

@@ -26,6 +26,12 @@
             <version>1.0.0</version>
             <type>jar</type>
         </dependency>
+        <dependency>
+            <groupId>org.sunbird</groupId>
+            <artifactId>metrics-data-transformer</artifactId>
+            <version>1.0.0</version>
+            <type>jar</type>
+        </dependency>
     </dependencies>
 
     <build>

kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml

@@ -46,6 +46,13 @@ flink_job_names:
     taskmanager_memory: 2048m
     taskslots: 1
     cpu_requests: 0.3
+  metrics-data-transformer:
+    job_class_name: 'org.sunbird.job.metricstransformer.task.MetricsDataTransformerStreamTask'
+    replica: 1
+    jobmanager_memory: 2048m
+    taskmanager_memory: 2048m
+    taskslots: 1
+    cpu_requests: 0.3
 
 ### Global vars
 cloud_store: azure
@@ -58,3 +65,15 @@ flink_libraries_log_level: ERROR
 
 user_delete_kafka_topic_name: "{{ sunbird_instance }}.delete.user"
 user_delete_group: "{{ sunbird_instance }}_cokreat_user_delete_group"
+
+program_service_baseurl: "http://{{ private_ingressgateway_ip }}/program"
+
+metrics_data_transformer_kafka_topic_name: "{{ sunbird_instance }}.learning.graph.events"
+metrics_data_transformer_group: "{{ sunbird_instance }}-metrics-data-transformer-group"
+
+metrics_data_transformer_consumer_parallelism: 1
+metrics_data_transformer_parallelism: 1
+metrics_data_transformer_producer_parallelism: 1
+
+ed_content_service_base_url: "http://{{ private_ingressgateway_ip }}/content"
+dock_content_service_base_url: "http://{{ dock_private_ingressgateway_ip }}/content"

kubernetes/helm_charts/datapipeline_jobs/values.j2

@@ -147,12 +147,42 @@ cokreat-user-delete:
       parallelism = 1
       router.parallelism = 1
     }
-    program-service-baseUrl="{{ program_service_baseurl}}"
+    program-service-baseUrl="{{ program_service_baseurl }}"
 
   flink-conf: |+
     jobmanager.memory.flink.size: {{ flink_job_names['cokreat-user-delete'].jobmanager_memory }}
     taskmanager.memory.flink.size: {{ flink_job_names['cokreat-user-delete'].taskmanager_memory }}
     taskmanager.numberOfTaskSlots: {{ flink_job_names['cokreat-user-delete'].taskslots }}
     parallelism.default: 1
     jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+
+metrics-data-transformer:
+  metrics-data-transformer: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.topic = "{{ metrics_data_transformer_kafka_topic_name }}"
+      groupId = "{{ metrics_data_transformer_group }}"
+    }
+
+    task {
+      consumer.parallelism = {{ metrics_data_transformer_consumer_parallelism }}
+      parallelism = {{ metrics_data_transformer_parallelism }}
+      producer.parallelism = {{ metrics_data_transformer_producer_parallelism }}
+    }
+    content_read_api = "/content/v3/read"
+    content_update_api = "/content/v4/system/update"
+    service {
+      content.basePath = "{{ ed_content_service_base_url }}"
+      sourcing.content.basePath = "{{ dock_content_service_base_url }}"
+    }
+    sourcing.update.api.response.error.code = ["404", "400"]
+    data.metrics = ["me_totalRatingsCount","me_averageRating","me_totalTimeSpentInSec","me_totalPlaySessionCount", "name", "description"]
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['metrics-data-transformer'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['metrics-data-transformer'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['metrics-data-transformer'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
     taskmanager.memory.network.fraction: 0.1