Merge branch 'linkedin:master' into master

.gitignore

@@ -39,7 +39,17 @@ MANIFEST
 # Mac OS
 **/.DS_Store
 
+#spark-lineage
+**/spark-lineage/metastore_db/
+**/spark-lineage/**/derby.log
+**/spark-lineage/**/hive/
+**/spark-lineage/**/out.csv/
 .vscode
 
+# cypress integration test generated files
+**/cypress/videos
+**/cypress/screenshots
+**/cypress/node_modules
+
 # Metadata Ingestion Generated
 metadata-ingestion/generated/**

build.gradle

@@ -3,14 +3,14 @@ buildscript {
   ext.gmaVersion = '0.2.81'
   ext.pegasusVersion = '28.3.7'
   ext.mavenVersion = '3.6.3'
-
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
   dependencies {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
     classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.1.4.RELEASE'
+    classpath 'com.github.jengelman.gradle.plugins:shadow:5.2.0'
   }
 }
 
@@ -68,6 +68,8 @@ project.ext.externalDependency = [
     'guava': 'com.google.guava:guava:27.0.1-jre',
     'h2': 'com.h2database:h2:1.4.196',
     'hadoopClient': 'org.apache.hadoop:hadoop-client:3.1.1',
+    'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
+    'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2',
     'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final',
     'httpClient': 'org.apache.httpcomponents:httpclient:4.5.9',
     'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
@@ -114,6 +116,8 @@ project.ext.externalDependency = [
     'rythmEngine': 'org.rythmengine:rythm-engine:1.3.0',
     'servletApi': 'javax.servlet:javax.servlet-api:3.1.0',
     'shiroCore': 'org.apache.shiro:shiro-core:1.7.1',
+    'sparkSql' : 'org.apache.spark:spark-sql_2.11:2.4.8',
+    'sparkHive' : 'org.apache.spark:spark-hive_2.11:2.4.8',
     'springBeans': 'org.springframework:spring-beans:5.2.3.RELEASE',
     'springContext': 'org.springframework:spring-context:5.2.3.RELEASE',
     'springCore': 'org.springframework:spring-core:5.2.3.RELEASE',
@@ -129,7 +133,9 @@ project.ext.externalDependency = [
     'testng': 'org.testng:testng:7.3.0',
     'testContainers': 'org.testcontainers:testcontainers:1.15.1',
     'testContainersJunit': 'org.testcontainers:junit-jupiter:1.15.1',
+    'testContainersPostgresql':'org.testcontainers:postgresql:1.2.0',
     'testContainersElasticsearch': 'org.testcontainers:elasticsearch:1.15.3',
+    'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
     'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14'
 ]
 

docker/elasticsearch/env/docker.env

@@ -1,3 +1,3 @@
 discovery.type=single-node
 xpack.security.enabled=false
-ES_JAVA_OPTS=-Xms256m -Xmx256m
+ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true