Assorted jinjava template changes for replayer...

Switched out Google's re2j for Java's implementation. I didn't realize that the java implementation is closer to python regexes and that re2j (which I had thought was closer to python). Minor tweaks to the route jinja template to make it easier to read when it's invoked. Switched the replayer template to use the route functionality, which is considerably easier to get one's head around. TODO - * I'd like to make the feature flag names the same as the macro name for the route when the feature flag name is omitted. * The test cases for the type mappings sanitization need a lot of work. * When indices are created that DO NOT use type mappings, we should handle those more gracefully. Signed-off-by: Greg Schohn <[email protected]>
opensearch-project · Nov 24, 2024 · b1f3756 · b1f3756
1 parent e8dfb3f
commit b1f3756
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 73 deletions.
diff --git a/...rmation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/build.gradle b/...rmation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/build.gradle
@@ -6,7 +6,6 @@ dependencies {
     implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface')
 
     implementation group: 'com.hubspot.jinjava', name: 'jinjava', version: "2.7.3"
-    implementation group: 'com.google.re2j', name: 're2j', version: "1.7"
 
     testImplementation project(':TrafficCapture:trafficReplayer')
     testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders')

diff --git a/...sformer/src/main/java/org/opensearch/migrations/transform/jinjava/RegexCaptureFilter.java b/...sformer/src/main/java/org/opensearch/migrations/transform/jinjava/RegexCaptureFilter.java
@@ -1,9 +1,9 @@
 package org.opensearch.migrations.transform.jinjava;
 
 import java.util.HashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
-import com.google.re2j.Matcher;
-import com.google.re2j.Pattern;
 import com.hubspot.jinjava.interpret.JinjavaInterpreter;
 import com.hubspot.jinjava.lib.filter.Filter;
 

diff --git a/...MessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/route-bkp.j2 b/...MessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/route-bkp.j2
diff --git a/...jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/route.j2 b/...jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/route.j2
@@ -1,7 +1,8 @@
 {% import "common/featureEnabled.j2" as fscope %}
 {% macro route(input, field_to_match, feature_flags, default_action, routes) %}
     {%- set ns = namespace(result=none, matched=false) -%}
-    {%- for feature_name, pattern, action_fn in routes if not ns.matched -%}
+    {%- for pattern, action_fn, feature_name_param in routes if not ns.matched -%}
+        {% set feature_name = feature_name_param | default(action_fn) %}
         {%- if not ns.matched -%} {# we haven't found a match yet, otherwise skip the rest #}
             {%- set match = field_to_match | regex_capture(pattern) -%}
             {%- if match is not none -%}

diff --git a/...s/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java b/...s/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java
@@ -19,7 +19,7 @@ public class RouteTest {
     public Map<String, Object> doRouting(Map<String, Object> flags, Map<String, Object> inputDoc) {
         log.atInfo().setMessage("parsed flags: {}").addArgument(flags).log();
         final var template = "" +
-            "{%- macro doDefault() -%}" +
+            "{%- macro doDefault(ignored_input) -%}" +
             "  {}" +
             "{%- endmacro -%}\n" +
 
@@ -38,9 +38,9 @@ public Map<String, Object> doRouting(Map<String, Object> flags, Map<String, Obje
             "{%- import \"common/route.j2\" as rscope -%}" +
             "{{- rscope.route(source, source.label, flags, 'doDefault'," +
             "  [" +
-            "    ('matchA', 'Thing_A(.*)', 'echoFirstMatch')," +
-            "    ('matchA', 'Thing_A(.*)', 'echoFirstMatchAgain')," + // make sure that we don't get duplicate results
-            "    ('matchB', 'B(.*)', 'switchStuff')" +
+            "    ('Thing_A(.*)',        'echoFirstMatch',      'matchA')," +
+            "    ('Thing_A(.*)', 'echoFirstMatchAgain', 'matchA')," + // make sure that we don't get duplicate results
+            "    ('B(.*)',       'switchStuff',         'matchB')" +
             "  ])" +
             "-}}";
 
@@ -59,15 +59,13 @@ public void test() throws IOException {
             "stuff", Map.of(
                 "inner1", "data1",
                 "inner2", "data2"
-            )
-        );
+            ));
         var docB = Map.of(
             "label", "B-hive",
             "stuff", List.of(
                 "data1",
                 "data2"
-            )
-        );
+            ));
         {
             var resultMap = doRouting(null, docA);
             Assertions.assertEquals(1, resultMap.size());

diff --git a/...onTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/replayer.j2 b/...onTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/replayer.j2
@@ -1,32 +1,33 @@
 {%- include "common/featureEnabled.j2" -%}
 
-{%- macro handle_else(uri_parts) -%}
+{%- macro default_action(source_and_mappings) -%}
 {
+    "diagPair": {{ source_and_mappings }},
     "preserve": "*"
 }
 {%- endmacro -%}
 
-{%- macro rewrite_uri_to_strip_types(uri_parts, index_mappings, request) -%}
+{%- macro rewrite_uri_to_strip_types(match, input_map) -%}
 {
-  "method": "{{ request.method }}",
-  "URI": "/{{ index_mappings[uri_parts[0]][uri_parts[1]] }}/_doc/{{ uri_parts[2] }}",
+  "method": "{{ input_map.request.method }}",
+  "URI": "/{{ input_map.index_mappings[match.group1][match.group2] }}/_doc/{{ match.group3 }}",
   "preserve": ["headers","payload"]
 }
 {%- endmacro -%}
 
-
-{% macro rewrite_typemapping_to_target_index(uri_parts, index_mappings, request) -%}
+{% macro rewrite_create_index(match, input_map) -%}
+{# In reality, this is going to need to figure out if there are multiple type mappings and deal with them according.y #}
 {
-  "method": "{{ request.method }}",
-  "URI": "{{ index_mappings[uri_parts[0]][uri_parts[1]] }}",
+  "method": "{{ input_map.request.method }}",
+  "URI": "{{ input_map.index_mappings[match.group1] }}",
   "payload": {
     "inlinedJsonBody": {
       "mappings": {
         "properties": {
           "type": {
             "type": "keyword"
           }
-          {%- for type_name, type_props in request.body.mappings.items() -%}
+          {%- for type_name, type_props in input_map.request.body.mappings.items() -%}
             {%- for prop_name, prop_def in type_props.properties.items() -%}
               ,
               "{{- prop_name -}}": {{- prop_def | tojson -}}
@@ -39,19 +40,12 @@
 }
 {%- endmacro -%}
 
-
-{%- set uri_parts = request.URI.split('/') | reject('equalto', '') | list -%}
-
-{%- if uri_parts | length == 3 and uri_parts[0] in index_mappings and uri_parts[1] in index_mappings[uri_parts[0]] -%}
-  {%- if is_enabled(features, 'rewrite_uri_to_strip_types') -%}
-    {{ rewrite_uri_to_strip_types(uri_parts, index_mappings, request) -}}
-  {%- endif -%}
-{%- elif uri_parts | length == 2 and uri_parts[0] in index_mappings -%}
-  {%- if is_enabled(features, 'rewrite_typemapping_to_target_index') -%}
-    {{ rewrite_typemapping_to_target_index(uri_parts, index_mappings, request) -}}
-  {%- endif -%}
-{%- else -%}
-  {%- if is_enabled(features, 'handle_else') -%}
-    {{- handle_else(uri_parts) -}}
-  {%- endif -%}
-{%- endif -%}
+{% set source_and_mappings = { 'request': request, 'index_mappings': index_mappings} %}
+{%- import "common/route.j2" as rscope -%}
+{{- rscope.route(source_and_mappings, request.method + " " + request.URI, flags, 'default_action',
+    [
+        ('(?:PUT|POST) /([^/]*)/([^/]*)/.*', 'rewrite_uri_to_strip_types', 'rewrite_add_request_to_strip_types'),
+        (         'GET /([^/]*)/([^/]*)/.*', 'rewrite_uri_to_strip_types', 'rewrite_get_request_to_strip_types'),
+        ('(?:PUT|POST) /([^/]*)',            'rewrite_create_index', 'rewrite_create_index')
+  ])
+-}}
diff --git a/...est/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerTest.java b/...est/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerTest.java
@@ -1,7 +1,9 @@
 package org.opensearch.migrations.transform;
 
 import java.io.IOException;
+import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -21,23 +23,61 @@ static void initialize() throws IOException {
                 "type2", "indexB"),
             "indexC", Map.of(
                 "type2", "indexC"));
+            //"time-(*)", Map.of("(*)", "time-\\1-\\2"));
         indexTypeMappingRewriter = new TypeMappingsSanitizationTransformer(indexMappings);
     }
 
     @Test
-    public void test() throws Exception {
+    public void testPutDoc() throws Exception {
         var testString =
-        "{\n" +
-            "  \"verb\": \"PUT\",\n" +
-            "  \"uri\": \"/indexA/type2/someuser\",\n" +
-            "  \"body\": {\n" +
-            "    \"name\": \"Some User\",\n" +
-            "    \"user_name\": \"user\",\n" +
-            "    \"email\": \"[email protected]\"\n" +
-            "  }\n" +
-            "}";
+            "{\n" +
+                "  \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" +
+                "  \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/indexA/type2/someuser\",\n" +
+                "  \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" +
+                "    \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\": {" +
+                "      \"name\": \"Some User\",\n" +
+                "      \"user_name\": \"user\",\n" +
+                "      \"email\": \"[email protected]\"\n" +
+                "    }\n" +
+                "  }\n" +
+                "}";
         var objMapper = new ObjectMapper();
-        var resultObj = indexTypeMappingRewriter.transformJson(objMapper.readValue(testString, Map.class));
+        var resultObj = indexTypeMappingRewriter.transformJson(objMapper.readValue(testString, LinkedHashMap.class));
+        var resultStr = objMapper.writeValueAsString(resultObj);
+        System.out.println("resultStr = " + resultStr);
+    }
+
+    @Test
+    public void testPutIndex() throws Exception {
+        var testString =
+            "{\n" +
+                "  \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" +
+                "  \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/indexA\",\n" +
+                "  \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" +
+                "    \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\": " +
+                "{\n" +
+                "  \"mappings\": {\n" +
+                "    \"user\": {\n" +
+                "      \"properties\": {\n" +
+                "        \"name\": { \"type\": \"text\" },\n" +
+                "        \"user_name\": { \"type\": \"keyword\" },\n" +
+                "        \"email\": { \"type\": \"keyword\" }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"tweet\": {\n" +
+                "      \"properties\": {\n" +
+                "        \"content\": { \"type\": \"text\" },\n" +
+                "        \"user_name\": { \"type\": \"keyword\" },\n" +
+                "        \"tweeted_at\": { \"type\": \"date\" }\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }\n" +
+                "}" +
+                "\n" +
+                "  }\n" +
+                "}";
+        var objMapper = new ObjectMapper();
+        var resultObj = indexTypeMappingRewriter.transformJson(objMapper.readValue(testString, LinkedHashMap.class));
         var resultStr = objMapper.writeValueAsString(resultObj);
         System.out.println("resultStr = " + resultStr);
     }