TP: Enable user guidance on GenAI recs, add ES 6.8 and 7.10 knowledge

Signed-off-by: Chris Helma <[email protected]>

TransformationPlayground/README.md

@@ -78,6 +78,67 @@ To run the Frontend, first start the Backend and ensure it's running.  Then, exe
 
 You should then be able to hit the Playground website in your web browser at `http://localhost:3000`.  The GUI should be pretty self-explanatory.
 
+Here's an example input JSON for ElasticSearch 6.8:
+```json
+{
+    "index_name": "test-index",
+    "index_json": {
+        "settings": {
+            "index": {
+                "number_of_shards": 1,
+                "number_of_replicas": 0
+            }
+        },
+        "mappings": {
+            "type1": {
+                "properties": {
+                    "title": {
+                        "type": "text"
+                    }
+                }
+            },
+            "type2": {
+                "properties": {
+                    "contents": {
+                        "type": "text"
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+
+Here's an example input JSON for ElasticSearch 7.10:
+```json
+{
+    "index_name": "test-index",
+    "index_json": {
+        "settings": {
+            "index": {
+                "soft_deletes": {
+                    "enabled": false
+                },
+                "number_of_shards": 1,
+                "number_of_replicas": 1
+            }
+        },
+        "mappings": {
+            "properties": {
+                "title": {
+                    "type": "text"
+                },
+                "content": {
+                    "type": "text"
+                }
+            }
+        }
+    }
+}
+```
+
+
 ### Dependencies
 `pipenv` is used to managed dependencies within the project.  The `Pipefile` and `Pipefile.lock` handle the local environment.  You can add dependencies like so:
 

TransformationPlayground/playground/playground/urls.py

@@ -1,12 +1,12 @@
 from django.urls import path
 from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView
 
-from transform_api.views import TransformsIndexView, TransformsIndexTestView
+from transform_api.views import TransformsIndexCreateView, TransformsIndexTestView
 
 
 urlpatterns = [
     path('api/schema/', SpectacularAPIView.as_view(), name='schema'),
     path('api/docs/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'),
-    path('transforms/index/', TransformsIndexView.as_view(), name='transforms_index'),
+    path('transforms/index/create/', TransformsIndexCreateView.as_view(), name='transforms_index_create'),
     path('transforms/index/test/', TransformsIndexTestView.as_view(), name='transforms_index_test'),
 ]

TransformationPlayground/playground/schema.json

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: Transformation API
-  version: 0.0.1
+  version: 0.1.0
   description: API for JSON transformation logic.
 paths:
   /api/schema/:
@@ -150,9 +150,9 @@ paths:
                 type: object
                 additionalProperties: {}
           description: ''
-  /transforms/index/:
+  /transforms/index/create/:
     post:
-      operationId: transforms_index_create
+      operationId: transforms_index_create_create
       tags:
       - transforms
       requestBody:
@@ -211,8 +211,11 @@ components:
     SourceVersionEnum:
       enum:
       - Elasticsearch 6.8
+      - Elasticsearch 7.10
       type: string
-      description: '* `Elasticsearch 6.8` - ES_6_8'
+      description: |-
+        * `Elasticsearch 6.8` - ES_6_8
+        * `Elasticsearch 7.10` - ES_7_10
     TargetVersionEnum:
       enum:
       - OpenSearch 2.17
@@ -226,12 +229,12 @@ components:
     TransformsIndexCreateRequest:
       type: object
       properties:
-        transform_language:
-          $ref: '#/components/schemas/TransformLanguageEnum'
         source_version:
           $ref: '#/components/schemas/SourceVersionEnum'
         target_version:
           $ref: '#/components/schemas/TargetVersionEnum'
+        transform_language:
+          $ref: '#/components/schemas/TransformLanguageEnum'
         input_shape:
           type: object
           properties:
@@ -242,6 +245,10 @@ components:
           required:
           - index_name
           - index_json
+        transform_logic:
+          type: string
+        user_guidance:
+          type: string
         test_target_url:
           type: string
           format: uri

TransformationPlayground/playground/transform_api/serializers.py

@@ -65,10 +65,12 @@ def to_representation(self, value: Dict[str, Any]) -> Dict[str, Any]:
 
 
 class TransformsIndexCreateRequestSerializer(serializers.Serializer):
-    transform_language = EnumChoiceField(enum=TransformLanguage)
     source_version = EnumChoiceField(enum=SourceVersion)
     target_version = EnumChoiceField(enum=TargetVersion)
+    transform_language = EnumChoiceField(enum=TransformLanguage)
     input_shape = IndexShapeField()
+    transform_logic = serializers.CharField(required=False, default=None)
+    user_guidance = serializers.CharField(required=False, default=None)
     test_target_url = serializers.URLField(required=False, default=None)
 
 class TransformsIndexCreateResponseSerializer(serializers.Serializer):

TransformationPlayground/playground/transform_api/tests/test_serializers.py

@@ -107,6 +107,8 @@ def test_valid_input(self):
                     }
                 }
             },
+            "transform_logic": "some transform logic",
+            "user_guidance": "some user guidance",
             "test_target_url": "http://localhost:29200"
         })
         self.assertTrue(test_serializer.is_valid())
@@ -165,7 +167,7 @@ def test_valid_input(self):
                     }
                 }
             ],
-            "transform_logic": "the transform logic",
+            "transform_logic": "the new transform logic",
             "validation_report": [
                 "Entry 1",
                 "Entry 2"

TransformationPlayground/playground/transform_api/views.py

@@ -21,7 +21,7 @@
 logger = logging.getLogger("transform_api")
 
 
-class TransformsIndexView(APIView):
+class TransformsIndexCreateView(APIView):
     @csrf_exempt
     @extend_schema(
         request=TransformsIndexCreateRequestSerializer,
@@ -86,7 +86,8 @@ def _perform_transformation(self, transform_id: str, request: TransformsIndexCre
         )
 
         system_message = expert.system_prompt_factory(
-            request.validated_data["input_shape"]           
+            user_guidance=request.validated_data["user_guidance"],
+            input_shape=request.validated_data["input_shape"]           
         )
         turns = [
             system_message,

TransformationPlayground/playground/transform_expert/expert.py

@@ -60,6 +60,9 @@ def get_expert(source_version: SourceVersion, target_version: TargetVersion, tra
         tools=tool_bundle
     )
 
+class NoTransformCreatedError(Exception):
+    pass
+
 def invoke_expert(expert: Expert, task: TransformTask) -> TransformTask:
     logger.info(f"Invoking the Transform Expert for transform_id: {task.transform_id}")
     logger.debug(f"Initial Transform Task: {str(task.to_json())}")

TransformationPlayground/playground/transform_expert/parameters.py

@@ -6,6 +6,7 @@
 
 class SourceVersion(Enum):
     ES_6_8 = "Elasticsearch 6.8"
+    ES_7_10 = "Elasticsearch 7.10"
 
 class TargetVersion(Enum):
     OS_2_17 = "OpenSearch 2.17"

TransformationPlayground/playground/transform_expert/prompting/generation.py

@@ -17,7 +17,7 @@ def _get_base_template(source_version: SourceVersion, target_version: TargetVers
 def get_system_prompt_factory(source_version: SourceVersion, target_version: TargetVersion, input_shape_type: TransformType, transform_language: TransformLanguage) -> Callable[[Dict[str, Any]], SystemMessage]:
     base_template = _get_base_template(source_version, target_version, input_shape_type, transform_language)
 
-    def factory(input_shape: Dict[str, Any]) -> SystemMessage:
+    def factory(user_guidance: str, input_shape: Dict[str, Any]) -> SystemMessage:
         return SystemMessage(
             content=base_template.format(
                 source_version=source_version,
@@ -26,7 +26,8 @@ def factory(input_shape: Dict[str, Any]) -> SystemMessage:
                 target_version=target_version,
                 target_guidance=get_target_guidance(source_version, target_version, input_shape_type, transform_language),
                 target_knowledge=get_target_knowledge(source_version, target_version, input_shape_type, transform_language),
-                source_json=input_shape
+                source_json=input_shape,
+                user_guidance=user_guidance
             )
         )
 

TransformationPlayground/playground/transform_expert/prompting/knowledge/__init__.py

@@ -1,20 +1,27 @@
 from transform_expert.parameters import SourceVersion, TargetVersion, TransformType, TransformLanguage
 
 from transform_expert.prompting.knowledge.es_6_8 import INDEX_GUIDANCE as es_6_8_index_guidance, INDEX_KNOWLEDGE as es_6_8_index_knowledge
+from transform_expert.prompting.knowledge.es_7_10 import INDEX_GUIDANCE as es_7_10_index_guidance, INDEX_KNOWLEDGE as es_7_10_index_knowledge
 from transform_expert.prompting.knowledge.os_2_17 import INDEX_GUIDANCE as os_2_17_index_guidance, INDEX_KNOWLEDGE as os_2_17_index_knowledge
 
 
 def get_source_guidance(source_version: SourceVersion, target_version: TargetVersion, input_shape_type: TransformType, transform_language: TransformLanguage) -> str:
     if source_version == SourceVersion.ES_6_8:
         if input_shape_type == TransformType.INDEX:
             return es_6_8_index_guidance
+    elif source_version == SourceVersion.ES_7_10:
+        if input_shape_type == TransformType.INDEX:
+            return es_7_10_index_guidance
 
     return ""
 
 def get_source_knowledge(source_version: SourceVersion, target_version: TargetVersion, input_shape_type: TransformType, transform_language: TransformLanguage) -> str:
     if source_version == SourceVersion.ES_6_8:
         if input_shape_type == TransformType.INDEX:
             return es_6_8_index_knowledge
+    elif source_version == SourceVersion.ES_7_10:
+        if input_shape_type == TransformType.INDEX:
+            return es_7_10_index_knowledge
 
     return ""