Working on converting to a standalone app.

o19s · Jan 1, 2025 · 5b7888b · 5b7888b
1 parent 2899bcf
commit 5b7888b
Show file tree

Hide file tree

Showing 13 changed files with 271 additions and 83 deletions.
diff --git a/data/esci/index.sh b/data/esci/index.sh
@@ -7,4 +7,4 @@ echo "Initializing UBI..."
 curl -s -X POST "http://localhost:9200/_plugins/ubi/initialize"
 
 echo "Indexing queries and events..."
-curl -s -T "http://localhost:9200/_bulk?pretty" -H "Content-Type: application/x-ndjson" --data-binary @ubi_queries_events.ndjson
+curl  -X POST 'http://localhost:9200/index-name/_bulk?pretty' --data-binary @ubi_queries_events.ndjson -H "Content-Type: application/x-ndjson"
diff --git a/opensearch-search-quality-evaluation-framework/Dockerfile b/opensearch-search-quality-evaluation-framework/Dockerfile
@@ -0,0 +1,3 @@
+FROM opensearchproject/opensearch:2.18.0
+
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.18.0.2/opensearch-ubi-2.18.0.2.zip
diff --git a/opensearch-search-quality-evaluation-framework/docker-compose.yaml b/opensearch-search-quality-evaluation-framework/docker-compose.yaml
@@ -0,0 +1,47 @@
+services:
+
+  opensearch_sef:
+    build: .
+    container_name: opensearch_sef
+    environment:
+      discovery.type: single-node
+      node.name: opensearch
+      plugins.security.disabled: "true"
+      logger.level: info
+      OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123
+      http.max_content_length: 500mb
+      OPENSEARCH_JAVA_OPTS: "-Xms16g -Xmx16g"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    ports:
+      - "9200:9200"
+      - "9600:9600"
+    networks:
+      - opensearch-net
+    volumes:
+      - opensearch-data1:/usr/share/opensearch/data
+
+  opensearch_sef_dashboards:
+    image: opensearchproject/opensearch-dashboards:2.18.0
+    container_name: opensearch_sef_dashboards
+    ports:
+      - "5601:5601"
+    environment:
+      OPENSEARCH_HOSTS: '["http://opensearch_sef:9200"]'
+      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
+    depends_on:
+      - opensearch_sef
+    networks:
+      - opensearch-net
+
+volumes:
+  opensearch-data1:
+
+networks:
+  opensearch-net:
+    driver: bridge
diff --git a/opensearch-search-quality-evaluation-framework/pom.xml b/opensearch-search-quality-evaluation-framework/pom.xml
@@ -1,62 +1,89 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <groupId>org.opensearch</groupId>
-  <artifactId>search-evaluation-framework</artifactId>
-  <version>1.0.0-SNAPSHOT</version>
-  <name>search-evaluation-framework</name>
-  <url>https://www.ubisearch.dev</url>
-  <properties>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <maven.compiler.release>21</maven.compiler.release>
-  </properties>
-  <dependencies>
-    <dependency>
-      <groupId>org.opensearch.client</groupId>
-      <artifactId>opensearch-java</artifactId>
-      <version>2.19.0</version>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-databind</artifactId>
-      <version>2.18.2</version>
-    </dependency>
-    <dependency>
-      <groupId>commons-cli</groupId>
-      <artifactId>commons-cli</artifactId>
-      <version>1.9.0</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-core</artifactId>
-      <version>2.24.3</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.httpcomponents.core5</groupId>
-      <artifactId>httpcore5</artifactId>
-      <version>5.3.1</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.httpcomponents.client5</groupId>
-      <artifactId>httpclient5</artifactId>
-      <version>5.4.1</version>
-    </dependency>
-    <dependency>
-      <groupId>commons-logging</groupId>
-      <artifactId>commons-logging</artifactId>
-      <version>1.3.4</version>
-    </dependency>
-    <dependency>
-      <groupId>com.google.code.gson</groupId>
-      <artifactId>gson</artifactId>
-      <version>2.11.0</version>
-    </dependency>
-    <dependency>
-      <groupId>org.junit.jupiter</groupId>
-      <artifactId>junit-jupiter-api</artifactId>
-      <version>5.11.4</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.opensearch</groupId>
+    <artifactId>search-evaluation-framework</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <name>search-evaluation-framework</name>
+    <url>https://www.ubisearch.dev</url>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.release>21</maven.compiler.release>
+    </properties>
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <mainClass>org.opensearch.eval.SearchQualityEvaluationFrameworkApp</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-my-jar-with-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+    <dependencies>
+        <dependency>
+            <groupId>org.opensearch.client</groupId>
+            <artifactId>opensearch-java</artifactId>
+            <version>2.19.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.18.2</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>2.24.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents.core5</groupId>
+            <artifactId>httpcore5</artifactId>
+            <version>5.3.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents.client5</groupId>
+            <artifactId>httpclient5</artifactId>
+            <version>5.4.1</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>1.3.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.11.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>5.11.4</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>
diff --git a/opensearch-search-quality-evaluation-framework/run.sh b/opensearch-search-quality-evaluation-framework/run.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -e
+
+# Create a click model.
+java -jar ./target/search-evaluation-framework-1.0.0-SNAPSHOT-jar-with-dependencies.jar -c
diff --git a/...luation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationFramework.java b/...luation-framework/src/main/java/org/opensearch/eval/SearchQualityEvaluationFramework.java
diff --git a/...arch/eval/SearchQualityEvaluationApp.java → .../SearchQualityEvaluationFrameworkApp.java b/...arch/eval/SearchQualityEvaluationApp.java → .../SearchQualityEvaluationFrameworkApp.java
@@ -15,23 +15,38 @@
 import org.apache.commons.cli.ParseException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.eval.engine.OpenSearchEngine;
+import org.opensearch.eval.engine.SearchEngine;
+import org.opensearch.eval.judgments.clickmodel.ClickModel;
+import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel;
+import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters;
 
-public class SearchQualityEvaluationApp {
+public class SearchQualityEvaluationFrameworkApp {
 
-    private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationApp.class);
+    private static final Logger LOGGER = LogManager.getLogger(SearchQualityEvaluationFrameworkApp.class);
 
-    public static void main(String args[]) throws ParseException {
+    public static void main(String[] args) throws Exception {
+
+        System.out.println("Search Quality Evaluation Framework");
 
         final Options options = new Options();
-        options.addOption("c", true, "create a click model");
+        options.addOption("c", false, "create a click model");
         options.addOption("q", true, "run a query set");
 
         final CommandLineParser parser = new DefaultParser();
         final CommandLine cmd = parser.parse(options, args);
 
         if(cmd.hasOption("c")) {
-            final String clickModel = cmd.getOptionValue("c");
-            // TODO: Create the click model.
+
+            //final String clickModel = cmd.getOptionValue("c");
+            System.out.println("Creating click model...");
+
+            final SearchEngine searchEngine = new OpenSearchEngine();
+            final CoecClickModelParameters coecClickModelParameters = new CoecClickModelParameters(10);
+
+            final ClickModel cm = new CoecClickModel(searchEngine, coecClickModelParameters);
+            cm.calculateJudgments();
+
         } else {
 
         }

diff --git a/...ality-evaluation-framework/src/main/java/org/opensearch/eval/engine/OpenSearchEngine.java b/...ality-evaluation-framework/src/main/java/org/opensearch/eval/engine/OpenSearchEngine.java
@@ -9,7 +9,6 @@
 package org.opensearch.eval.engine;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.gson.Gson;
 import org.apache.hc.core5.http.HttpHost;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -86,13 +85,13 @@ public class OpenSearchEngine extends SearchEngine {
     private static final Logger LOGGER = LogManager.getLogger(OpenSearchEngine.class.getName());
 
     private final OpenSearchClient client;
-    private final Gson gson = new Gson();
 
     // Used to cache the query ID->user_query to avoid unnecessary lookups to OpenSearch.
     private static final Map<String, String> userQueryCache = new HashMap<>();
 
     public OpenSearchEngine() {
 
+        // TODO: Parameterize the host.
         final HttpHost[] hosts = new HttpHost[] {
                 new HttpHost("http", "localhost", 9200)
         };
@@ -207,7 +206,9 @@ public Collection<UbiQuery> getUbiQueries() throws IOException {
 
         final Collection<UbiQuery> ubiQueries = new ArrayList<>();
 
-        final SearchResponse<UbiQuery> searchResponse = client.search(s -> s.index(Constants.UBI_QUERIES_INDEX_NAME).size(1000).scroll(Time.of(t -> t.offset(1000))), UbiQuery.class);
+        final Time scrollTime = new Time.Builder().time("10m").build();
+
+        final SearchResponse<UbiQuery> searchResponse = client.search(s -> s.index(Constants.UBI_QUERIES_INDEX_NAME).size(1000).scroll(scrollTime), UbiQuery.class);
 
         String scrollId = searchResponse.scrollId();
         List<Hit<UbiQuery>> searchHits = searchResponse.hits().hits();
@@ -234,7 +235,9 @@ public Collection<Judgment> getJudgments(final String index) throws IOException
 
         final Collection<Judgment> judgments = new ArrayList<>();
 
-        final SearchResponse<Judgment> searchResponse = client.search(s -> s.index(index).size(1000).scroll(Time.of(t -> t.offset(1000))), Judgment.class);
+        final Time scrollTime = new Time.Builder().time("10m").build();
+
+        final SearchResponse<Judgment> searchResponse = client.search(s -> s.index(index).size(1000).scroll(scrollTime), Judgment.class);
 
         String scrollId = searchResponse.scrollId();
         List<Hit<Judgment>> searchHits = searchResponse.hits().hits();
@@ -431,12 +434,14 @@ public Map<String, Set<ClickthroughRate>> getClickthroughRate(final int maxRank)
                 .query(encodedQuery)
                 .build();
 
+        final Time scrollTime = new Time.Builder().time("10m").build();
+
         final SearchRequest searchRequest = new SearchRequest.Builder()
                 .index(Constants.UBI_EVENTS_INDEX_NAME)
                 .query(q -> q.wrapper(wrapperQuery))
                 .from(0)
                 .size(1000)
-                .scroll(Time.of(t -> t.offset(1000)))
+                .scroll(scrollTime)
                 .build();
 
         final SearchResponse<UbiEvent> searchResponse = client.search(searchRequest, UbiEvent.class);
@@ -517,7 +522,7 @@ public Map<Integer, Double> getRankAggregatedClickThrough(final int maxRank) thr
         final Aggregation positionsAggregator = Aggregation.of(a -> a
                 .terms(t -> t
                         .field("event_attributes.position.ordinal")
-                        .name("By_Position")
+                        //.name("By_Position")
                         .size(maxRank)
                         .order(sort)
                 )
@@ -526,7 +531,7 @@ public Map<Integer, Double> getRankAggregatedClickThrough(final int maxRank) thr
         final Aggregation actionNameAggregation = Aggregation.of(a -> a
                 .terms(t -> t
                         .field("action_name")
-                        .name("By_Action")
+                        //.name("By_Action")
                         .size(maxRank)
                         .order(sort)
                 )
@@ -545,6 +550,8 @@ public Map<Integer, Double> getRankAggregatedClickThrough(final int maxRank) thr
                 .size(0)
                 .build();
 
+        System.out.println(searchRequest.toJsonString());
+
         final SearchResponse<Void> searchResponse = client.search(searchRequest, Void.class);
 
         final Map<String, Aggregate> aggs = searchResponse.aggregations();

diff --git a/...aluation-framework/src/main/java/org/opensearch/eval/model/ubi/event/EventAttributes.java b/...aluation-framework/src/main/java/org/opensearch/eval/model/ubi/event/EventAttributes.java
@@ -8,6 +8,7 @@
  */
 package org.opensearch.eval.model.ubi.event;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.gson.annotations.SerializedName;
 
 /**
@@ -18,6 +19,7 @@ public class EventAttributes {
     @SerializedName("object")
     private EventObject object;
 
+    @JsonProperty("session_id")
     @SerializedName("session_id")
     private String sessionId;
 

diff --git a/...y-evaluation-framework/src/main/java/org/opensearch/eval/model/ubi/event/EventObject.java b/...y-evaluation-framework/src/main/java/org/opensearch/eval/model/ubi/event/EventObject.java
@@ -8,13 +8,16 @@
  */
 package org.opensearch.eval.model.ubi.event;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.gson.annotations.SerializedName;
 
 public class EventObject {
 
+    @JsonProperty("object_id_field")
     @SerializedName("object_id_field")
     private String objectIdField;
 
+    @JsonProperty("object_id")
     @SerializedName("object_id")
     private String objectId;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		FROM opensearchproject/opensearch:2.18.0

		RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.18.0.2/opensearch-ubi-2.18.0.2.zip