feat: pull OpenLinkSW fork changes (#86)

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
berezovskyi · Jan 4, 2025 · 9d5d4a3 · 9d5d4a3
1 parent 596001a
commit 9d5d4a3
Show file tree

Hide file tree

Showing 161 changed files with 40,810 additions and 19,353 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -7,4 +7,4 @@ indent_size = 2
 max_line_length = 100
 
 [*.java]
-indent_size = 4
+indent_size = 2
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,7 @@
 /data/
 
 *.tmp
+*~
 
 # Node
 npm-debug.log

diff --git a/backend/pom.xml b/backend/pom.xml
@@ -6,7 +6,7 @@
   <artifactId>sparqles</artifactId>
   <version>0.0.1-SNAPSHOT</version>
   <properties>
-    <jdkLevel>11</jdkLevel>
+    <jdkLevel>17</jdkLevel>
     <maven.compiler.release>${jdkLevel}</maven.compiler.release>
     <maven.compiler.target>${jdkLevel}</maven.compiler.target>
     <maven.compiler.source>${jdkLevel}</maven.compiler.source>
@@ -55,7 +55,7 @@
                   <version>3.6.0</version>
                 </requireMavenVersion>
                 <requireJavaVersion>
-                  <version>11</version>
+                  <version>17</version>
                 </requireJavaVersion>
                 <bannedDependencies>
                   <excludes>
@@ -177,9 +177,12 @@
           <!-- optional: limit format enforcement to just the files changed by this feature branch -->
           <!-- <ratchetFrom>origin/main</ratchetFrom> -->
           <java>
+            <toggleOffOn>
+              <off>Autogenerated by Avro</off>
+            </toggleOffOn>
             <googleJavaFormat>
               <version>1.24.0</version>
-              <style>AOSP</style>
+              <style>GOOGLE</style>
               <reflowLongStrings>true</reflowLongStrings>
               <formatJavadoc>true</formatJavadoc>
             </googleJavaFormat>
@@ -188,6 +191,23 @@
           </java>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.avro</groupId>
+        <artifactId>avro-maven-plugin</artifactId>
+        <version>${avro.version}</version>
+        <executions>
+          <execution>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>schema</goal>
+            </goals>
+            <configuration>
+              <sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory>
+              <outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
   <dependencyManagement>
@@ -262,7 +282,6 @@
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-log4j12</artifactId>
         </exclusion>
-
       </exclusions>
     </dependency>
     <dependency>

diff --git a/backend/run.sh b/backend/run.sh
@@ -15,29 +15,37 @@ export JAVA_OPTS="-XX:MaxRAMPercentage=80"
 
 # while :
 # do
-	echo "Running SPARQLes full cycle"
+
+   #echo "Running SPARQLes full cycle"
    # interop
-	echo "Running SPARQLes full cycle [ftask]"
-	bin/sparqles $CMDARGS -run ftask
+	#echo "Running SPARQLes full cycle [ftask]"
+	#bin/sparqles $CMDARGS -run ftask
    # # availability
-	# echo "Running SPARQLes full cycle [atask]"
-	# bin/sparqles $CMDARGS -run atask
+	#echo "Running SPARQLes full cycle [atask]"
+	#bin/sparqles $CMDARGS -run atask
+   # coherence
+	#echo "Running SPARQLes full cycle [ctask]"
+	#bin/sparqles $CMDARGS -run ctask
    # # performance
-	echo "Running SPARQLes full cycle [ptask]"
-	bin/sparqles $CMDARGS -run ptask
+	#echo "Running SPARQLes full cycle [ptask]"
+	#bin/sparqles $CMDARGS -run ptask
 	# # discoverability
-   echo "Running SPARQLes full cycle [dtask]"
-	bin/sparqles $CMDARGS -run dtask
-   # index view
-	echo "Running SPARQLes full cycle [iv]"
-	bin/sparqles $CMDARGS -iv
+   #echo "Running SPARQLes full cycle [dtask]"
+	#bin/sparqles $CMDARGS -run dtask
    # stats
-	echo "Running SPARQLes full cycle [st]"
-	bin/sparqles $CMDARGS -st
+	#echo "Running SPARQLes full cycle [st]"
+	#bin/sparqles $CMDARGS -st
 
    # # recompute
-	# echo "Running SPARQLes full cycle [r]"
-	# bin/sparqles $CMDARGS -r
+	#echo "Running SPARQLes full cycle [r]"
+	#bin/sparqles $CMDARGS -r
+	#echo "Running SPARQLes - recompute last [rl]"
+	#bin/sparqles $CMDARGS -rl
+	# index view
+	# FIXME: crashes on SPARQLES.recomputeIndexView
+   #echo "Running SPARQLes full cycle [iv]"
+	#bin/sparqles $CMDARGS -iv
+
    # index from old.datahub.io
 	# echo "Running SPARQLes full cycle [itask]"
 	# bin/sparqles $CMDARGS -run itask
@@ -46,8 +54,12 @@ export JAVA_OPTS="-XX:MaxRAMPercentage=80"
 # 	sleep $DELAY
 # done
 
-echo "${JAVA_OPTS}"
+#echo "Running SPARQLes [reschedule all tasks]"
+#bin/sparqles $CMDARGS -run reschedule
+
+#echo "${JAVA_OPTS}"
 
+echo "Running SPARQLes [start service]"
 ## Fully automatic
 JAVA_OPTS="${JAVA_OPTS} " bin/sparqles $CMDARGS --start
 
diff --git a/backend/src/main/avro/AResult.avsc b/backend/src/main/avro/AResult.avsc
@@ -1,5 +1,6 @@
 {"namespace": "sparqles.avro.availability",
  "type": "record",
+ "import" : "EndpointResult.avsc",
  "name": "AResult",
  "fields": [
      {"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},

diff --git a/backend/src/main/avro/CResult.avsc b/backend/src/main/avro/CResult.avsc
@@ -0,0 +1,28 @@
+{
+	"namespace": "sparqles.avro.calculation",
+ 	"type": "record",
+ 	"import" : "EndpointResult.avsc",
+ 	"name": "CResult",
+ 	"fields": [
+    	{"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},
+	{"name": "triples", "type": "long"},
+	{"name": "entities", "type": "long"},
+	{"name": "classes", "type": "long"},
+	{"name": "properties", "type": "long"},
+	{"name": "distinctSubjects", "type": "long"},
+	{"name": "distinctObjects", "type": "long"},
+	{"name": "exampleResources", "type":
+    	    {"type": "array", "items": 
+    		{	
+		    "name": "uri", "type": "string"
+      		}
+      	    }
+	},
+	{"name": "VoID", "type": "string"},
+	{"name": "VoIDPart", "type": "boolean"},
+	{"name": "SD", "type": "string"},
+	{"name": "SDPart", "type": "boolean"},
+	{"name": "coherence", "type": "double"},
+	{"name": "RS", "type": "double"}
+ 	]
+}
diff --git a/backend/src/main/avro/CalculationView.avsc b/backend/src/main/avro/CalculationView.avsc
@@ -0,0 +1,15 @@
+{
+"namespace": "sparqles.avro.analytics",
+"type": "record",
+"name": "CalculationView",
+"fields": [ 
+	{"name": "endpoint", "type": "sparqles.avro.Endpoint"},
+	{"name": "VoID", "type": "boolean"},
+	{"name": "VoIDPart", "type": "boolean"},
+	{"name": "SD", "type": "boolean"},
+	{"name": "SDPart", "type": "boolean"},
+	{"name": "coherence", "type": "double"},
+	{"name": "RS", "type": "double"},
+	{"name": "lastUpdate", "type": "long"}
+]
+}
diff --git a/backend/src/main/avro/DResult.avsc b/backend/src/main/avro/DResult.avsc
@@ -1,6 +1,7 @@
 {
 	"namespace": "sparqles.avro.discovery",
  	"type": "record",
+ 	"import" : "EndpointResult.avsc",
  	"name": "DResult",
  	"fields": [
     	{"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},

diff --git a/backend/src/main/avro/EPView.avsc b/backend/src/main/avro/EPView.avsc
@@ -2,10 +2,10 @@
 "namespace": "sparqles.avro.analytics",
 "type": "record",
 "name": "EPView",
-"fields": [ 
+"fields": [
 	{"name": "endpoint", "type": "sparqles.avro.Endpoint"},
-	{"name": "availability", "type": { 
-		"namespace": "sparqles.avro.analytics",	
+	{"name": "availability", "type": {
+		"namespace": "sparqles.avro.analytics",
 		"name": "EPViewAvailability",
       	"type": "record",
       	"fields" : [
@@ -16,15 +16,15 @@
       		{ "name": "uptimeLast31d", "type": "double"},
       		{ "name": "uptimeOverall", "type": "double"},
       		{ "name": "data", "type": {
-      			"namespace": "sparqles.avro.analytics",	
+      			"namespace": "sparqles.avro.analytics",
 				"name": "EPViewAvailabilityData",
       			"type": "record",
       			"fields" : [
       				{ "name": "key", "type": "string"},
-      				{ "name": "values", "type": 
-      					{"type": "array", "items": 
+      				{ "name": "values", "type":
+      					{"type": "array", "items":
 							{
-    							"namespace": "sparqles.avro.analytics",	
+    							"namespace": "sparqles.avro.analytics",
  								"name": "EPViewAvailabilityDataPoint",
       							"type": "record",
         						"fields" : [
@@ -46,19 +46,19 @@
 			"name":"EPViewPerformance",
 			"fields":[
 				{"name": "threshold", "type": "long"},
-				{"name": "ask" , "type": 
-					{"type": "array", "items": 
+				{"name": "ask" , "type":
+					{"type": "array", "items":
 						{
-    						"namespace": "sparqles.avro.analytics",	
+    						"namespace": "sparqles.avro.analytics",
  							"name": "EPViewPerformanceData",
       						"type": "record",
         					"fields" : [
         						{ "name": "key", "type": "string"},
         						{ "name": "color", "type": "string"},
         						{ "name": "data" , "type":
-   	 								{ "type": "array", "items": 
-    									{	
-    										"namespace": "sparqles.avro.analytics",	
+   	 								{ "type": "array", "items":
+    									{
+    										"namespace": "sparqles.avro.analytics",
  											"name": "EPViewPerformanceDataValues",
      										"type": "record",
         									"fields" : [
@@ -70,7 +70,7 @@
       								}
    	 							}
         					]
-      					}	
+      					}
       				}
 				},
 				{"name": "join" , "type": {"type": "array", "items": "array", "items" : "sparqles.avro.analytics.EPViewPerformanceData"}}
@@ -83,18 +83,18 @@
 			"type":"record",
 			"name":"EPViewInteroperability",
 			"fields":[
-				{"name": "SPARQL1Features" , "type": 
-					{"type": "array", "items": 
+				{"name": "SPARQL1Features" , "type":
+					{"type": "array", "items":
 						{
-    						"namespace": "sparqles.avro.analytics",	
+    						"namespace": "sparqles.avro.analytics",
  							"name": "EPViewInteroperabilityData",
       						"type": "record",
         					"fields" : [
         						{ "name": "label", "type": "string"},
         						{ "name": "value", "type": "boolean"},
         						{ "name": "exception", "type": ["string", "null"]}
         					]
-      					}	
+      					}
       				}
 				},
 				{"name": "SPARQL11Features" , "type": {"type": "array", "items": "array", "items" : "sparqles.avro.analytics.EPViewInteroperabilityData"}}
@@ -107,23 +107,65 @@
 			"type":"record",
 			"name":"EPViewDiscoverability",
 			"fields":[
-				{"name": "serverName" , "type" : "string"}, 
-				{"name": "VoIDDescription" , "type": 
-					{"type": "array", "items": 
+				{"name": "serverName" , "type" : "string"},
+				{"name": "VoIDDescription" , "type":
+					{"type": "array", "items":
 						{
-    						"namespace": "sparqles.avro.analytics",	
+    						"namespace": "sparqles.avro.analytics",
  							"name": "EPViewDiscoverabilityData",
       						"type": "record",
         					"fields" : [
         						{ "name": "label", "type": "string"},
         						{ "name": "value", "type": "boolean"}
         					]
-      					}	
+      					}
       				}
 				},
 				{"name": "SDDescription" , "type": {"type": "array", "items": "array", "items" : "sparqles.avro.analytics.EPViewDiscoverabilityData"}}
    	 		]
    	 	}
+ 	},
+ 	{"name": "calculation", "type":	{
+			"namespace":"sparqles.avro.analytics",
+			"type":"record",
+			"name":"EPViewCalculation",
+			"fields":[
+				{"name": "triples", "type": "long"},
+				{"name": "entities", "type": "long"},
+				{"name": "classes", "type": "long"},
+				{"name": "properties", "type": "long"},
+				{"name": "distinctSubjects", "type": "long"},
+				{"name": "distinctObjects", "type": "long"},
+				{"name": "exampleResources", "type":
+					 {"type": "array", "items":
+					 	  {
+						  	"name": "uri", "type": "string"
+						  }
+					 }
+        },
+				{"name": "VoID", "type": "string"},
+				{"name": "VoIDPart", "type": "boolean"},
+				{"name": "SD", "type": "string"},
+				{"name": "SDPart", "type": "boolean"},
+				{"name": "coherence", "type": "double"},
+				{"name": "RS", "type": "double"}
+   	 		]
+   	 	},
+      "default": {
+        "triples": -1,
+        "entities": -1,
+        "classes": -1,
+        "properties": -1,
+        "distinctSubjects": -1,
+        "distinctObjects": -1,
+        "exampleResources": [],
+        "VoID": "n/a",
+        "VoIDPart": false,
+        "SD": "n/a",
+        "SDPart": false,
+        "coherence": -1.0,
+        "RS": -1.0
+      }
  	}
-   	]
+]
 }
diff --git a/backend/src/main/avro/FResult.avsc b/backend/src/main/avro/FResult.avsc
@@ -1,6 +1,8 @@
 {
 	"namespace": "sparqles.avro.features",
  	"type": "record",
+ 	"import" : "EndpointResult.avsc",
+	"import" : "Run.avsc",
  	"name": "FResult",
  	"fields": [
  		{"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ @@
     /data/
     *.tmp
+    *~
     # Node
     npm-debug.log
@@ Expand Down @@