diff --git a/backend/run.sh b/backend/run.sh
index f5e2f042..b874df4d 100755
--- a/backend/run.sh
+++ b/backend/run.sh
@@ -15,29 +15,37 @@ export JAVA_OPTS="-XX:MaxRAMPercentage=80"
# while :
# do
- echo "Running SPARQLes full cycle"
+
+ #echo "Running SPARQLes full cycle"
# interop
- echo "Running SPARQLes full cycle [ftask]"
- bin/sparqles $CMDARGS -run ftask
+ #echo "Running SPARQLes full cycle [ftask]"
+ #bin/sparqles $CMDARGS -run ftask
# # availability
- # echo "Running SPARQLes full cycle [atask]"
- # bin/sparqles $CMDARGS -run atask
+ #echo "Running SPARQLes full cycle [atask]"
+ #bin/sparqles $CMDARGS -run atask
+ # coherence
+ #echo "Running SPARQLes full cycle [ctask]"
+ #bin/sparqles $CMDARGS -run ctask
# # performance
- echo "Running SPARQLes full cycle [ptask]"
- bin/sparqles $CMDARGS -run ptask
+ #echo "Running SPARQLes full cycle [ptask]"
+ #bin/sparqles $CMDARGS -run ptask
# # discoverability
- echo "Running SPARQLes full cycle [dtask]"
- bin/sparqles $CMDARGS -run dtask
- # index view
- echo "Running SPARQLes full cycle [iv]"
- bin/sparqles $CMDARGS -iv
+ #echo "Running SPARQLes full cycle [dtask]"
+ #bin/sparqles $CMDARGS -run dtask
# stats
- echo "Running SPARQLes full cycle [st]"
- bin/sparqles $CMDARGS -st
+ #echo "Running SPARQLes full cycle [st]"
+ #bin/sparqles $CMDARGS -st
# # recompute
- # echo "Running SPARQLes full cycle [r]"
- # bin/sparqles $CMDARGS -r
+ #echo "Running SPARQLes full cycle [r]"
+ #bin/sparqles $CMDARGS -r
+ #echo "Running SPARQLes - recompute last [rl]"
+ #bin/sparqles $CMDARGS -rl
+ # index view
+ # FIXME: crashes on SPARQLES.recomputeIndexView
+ #echo "Running SPARQLes full cycle [iv]"
+ #bin/sparqles $CMDARGS -iv
+
# index from old.datahub.io
# echo "Running SPARQLes full cycle [itask]"
# bin/sparqles $CMDARGS -run itask
@@ -46,8 +54,12 @@ export JAVA_OPTS="-XX:MaxRAMPercentage=80"
# sleep $DELAY
# done
-echo "${JAVA_OPTS}"
+#echo "Running SPARQLes [reschedule all tasks]"
+#bin/sparqles $CMDARGS -run reschedule
+
+#echo "${JAVA_OPTS}"
+echo "Running SPARQLes [start service]"
## Fully automatic
JAVA_OPTS="${JAVA_OPTS} " bin/sparqles $CMDARGS --start
diff --git a/backend/src/main/avro/AResult.avsc b/backend/src/main/avro/AResult.avsc
index a8263571..51672520 100644
--- a/backend/src/main/avro/AResult.avsc
+++ b/backend/src/main/avro/AResult.avsc
@@ -1,5 +1,6 @@
{"namespace": "sparqles.avro.availability",
"type": "record",
+ "import" : "EndpointResult.avsc",
"name": "AResult",
"fields": [
{"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},
diff --git a/backend/src/main/avro/CResult.avsc b/backend/src/main/avro/CResult.avsc
new file mode 100644
index 00000000..a4f55328
--- /dev/null
+++ b/backend/src/main/avro/CResult.avsc
@@ -0,0 +1,28 @@
+{
+ "namespace": "sparqles.avro.calculation",
+ "type": "record",
+ "import" : "EndpointResult.avsc",
+ "name": "CResult",
+ "fields": [
+ {"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},
+ {"name": "triples", "type": "long"},
+ {"name": "entities", "type": "long"},
+ {"name": "classes", "type": "long"},
+ {"name": "properties", "type": "long"},
+ {"name": "distinctSubjects", "type": "long"},
+ {"name": "distinctObjects", "type": "long"},
+ {"name": "exampleResources", "type":
+ {"type": "array", "items":
+ {
+ "name": "uri", "type": "string"
+ }
+ }
+ },
+ {"name": "VoID", "type": "string"},
+ {"name": "VoIDPart", "type": "boolean"},
+ {"name": "SD", "type": "string"},
+ {"name": "SDPart", "type": "boolean"},
+ {"name": "coherence", "type": "double"},
+ {"name": "RS", "type": "double"}
+ ]
+}
diff --git a/backend/src/main/avro/CalculationView.avsc b/backend/src/main/avro/CalculationView.avsc
new file mode 100644
index 00000000..798af624
--- /dev/null
+++ b/backend/src/main/avro/CalculationView.avsc
@@ -0,0 +1,15 @@
+{
+"namespace": "sparqles.avro.analytics",
+"type": "record",
+"name": "CalculationView",
+"fields": [
+ {"name": "endpoint", "type": "sparqles.avro.Endpoint"},
+ {"name": "VoID", "type": "boolean"},
+ {"name": "VoIDPart", "type": "boolean"},
+ {"name": "SD", "type": "boolean"},
+ {"name": "SDPart", "type": "boolean"},
+ {"name": "coherence", "type": "double"},
+ {"name": "RS", "type": "double"},
+ {"name": "lastUpdate", "type": "long"}
+]
+}
diff --git a/backend/src/main/avro/DResult.avsc b/backend/src/main/avro/DResult.avsc
index e3556281..11977616 100644
--- a/backend/src/main/avro/DResult.avsc
+++ b/backend/src/main/avro/DResult.avsc
@@ -1,6 +1,7 @@
{
"namespace": "sparqles.avro.discovery",
"type": "record",
+ "import" : "EndpointResult.avsc",
"name": "DResult",
"fields": [
{"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},
diff --git a/backend/src/main/avro/EPView.avsc b/backend/src/main/avro/EPView.avsc
index e8bae891..7b1692af 100644
--- a/backend/src/main/avro/EPView.avsc
+++ b/backend/src/main/avro/EPView.avsc
@@ -2,10 +2,10 @@
"namespace": "sparqles.avro.analytics",
"type": "record",
"name": "EPView",
-"fields": [
+"fields": [
{"name": "endpoint", "type": "sparqles.avro.Endpoint"},
- {"name": "availability", "type": {
- "namespace": "sparqles.avro.analytics",
+ {"name": "availability", "type": {
+ "namespace": "sparqles.avro.analytics",
"name": "EPViewAvailability",
"type": "record",
"fields" : [
@@ -16,15 +16,15 @@
{ "name": "uptimeLast31d", "type": "double"},
{ "name": "uptimeOverall", "type": "double"},
{ "name": "data", "type": {
- "namespace": "sparqles.avro.analytics",
+ "namespace": "sparqles.avro.analytics",
"name": "EPViewAvailabilityData",
"type": "record",
"fields" : [
{ "name": "key", "type": "string"},
- { "name": "values", "type":
- {"type": "array", "items":
+ { "name": "values", "type":
+ {"type": "array", "items":
{
- "namespace": "sparqles.avro.analytics",
+ "namespace": "sparqles.avro.analytics",
"name": "EPViewAvailabilityDataPoint",
"type": "record",
"fields" : [
@@ -46,19 +46,19 @@
"name":"EPViewPerformance",
"fields":[
{"name": "threshold", "type": "long"},
- {"name": "ask" , "type":
- {"type": "array", "items":
+ {"name": "ask" , "type":
+ {"type": "array", "items":
{
- "namespace": "sparqles.avro.analytics",
+ "namespace": "sparqles.avro.analytics",
"name": "EPViewPerformanceData",
"type": "record",
"fields" : [
{ "name": "key", "type": "string"},
{ "name": "color", "type": "string"},
{ "name": "data" , "type":
- { "type": "array", "items":
- {
- "namespace": "sparqles.avro.analytics",
+ { "type": "array", "items":
+ {
+ "namespace": "sparqles.avro.analytics",
"name": "EPViewPerformanceDataValues",
"type": "record",
"fields" : [
@@ -70,7 +70,7 @@
}
}
]
- }
+ }
}
},
{"name": "join" , "type": {"type": "array", "items": "array", "items" : "sparqles.avro.analytics.EPViewPerformanceData"}}
@@ -83,10 +83,10 @@
"type":"record",
"name":"EPViewInteroperability",
"fields":[
- {"name": "SPARQL1Features" , "type":
- {"type": "array", "items":
+ {"name": "SPARQL1Features" , "type":
+ {"type": "array", "items":
{
- "namespace": "sparqles.avro.analytics",
+ "namespace": "sparqles.avro.analytics",
"name": "EPViewInteroperabilityData",
"type": "record",
"fields" : [
@@ -94,7 +94,7 @@
{ "name": "value", "type": "boolean"},
{ "name": "exception", "type": ["string", "null"]}
]
- }
+ }
}
},
{"name": "SPARQL11Features" , "type": {"type": "array", "items": "array", "items" : "sparqles.avro.analytics.EPViewInteroperabilityData"}}
@@ -107,23 +107,65 @@
"type":"record",
"name":"EPViewDiscoverability",
"fields":[
- {"name": "serverName" , "type" : "string"},
- {"name": "VoIDDescription" , "type":
- {"type": "array", "items":
+ {"name": "serverName" , "type" : "string"},
+ {"name": "VoIDDescription" , "type":
+ {"type": "array", "items":
{
- "namespace": "sparqles.avro.analytics",
+ "namespace": "sparqles.avro.analytics",
"name": "EPViewDiscoverabilityData",
"type": "record",
"fields" : [
{ "name": "label", "type": "string"},
{ "name": "value", "type": "boolean"}
]
- }
+ }
}
},
{"name": "SDDescription" , "type": {"type": "array", "items": "array", "items" : "sparqles.avro.analytics.EPViewDiscoverabilityData"}}
]
}
+ },
+ {"name": "calculation", "type": {
+ "namespace":"sparqles.avro.analytics",
+ "type":"record",
+ "name":"EPViewCalculation",
+ "fields":[
+ {"name": "triples", "type": "long"},
+ {"name": "entities", "type": "long"},
+ {"name": "classes", "type": "long"},
+ {"name": "properties", "type": "long"},
+ {"name": "distinctSubjects", "type": "long"},
+ {"name": "distinctObjects", "type": "long"},
+ {"name": "exampleResources", "type":
+ {"type": "array", "items":
+ {
+ "name": "uri", "type": "string"
+ }
+ }
+ },
+ {"name": "VoID", "type": "string"},
+ {"name": "VoIDPart", "type": "boolean"},
+ {"name": "SD", "type": "string"},
+ {"name": "SDPart", "type": "boolean"},
+ {"name": "coherence", "type": "double"},
+ {"name": "RS", "type": "double"}
+ ]
+ },
+ "default": {
+ "triples": -1,
+ "entities": -1,
+ "classes": -1,
+ "properties": -1,
+ "distinctSubjects": -1,
+ "distinctObjects": -1,
+ "exampleResources": [],
+ "VoID": "n/a",
+ "VoIDPart": false,
+ "SD": "n/a",
+ "SDPart": false,
+ "coherence": -1.0,
+ "RS": -1.0
+ }
}
- ]
+]
}
diff --git a/backend/src/main/avro/FResult.avsc b/backend/src/main/avro/FResult.avsc
index a37772d2..ba3aa422 100644
--- a/backend/src/main/avro/FResult.avsc
+++ b/backend/src/main/avro/FResult.avsc
@@ -1,6 +1,8 @@
{
"namespace": "sparqles.avro.features",
"type": "record",
+ "import" : "EndpointResult.avsc",
+ "import" : "Run.avsc",
"name": "FResult",
"fields": [
{"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},
diff --git a/backend/src/main/avro/Index.avsc b/backend/src/main/avro/Index.avsc
index 72e6fe50..7b5042ae 100644
--- a/backend/src/main/avro/Index.avsc
+++ b/backend/src/main/avro/Index.avsc
@@ -91,7 +91,7 @@
]
}
}
- }
+ }
]
}
}
@@ -99,7 +99,6 @@
]
}
},
-
{"name": "discoverability", "type":
{
"namespace":"sparqles.avro.analytics",
@@ -137,7 +136,50 @@
]
}
+ },
+ {"name": "calculation", "type":
+ {
+ "namespace":"sparqles.avro.analytics",
+ "type":"record",
+ "name":"IndexViewCalculation",
+ "fields":[
+ {"name": "coherences" , "type":
+ {"type": "array", "items":
+ {
+ "namespace": "sparqles.avro.analytics",
+ "name": "IndexViewCalculationData",
+ "type": "record",
+ "fields" : [
+ { "name": "key", "type": "string"},
+ { "name": "values" , "type":
+ { "type": "array", "items":
+ {
+ "namespace": "sparqles.avro.analytics",
+ "name": "IndexViewCalculationDataValues",
+ "type": "record",
+ "fields" : [
+ { "name": "label", "type": "string"},
+ { "name": "value", "type": "double"}
+ ]
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {"name": "rss" , "type":
+ {"type": "array", "items": "sparqles.avro.analytics.IndexViewCalculationData"}
+ },
+ { "name": "VoID", "type": "double"},
+ { "name": "VoIDPart", "type": "double"},
+ { "name": "SD", "type": "double"},
+ { "name": "SDPart", "type": "double"},
+ { "name": "Coherence", "type": "double"},
+ { "name": "RS", "type": "double"}
+ ]
+ }
+
}
]
}
-
diff --git a/backend/src/main/avro/PResult.avsc b/backend/src/main/avro/PResult.avsc
index f83398d3..8cc0703b 100644
--- a/backend/src/main/avro/PResult.avsc
+++ b/backend/src/main/avro/PResult.avsc
@@ -1,6 +1,8 @@
{
"namespace": "sparqles.avro.performance",
"type": "record",
+ "import" : "EndpointResult.avsc",
+ "import" : "Run.avsc",
"name": "PResult",
"fields": [
{"name": "endpointResult", "type": "sparqles.avro.EndpointResult"},
@@ -10,17 +12,7 @@
"type": "record",
"fields" : [
{ "name": "query", "type": "string"},
- { "name": "cold", "type":
-{"type":"record","name":"Run","namespace":"sparqles.avro.performance",
- "fields":[
- {"name": "frestout", "type": "long"},
- {"name": "solutions", "type": "int"},
- {"name": "inittime", "type": "long"},
- {"name": "exectime", "type": "long"},
- {"name": "closetime", "type": "long"},
- {"name": "Exception", "type": ["string", "null"]},
- {"name": "exectout", "type": "long"}
- ]}},
+ { "name": "cold", "type": "sparqles.avro.performance.Run"},
{ "name": "warm", "type": "sparqles.avro.performance.Run"}
]}
}
diff --git a/backend/src/main/avro/Run.avsc b/backend/src/main/avro/Run.avsc
new file mode 100644
index 00000000..505644ee
--- /dev/null
+++ b/backend/src/main/avro/Run.avsc
@@ -0,0 +1,14 @@
+{
+ "namespace":"sparqles.avro.performance",
+ "type":"record",
+ "name":"Run",
+ "fields":[
+ {"name": "frestout", "type": "long"},
+ {"name": "solutions", "type": "int"},
+ {"name": "inittime", "type": "long"},
+ {"name": "exectime", "type": "long"},
+ {"name": "closetime", "type": "long"},
+ {"name": "Exception", "type": ["string", "null"]},
+ {"name": "exectout", "type": "long"}
+ ]
+}
diff --git a/backend/src/main/avro/Schedule.avsc b/backend/src/main/avro/Schedule.avsc
index 85daf388..cfba9483 100644
--- a/backend/src/main/avro/Schedule.avsc
+++ b/backend/src/main/avro/Schedule.avsc
@@ -8,6 +8,7 @@
{"name": "FTask", "type": ["string", "null"]},
{"name": "PTask", "type": ["string", "null"]},
{"name": "DTask", "type": ["string", "null"]},
+ {"name": "CTask", "type": ["string", "null"]},
{"name": "ITask", "type": ["string", "null"]},
{"name": "ETask", "type": ["string", "null"]}
]
diff --git a/backend/src/main/config/log4j.properties b/backend/src/main/config/log4j.properties
index 2dfb4cd6..3ae37ea4 100644
--- a/backend/src/main/config/log4j.properties
+++ b/backend/src/main/config/log4j.properties
@@ -6,17 +6,73 @@ log4j.rootLogger=INFO, stdout, stderr
#log4j.logger.sparqles.core.features=DEBUG, flog
#log4j.logger.sparqles.core.performance=DEBUG, plog
#log4j.logger.sparqles.utils.ExceptionHandler=INFO, exlog
-#DISABLE certain packages
-log4j.logger.org.apache.http=WARN
-log4j.logger.org.apache.commons.httpclient.params.DefaultHttpParams=INFO
-log4j.logger.com.hp.hpl.jena.sparql=WARN
-log4j.logger.org.apache.jena=WARN
+# Direct log messages to a log file
+log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.file.DatePattern = '.'yyyy-MM-dd
+log4j.appender.file.Append = true
+log4j.appender.file.Threshold=INFO
+log4j.appender.file.File=logs/main.log
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
+# Log all information for ATask log messages to a log file
+log4j.appender.alog=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.alog.DatePattern = '.'yyyy-MM-dd
+log4j.appender.alog.Append = true
+log4j.appender.alog.Threshold=INFO
+log4j.appender.alog.File=logs/availability.log
+log4j.appender.alog.layout=org.apache.log4j.PatternLayout
+log4j.appender.alog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
+
+
+# Log all information for ATask log messages to exception log file
+log4j.appender.exlog=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.exlog.DatePattern = '.'yyyy-MM-dd
+log4j.appender.exlog.Append = true
+log4j.appender.exlog.Threshold=INFO
+log4j.appender.exlog.File=logs/exception.log
+log4j.appender.exlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.exlog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
+
+
+
+# Log all information for ATask log messages to a log file
+log4j.appender.flog=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.flog.DatePattern = '.'yyyy-MM-dd
+log4j.appender.flog.Append = true
+log4j.appender.flog.Threshold=INFO
+log4j.appender.flog.File=logs/interoperability.log
+log4j.appender.flog.layout=org.apache.log4j.PatternLayout
+log4j.appender.flog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
+
+# Log all information for ATask log messages to a log file
+log4j.appender.dlog=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.dlog.DatePattern = '.'yyyy-MM-dd
+log4j.appender.dlog.Append = true
+log4j.appender.dlog.Threshold=INFO
+log4j.appender.dlog.File=logs/discoverability.log
+log4j.appender.dlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.dlog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
+
+# Log all information for ATask log messages to a log file
+log4j.appender.plog=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.plog.DatePattern = '.'yyyy-MM-dd
+log4j.appender.plog.Append = true
+log4j.appender.plog.Threshold=INFO
+log4j.appender.plog.File=logs/performance.log
+log4j.appender.plog.layout=org.apache.log4j.PatternLayout
+log4j.appender.plog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
+
+
+
# Direct log messages to stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+#log4j.appender.stdout.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %5p [%c{1}:%M:%L] - %m%n
+#log4j.appender.stdout.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
+
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.Target=System.err
log4j.appender.stderr.Threshold=ERROR
@@ -32,65 +88,16 @@ log4j.appender.stdout.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} [%5p] %c:%L
log4j.appender.stderr.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} [%5p] %c:%L - %m%n
-# Don't want to log to file when in Docker
-## Direct log messages to a log file
-#log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.file.DatePattern = '.'yyyy-MM-dd
-#log4j.appender.file.Append = true
-#log4j.appender.file.Threshold=INFO
-#log4j.appender.file.File=logs/main.log
-#log4j.appender.file.layout=org.apache.log4j.PatternLayout
-#log4j.appender.file.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
-#
-## Log all information for ATask log messages to a log file
-#log4j.appender.alog=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.alog.DatePattern = '.'yyyy-MM-dd
-#log4j.appender.alog.Append = true
-#log4j.appender.alog.Threshold=INFO
-#log4j.appender.alog.File=logs/availability.log
-#log4j.appender.alog.layout=org.apache.log4j.PatternLayout
-#log4j.appender.alog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
-#
-#
-## Log all information for ATask log messages to exception log file
-#log4j.appender.exlog=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.exlog.DatePattern = '.'yyyy-MM-dd
-#log4j.appender.exlog.Append = true
-#log4j.appender.exlog.Threshold=INFO
-#log4j.appender.exlog.File=logs/exception.log
-#log4j.appender.exlog.layout=org.apache.log4j.PatternLayout
-#log4j.appender.exlog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
-#
-#
-#
-## Log all information for ATask log messages to a log file
-#log4j.appender.flog=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.flog.DatePattern = '.'yyyy-MM-dd
-#log4j.appender.flog.Append = true
-#log4j.appender.flog.Threshold=INFO
-#log4j.appender.flog.File=logs/interoperability.log
-#log4j.appender.flog.layout=org.apache.log4j.PatternLayout
-#log4j.appender.flog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
-#
-## Log all information for ATask log messages to a log file
-#log4j.appender.dlog=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.dlog.DatePattern = '.'yyyy-MM-dd
-#log4j.appender.dlog.Append = true
-#log4j.appender.dlog.Threshold=INFO
-#log4j.appender.dlog.File=logs/discoverability.log
-#log4j.appender.dlog.layout=org.apache.log4j.PatternLayout
-#log4j.appender.dlog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
-#
-## Log all information for ATask log messages to a log file
-#log4j.appender.plog=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.plog.DatePattern = '.'yyyy-MM-dd
-#log4j.appender.plog.Append = true
-#log4j.appender.plog.Threshold=INFO
-#log4j.appender.plog.File=logs/performance.log
-#log4j.appender.plog.layout=org.apache.log4j.PatternLayout
-#log4j.appender.plog.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss} %15.15c{1}:%-3.3L %5p - %m%n
-#
-#log4j.appender.HTML=org.apache.log4j.FileAppender
-#log4j.appender.HTML.File=logs/main.html
-#log4j.appender.HTML.layout=org.apache.log4j.HTMLLayout
-#log4j.appender.HTML.Threshold=DEBUG
+log4j.appender.HTML=org.apache.log4j.FileAppender
+log4j.appender.HTML.File=logs/main.html
+log4j.appender.HTML.layout=org.apache.log4j.HTMLLayout
+log4j.appender.HTML.Threshold=DEBUG
+
+#DISABLE certain packages
+log4j.logger.org.apache.http=WARN
+log4j.logger.org.apache.commons.httpclient.params.DefaultHttpParams=INFO
+log4j.logger.com.hp.hpl.jena.sparql=WARN
+log4j.logger.org.apache.jena=WARN
+log4j.logger.sparqles.core.availability.ATask=DEBUG
+#log4j.logger.sparqles.utils.FaultDiagnostic=TRACE
+log4j.logger.sparqles.analytics.IndexViewAnalytics=DEBUG
diff --git a/backend/src/main/java/org/osjava/norbert/AbstractRule.java b/backend/src/main/java/org/osjava/norbert/AbstractRule.java
index 415ff285..7ed1ec60 100644
--- a/backend/src/main/java/org/osjava/norbert/AbstractRule.java
+++ b/backend/src/main/java/org/osjava/norbert/AbstractRule.java
@@ -34,20 +34,20 @@
/** Provides implementation for the path property and a handy toString. */
abstract class AbstractRule implements Rule {
- private String path;
+ private String path;
- public AbstractRule(String path) {
- this.path = path.trim();
- }
+ public AbstractRule(String path) {
+ this.path = path.trim();
+ }
- /** A url path snippet for which a rule exists */
- public String getPath() {
- return this.path;
- }
+ /** A url path snippet for which a rule exists */
+ public String getPath() {
+ return this.path;
+ }
- public abstract Boolean isAllowed(String query);
+ public abstract Boolean isAllowed(String query);
- public String toString() {
- return getClass().getName() + " on " + this.path;
- }
+ public String toString() {
+ return getClass().getName() + " on " + this.path;
+ }
}
diff --git a/backend/src/main/java/org/osjava/norbert/AllowedRule.java b/backend/src/main/java/org/osjava/norbert/AllowedRule.java
index 1c04a135..d28c14c8 100644
--- a/backend/src/main/java/org/osjava/norbert/AllowedRule.java
+++ b/backend/src/main/java/org/osjava/norbert/AllowedRule.java
@@ -34,20 +34,20 @@
/** A norobots Allow: rule. Any path which begins with the rule's path is allowed. */
class AllowedRule extends AbstractRule {
- public AllowedRule(String path) {
- super(path);
- }
+ public AllowedRule(String path) {
+ super(path);
+ }
- public Boolean isAllowed(String query) {
- if ("".equals(super.getPath())) {
- // What does the spec say here? Until I know, I'll just ignore this.
- return null;
- }
- boolean test = query.startsWith(super.getPath());
- if (!test) {
- return null;
- } else {
- return Boolean.TRUE;
- }
+ public Boolean isAllowed(String query) {
+ if ("".equals(super.getPath())) {
+ // What does the spec say here? Until I know, I'll just ignore this.
+ return null;
+ }
+ boolean test = query.startsWith(super.getPath());
+ if (!test) {
+ return null;
+ } else {
+ return Boolean.TRUE;
}
+ }
}
diff --git a/backend/src/main/java/org/osjava/norbert/DisallowedRule.java b/backend/src/main/java/org/osjava/norbert/DisallowedRule.java
index b58a783d..e7c017bd 100644
--- a/backend/src/main/java/org/osjava/norbert/DisallowedRule.java
+++ b/backend/src/main/java/org/osjava/norbert/DisallowedRule.java
@@ -34,19 +34,19 @@
/** A norobots Disallow: rule. Any path which begins with the rule's path is not allowed. */
class DisallowedRule extends AbstractRule {
- public DisallowedRule(String path) {
- super(path);
- }
+ public DisallowedRule(String path) {
+ super(path);
+ }
- public Boolean isAllowed(String query) {
- if ("".equals(super.getPath())) {
- return Boolean.TRUE;
- }
- boolean test = query.startsWith(super.getPath());
- if (!test) {
- return null;
- } else {
- return Boolean.FALSE;
- }
+ public Boolean isAllowed(String query) {
+ if ("".equals(super.getPath())) {
+ return Boolean.TRUE;
+ }
+ boolean test = query.startsWith(super.getPath());
+ if (!test) {
+ return null;
+ } else {
+ return Boolean.FALSE;
}
+ }
}
diff --git a/backend/src/main/java/org/osjava/norbert/NoRobotClient.java b/backend/src/main/java/org/osjava/norbert/NoRobotClient.java
index 1af7ec1e..45b30b96 100644
--- a/backend/src/main/java/org/osjava/norbert/NoRobotClient.java
+++ b/backend/src/main/java/org/osjava/norbert/NoRobotClient.java
@@ -43,200 +43,201 @@
*/
public class NoRobotClient {
- private String userAgent;
- private RulesEngine rules;
- private RulesEngine wildcardRules;
- private URL baseUrl;
-
- /**
- * Create a Client for a particular user-agent name.
- *
- * @param userAgent name for the robot
- */
- public NoRobotClient(String userAgent) {
- this.userAgent = userAgent;
- }
-
- /**
- * Head to a website and suck in their robots.txt file. Note that the URL passed in is for the
- * website and does not include the robots.txt file itself.
- *
- * @param baseUrl of the site
- */
- // public void parse(URL baseUrl) throws NoRobotException {
- //
- // this.rules = new RulesEngine();
- //
- // this.baseUrl = baseUrl;
- //
- // URL txtUrl = null;
- // try {
- // // fetch baseUrl+"robots.txt"
- // txtUrl = new URL(baseUrl, "robots.txt");
- // } catch(MalformedURLException murle) {
- // throw new NoRobotException("Bad URL: "+baseUrl+", robots.txt. ", murle);
- // }
- //
- // String txt = null;
- // try {
- // txt = loadContent(txtUrl, this.userAgent);
- // if(txt == null) {
- // throw new NoRobotException("No content found for: "+txtUrl);
- // }
- // } catch(IOException ioe) {
- // throw new NoRobotException("Unable to get content for: "+txtUrl, ioe);
- // }
- //
- // try {
- // parseText(txt);
- // } catch(NoRobotException nre) {
- // throw new NoRobotException("Problem while parsing "+txtUrl, nre);
- // }
- // }
- public void parse(String txt, URL baseUrl) throws NoRobotException {
- this.baseUrl = baseUrl;
- parseText(txt);
- }
-
- public void parseText(String txt) throws NoRobotException {
- this.rules = parseTextForUserAgent(txt, this.userAgent);
- this.wildcardRules = parseTextForUserAgent(txt, "*");
- }
+ private String userAgent;
+ private RulesEngine rules;
+ private RulesEngine wildcardRules;
+ private URL baseUrl;
+
+ /**
+ * Create a Client for a particular user-agent name.
+ *
+ * @param userAgent name for the robot (short one, e.g. {@link
+ * sparqles.core.CONSTANTS#USER_AGENT_STRING_RAW} )
+ */
+ public NoRobotClient(String userAgent) {
+ this.userAgent = userAgent;
+ }
+
+ /**
+ * Head to a website and suck in their robots.txt file. Note that the URL passed in is for the
+ * website and does not include the robots.txt file itself.
+ *
+ * @param baseUrl of the site
+ */
+ // public void parse(URL baseUrl) throws NoRobotException {
+ //
+ // this.rules = new RulesEngine();
+ //
+ // this.baseUrl = baseUrl;
+ //
+ // URL txtUrl = null;
+ // try {
+ // // fetch baseUrl+"robots.txt"
+ // txtUrl = new URL(baseUrl, "robots.txt");
+ // } catch(MalformedURLException murle) {
+ // throw new NoRobotException("Bad URL: "+baseUrl+", robots.txt. ", murle);
+ // }
+ //
+ // String txt = null;
+ // try {
+ // txt = loadContent(txtUrl, this.userAgent);
+ // if(txt == null) {
+ // throw new NoRobotException("No content found for: "+txtUrl);
+ // }
+ // } catch(IOException ioe) {
+ // throw new NoRobotException("Unable to get content for: "+txtUrl, ioe);
+ // }
+ //
+ // try {
+ // parseText(txt);
+ // } catch(NoRobotException nre) {
+ // throw new NoRobotException("Problem while parsing "+txtUrl, nre);
+ // }
+ // }
+
+ public void parse(String txt, URL baseUrl) throws NoRobotException {
+ this.baseUrl = baseUrl;
+ parseText(txt);
+ }
+
+ public void parseText(String txt) throws NoRobotException {
+ this.rules = parseTextForUserAgent(txt, this.userAgent);
+ this.wildcardRules = parseTextForUserAgent(txt, "*");
+ }
+
+ private RulesEngine parseTextForUserAgent(String txt, String userAgent) throws NoRobotException {
+
+ RulesEngine engine = new RulesEngine();
+
+ // Classic basic parser style, read an element at a time,
+ // changing a state variable [parsingAllowBlock]
+
+ // take each line, one at a time
+ BufferedReader rdr = new BufferedReader(new StringReader(txt));
+ String line = "";
+ String value = null;
+ boolean parsingAllowBlock = false;
+ try {
+ while ((line = rdr.readLine()) != null) {
+ // trim whitespace from either side
+ line = line.trim();
+
+ // ignore startsWith('#')
+ if (line.startsWith("#")) {
+ continue;
+ }
- private RulesEngine parseTextForUserAgent(String txt, String userAgent)
- throws NoRobotException {
-
- RulesEngine engine = new RulesEngine();
-
- // Classic basic parser style, read an element at a time,
- // changing a state variable [parsingAllowBlock]
-
- // take each line, one at a time
- BufferedReader rdr = new BufferedReader(new StringReader(txt));
- String line = "";
- String value = null;
- boolean parsingAllowBlock = false;
- try {
- while ((line = rdr.readLine()) != null) {
- // trim whitespace from either side
- line = line.trim();
-
- // ignore startsWith('#')
- if (line.startsWith("#")) {
- continue;
- }
-
- // if User-agent == userAgent
- // record the rest up until end or next User-agent
- // then quit (? check spec)
- if (line.toLowerCase().startsWith("user-agent:")) {
-
- if (parsingAllowBlock) {
- // we've just finished reading allows/disallows
- if (engine.isEmpty()) {
- // multiple user agents in a line, let's
- // wait til we get rules
- continue;
- } else {
- break;
- }
- }
-
- value = line.toLowerCase().substring("user-agent:".length()).trim();
- if (value.equalsIgnoreCase(userAgent)) {
- parsingAllowBlock = true;
- continue;
- }
- } else {
- // if not, then store if we're currently the user agent
- if (parsingAllowBlock) {
- if (line.startsWith("Allow:")) {
- value = line.substring("Allow:".length()).trim();
- value = URLDecoder.decode(value, "UTF-8");
- engine.allowPath(value);
- } else if (line.startsWith("Disallow:")) {
- value = line.substring("Disallow:".length()).trim();
- value = URLDecoder.decode(value, "UTF-8");
- engine.disallowPath(value);
- } else {
- // ignore
- continue;
- }
- } else {
- // ignore
- continue;
- }
- }
+ // if User-agent == userAgent
+ // record the rest up until end or next User-agent
+ // then quit (? check spec)
+ if (line.toLowerCase().startsWith("user-agent:")) {
+
+ if (parsingAllowBlock) {
+ // we've just finished reading allows/disallows
+ if (engine.isEmpty()) {
+ // multiple user agents in a line, let's
+ // wait til we get rules
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ value = line.toLowerCase().substring("user-agent:".length()).trim();
+ if (value.equalsIgnoreCase(userAgent.toLowerCase())) {
+ parsingAllowBlock = true;
+ continue;
+ }
+ } else {
+ // if not, then store if we're currently the user agent
+ if (parsingAllowBlock) {
+ if (line.startsWith("Allow:")) {
+ value = line.substring("Allow:".length()).trim();
+ value = URLDecoder.decode(value, "UTF-8");
+ engine.allowPath(value);
+ } else if (line.startsWith("Disallow:")) {
+ value = line.substring("Disallow:".length()).trim();
+ value = URLDecoder.decode(value, "UTF-8");
+ engine.disallowPath(value);
+ } else {
+ // ignore
+ continue;
}
- } catch (IOException ioe) {
- // As this is parsing a String, it should not have an IOE
- throw new NoRobotException("Problem while parsing text. ", ioe);
+ } else {
+ // ignore
+ continue;
+ }
}
-
- return engine;
+ }
+ } catch (IOException ioe) {
+ // As this is parsing a String, it should not have an IOE
+ throw new NoRobotException("Problem while parsing text. ", ioe);
}
- /**
- * Decide if the parsed website will allow this URL to be be seen.
- *
- * Note that parse(URL) must be called before this method is called.
- *
- * @param url in question
- * @return is the url allowed?
- * @throws IllegalStateException when parse has not been called
- */
- public boolean isUrlAllowed(URL url) throws IllegalStateException, IllegalArgumentException {
- if (rules == null) {
- throw new IllegalStateException("You must call parse before you call this method. ");
- }
-
- if (!baseUrl.getHost().equals(url.getHost())
- || baseUrl.getPort() != url.getPort()
- || !baseUrl.getProtocol().equals(url.getProtocol())) {
- throw new IllegalArgumentException(
- "Illegal to use a different url, "
- + url.toExternalForm()
- + ", for this robots.txt: "
- + this.baseUrl.toExternalForm());
- }
- String urlStr = url.toExternalForm().substring(this.baseUrl.toExternalForm().length() - 1);
- if ("/robots.txt".equals(urlStr)) {
- return true;
- }
- urlStr = URLDecoder.decode(urlStr);
- Boolean allowed = this.rules.isAllowed(urlStr);
- if (allowed == null) {
- allowed = this.wildcardRules.isAllowed(urlStr);
- }
- if (allowed == null) {
- allowed = Boolean.TRUE;
- }
-
- return allowed.booleanValue();
+ return engine;
+ }
+
+ /**
+ * Decide if the parsed website will allow this URL to be be seen.
+ *
+ *
Note that parse(URL) must be called before this method is called.
+ *
+ * @param url in question
+ * @return is the url allowed?
+ * @throws IllegalStateException when parse has not been called
+ */
+ public boolean isUrlAllowed(URL url) throws IllegalStateException, IllegalArgumentException {
+ if (rules == null) {
+ throw new IllegalStateException("You must call parse before you call this method. ");
}
- // // INLINE: as such from genjava/gj-core's net package. Simple method
- // // stolen from Payload too.
- // private static String loadContent(URL url, String userAgent) throws IOException {
- // URLConnection urlConn = url.openConnection();
- // if(urlConn instanceof HttpURLConnection) {
- // if(userAgent != null) {
- // ((HttpURLConnection)urlConn).addRequestProperty("User-Agent", userAgent);
- // }
- // }
- // InputStream in = urlConn.getInputStream();
- // BufferedReader rdr = new BufferedReader(new InputStreamReader(in));
- // StringBuffer buffer = new StringBuffer();
- // String line = "";
- // while( (line = rdr.readLine()) != null) {
- // buffer.append(line);
- // buffer.append("\n");
- // }
- // in.close();
- // return buffer.toString();
- // }
-
- public String toString() {
- return this.rules.toString() + " " + this.wildcardRules.toString();
+ if (!baseUrl.getHost().equals(url.getHost())
+ || baseUrl.getPort() != url.getPort()
+ || !baseUrl.getProtocol().equals(url.getProtocol())) {
+ throw new IllegalArgumentException(
+ "Illegal to use a different url, "
+ + url.toExternalForm()
+ + ", for this robots.txt: "
+ + this.baseUrl.toExternalForm());
}
+ String urlStr = url.toExternalForm().substring(this.baseUrl.toExternalForm().length() - 1);
+ if ("/robots.txt".equals(urlStr)) {
+ return true;
+ }
+ urlStr = URLDecoder.decode(urlStr);
+ Boolean allowed = this.rules.isAllowed(urlStr);
+ if (allowed == null) {
+ allowed = this.wildcardRules.isAllowed(urlStr);
+ }
+ if (allowed == null) {
+ allowed = Boolean.TRUE;
+ }
+
+ return allowed.booleanValue();
+ }
+
+ // // INLINE: as such from genjava/gj-core's net package. Simple method
+ // // stolen from Payload too.
+ // private static String loadContent(URL url, String userAgent) throws IOException {
+ // URLConnection urlConn = url.openConnection();
+ // if(urlConn instanceof HttpURLConnection) {
+ // if(userAgent != null) {
+ // ((HttpURLConnection)urlConn).addRequestProperty("User-Agent", userAgent);
+ // }
+ // }
+ // InputStream in = urlConn.getInputStream();
+ // BufferedReader rdr = new BufferedReader(new InputStreamReader(in));
+ // StringBuffer buffer = new StringBuffer();
+ // String line = "";
+ // while( (line = rdr.readLine()) != null) {
+ // buffer.append(line);
+ // buffer.append("\n");
+ // }
+ // in.close();
+ // return buffer.toString();
+ // }
+
+ public String toString() {
+ return this.rules.toString() + " " + this.wildcardRules.toString();
+ }
}
diff --git a/backend/src/main/java/org/osjava/norbert/NoRobotException.java b/backend/src/main/java/org/osjava/norbert/NoRobotException.java
index fa5f7291..b16fd5a5 100644
--- a/backend/src/main/java/org/osjava/norbert/NoRobotException.java
+++ b/backend/src/main/java/org/osjava/norbert/NoRobotException.java
@@ -37,11 +37,11 @@
*/
public class NoRobotException extends Exception {
- public NoRobotException(String message) {
- super(message);
- }
+ public NoRobotException(String message) {
+ super(message);
+ }
- public NoRobotException(String message, Throwable t) {
- super(message + " :::: " + t.getMessage());
- }
+ public NoRobotException(String message, Throwable t) {
+ super(message + " :::: " + t.getMessage());
+ }
}
diff --git a/backend/src/main/java/org/osjava/norbert/Rule.java b/backend/src/main/java/org/osjava/norbert/Rule.java
index 26b8cca5..b6e993ae 100644
--- a/backend/src/main/java/org/osjava/norbert/Rule.java
+++ b/backend/src/main/java/org/osjava/norbert/Rule.java
@@ -34,9 +34,9 @@
/** A robots.txt rule. Is a particular path allowed? */
public interface Rule {
- /**
- * Boolean.TRUE means it is allowed. Boolean.FALSE means it is not allowed. null means that this
- * rule is not applicable.
- */
- Boolean isAllowed(String path);
+ /**
+ * Boolean.TRUE means it is allowed. Boolean.FALSE means it is not allowed. null means that this
+ * rule is not applicable.
+ */
+ Boolean isAllowed(String path);
}
diff --git a/backend/src/main/java/org/osjava/norbert/RulesEngine.java b/backend/src/main/java/org/osjava/norbert/RulesEngine.java
index b9d532ce..4c2c0aa8 100644
--- a/backend/src/main/java/org/osjava/norbert/RulesEngine.java
+++ b/backend/src/main/java/org/osjava/norbert/RulesEngine.java
@@ -41,47 +41,47 @@
// TODO: Make this package private?
class RulesEngine {
- private List rules;
+ private List rules;
- public RulesEngine() {
- this.rules = new ArrayList();
- }
-
- public void allowPath(String path) {
- add(new AllowedRule(path));
- }
+ public RulesEngine() {
+ this.rules = new ArrayList();
+ }
- public void disallowPath(String path) {
- add(new DisallowedRule(path));
- }
+ public void allowPath(String path) {
+ add(new AllowedRule(path));
+ }
- public void add(Rule rule) {
- this.rules.add(rule);
- }
+ public void disallowPath(String path) {
+ add(new DisallowedRule(path));
+ }
- /**
- * Run each Rule in series on the path. If a Rule returns a Boolean, return that. When no more
- * rules are left, return null to indicate there were no rules for this path..
- */
- public Boolean isAllowed(String path) {
+ public void add(Rule rule) {
+ this.rules.add(rule);
+ }
- Iterator iterator = this.rules.iterator();
- while (iterator.hasNext()) {
- Rule rule = (Rule) iterator.next();
- Boolean test = rule.isAllowed(path);
- if (test != null) {
- return test;
- }
- }
+ /**
+ * Run each Rule in series on the path. If a Rule returns a Boolean, return that. When no more
+ * rules are left, return null to indicate there were no rules for this path..
+ */
+ public Boolean isAllowed(String path) {
- return null;
+ Iterator iterator = this.rules.iterator();
+ while (iterator.hasNext()) {
+ Rule rule = (Rule) iterator.next();
+ Boolean test = rule.isAllowed(path);
+ if (test != null) {
+ return test;
+ }
}
- public boolean isEmpty() {
- return this.rules.isEmpty();
- }
+ return null;
+ }
- public String toString() {
- return "RulesEngine: " + this.rules;
- }
+ public boolean isEmpty() {
+ return this.rules.isEmpty();
+ }
+
+ public String toString() {
+ return "RulesEngine: " + this.rules;
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/AAnalyser.java b/backend/src/main/java/sparqles/analytics/AAnalyser.java
index 6926b1a9..058431cc 100644
--- a/backend/src/main/java/sparqles/analytics/AAnalyser.java
+++ b/backend/src/main/java/sparqles/analytics/AAnalyser.java
@@ -14,219 +14,218 @@
import sparqles.utils.MongoDBManager;
public class AAnalyser extends Analytics {
- public static final int LAST_HOUR = 0;
- public static final int LAST_24HOURS = 1;
- public static final int LAST_7DAYS = 2;
- public static final int LAST_31DAYS = 3;
- public static final int THIS_WEEK = 4;
- private static final Logger log = LoggerFactory.getLogger(AAnalyser.class);
- public static DateCalculator _dates = new DateCalculator();
-
- public AAnalyser(MongoDBManager dbm) {
- super(dbm);
- }
-
- static void setDateCalculator(DateCalculator calc) {
- _dates = calc;
- }
-
- /** Computes the aggregated statistics for the Availability task */
- public boolean analyse(AResult ares) {
- try {
- log.info("[ANALYSE] {}", ares);
-
- Calendar now = Calendar.getInstance();
- now.setTimeInMillis(ares.getEndpointResult().getStart());
- log.debug("Start date: {}", now.getTime());
-
- Endpoint ep = ares.getEndpointResult().getEndpoint();
- Calendar[] dates = _dates.getDates(ares.getEndpointResult().getStart());
-
- // get the views
- AvailabilityView aview = getView(ep);
- EPView epview = getEPView(ep);
-
- // query mongodb for all AResults in the last 31 days
- List results =
- _db.getResultsSince(
- ep,
- AResult.class,
- AResult.SCHEMA$,
- dates[LAST_31DAYS].getTimeInMillis(),
- now.getTimeInMillis());
- log.debug(
- "Query for {}< - >={} returned " + results.size() + " results",
- dates[LAST_31DAYS].getTime(),
- now.getTime());
-
- SummaryStatistics last24HoursStats = new SummaryStatistics();
- SummaryStatistics last7DaysStats = new SummaryStatistics();
- SummaryStatistics last31DaysStats = new SummaryStatistics();
- SummaryStatistics thisWeekStats = new SummaryStatistics();
-
- for (AResult res : results) {
- long start = res.getEndpointResult().getStart();
- Calendar next = Calendar.getInstance();
- next.setTimeInMillis(start);
-
- if (start > dates[LAST_24HOURS].getTimeInMillis()) {
- update(last24HoursStats, res);
- log.debug(" {} >24h {}", next.getTime(), dates[LAST_24HOURS].getTime());
- }
- if (start > dates[LAST_7DAYS].getTimeInMillis()) {
- update(last7DaysStats, res);
- log.debug(" {} >7d {}", next.getTime(), dates[LAST_7DAYS].getTime());
- }
- if (start > dates[LAST_31DAYS].getTimeInMillis()) {
- update(last31DaysStats, res);
- log.debug(" {} >31d {}", next.getTime(), dates[LAST_31DAYS].getTime());
- }
- if (start > dates[THIS_WEEK].getTimeInMillis()) {
- update(thisWeekStats, res);
- log.debug(" {} >week {}", next.getTime(), dates[THIS_WEEK].getTime());
- }
- }
-
- // Update the views
- EPViewAvailability epav = epview.getAvailability();
-
- double last24HouerMean = 0;
- if (!Double.isNaN(last24HoursStats.getMean()))
- last24HouerMean = last24HoursStats.getMean();
- epav.setUptimeLast24h(last24HouerMean);
- aview.setUptimeLast24h(last24HouerMean);
-
- boolean upNow = ares.getIsAvailable();
- aview.setUpNow(upNow);
- epav.setUpNow(upNow);
-
- double last7dayMean = 0;
- if (!Double.isNaN(last7DaysStats.getMean())) last7dayMean = last7DaysStats.getMean();
- aview.setUptimeLast7d(last7dayMean);
- epav.setUptimeLast7d(last7dayMean);
-
- double thisweek = 0D;
- if (!Double.isNaN(thisWeekStats.getMean())) {
- thisweek = thisWeekStats.getMean();
- }
-
- Long key = dates[THIS_WEEK].getTimeInMillis();
- boolean exists = false;
- for (EPViewAvailabilityDataPoint dd : epav.getData().getValues()) {
- // System.out.println(dd.getX()+" =?= "+key);
- if (dd.getX().equals(key)) {
- exists = true;
- dd.setY(thisweek);
- }
- }
- // System.out.println(exists);
- if (!exists) {
- epav.getData().getValues().add(new EPViewAvailabilityDataPoint(key, thisweek));
- log.debug("Add new week: " + key);
- }
-
- // if(thisweek<1D && thisweek>0D){
- // System.out.println("Hello");
- // }
-
- double last31dayMean = 0;
- if (!Double.isNaN(last31DaysStats.getMean())) last31dayMean = last31DaysStats.getMean();
- epav.setUptimeLast31d(last31dayMean);
-
- // update overallUp
- int runs = epav.getTestRuns();
- Double mean = epav.getUptimeOverall() * runs;
- if (mean == null) mean = 0D;
- if (upNow) mean += 1;
- epav.setTestRuns(runs + 1);
- epav.setUptimeOverall(mean / (double) (runs + 1));
-
- log.debug(" [AView] {}", aview);
- log.debug(" [EPView] {}", epview);
- aview.setLastUpdate(ares.getEndpointResult().getEnd());
-
- boolean succ = false;
- succ = _db.update(aview);
- succ = _db.update(epview);
-
- // System.err.println("AView (after)="+aview);
- // System.err.println("EPView (after)="+epview);
-
- return succ;
- } catch (Exception e) {
- log.warn("[EXEC] {}", e);
+ public static final int LAST_HOUR = 0;
+ public static final int LAST_24HOURS = 1;
+ public static final int LAST_7DAYS = 2;
+ public static final int LAST_31DAYS = 3;
+ public static final int THIS_WEEK = 4;
+ private static final Logger log = LoggerFactory.getLogger(AAnalyser.class);
+ public static DateCalculator _dates = new DateCalculator();
+
+ public AAnalyser(MongoDBManager dbm) {
+ super(dbm);
+ }
+
+ static void setDateCalculator(DateCalculator calc) {
+ _dates = calc;
+ }
+
+ /** Computes the aggregated statistics for the Availability task */
+ public boolean analyse(AResult ares) {
+ try {
+ log.info("[ANALYSE] {}", ares);
+
+ Calendar now = Calendar.getInstance();
+ now.setTimeInMillis(ares.getEndpointResult().getStart());
+ log.debug("Start date: {}", now.getTime());
+
+ Endpoint ep = ares.getEndpointResult().getEndpoint();
+ Calendar[] dates = _dates.getDates(ares.getEndpointResult().getStart());
+
+ // get the views
+ AvailabilityView aview = getView(ep);
+ EPView epview = getEPView(ep);
+
+ // query mongodb for all AResults in the last 31 days
+ List results =
+ _db.getResultsSince(
+ ep,
+ AResult.class,
+ AResult.SCHEMA$,
+ dates[LAST_31DAYS].getTimeInMillis(),
+ now.getTimeInMillis());
+ log.debug(
+ "Query for {}< - >={} returned " + results.size() + " results",
+ dates[LAST_31DAYS].getTime(),
+ now.getTime());
+
+ SummaryStatistics last24HoursStats = new SummaryStatistics();
+ SummaryStatistics last7DaysStats = new SummaryStatistics();
+ SummaryStatistics last31DaysStats = new SummaryStatistics();
+ SummaryStatistics thisWeekStats = new SummaryStatistics();
+
+ for (AResult res : results) {
+ long start = res.getEndpointResult().getStart();
+ Calendar next = Calendar.getInstance();
+ next.setTimeInMillis(start);
+
+ if (start > dates[LAST_24HOURS].getTimeInMillis()) {
+ update(last24HoursStats, res);
+ log.debug(" {} >24h {}", next.getTime(), dates[LAST_24HOURS].getTime());
}
- return false;
- }
-
- private void update(SummaryStatistics stats, AResult res) {
- if (res.getIsAvailable()) {
- stats.addValue(1);
- } else stats.addValue(0);
- }
-
- private AvailabilityView getView(Endpoint ep) {
- AvailabilityView view = null;
- List views =
- _db.getResults(ep, AvailabilityView.class, AvailabilityView.SCHEMA$);
- if (views.size() != 1) {
- log.warn("We have {} AvailabilityView, expected was 1", views.size());
+ if (start > dates[LAST_7DAYS].getTimeInMillis()) {
+ update(last7DaysStats, res);
+ log.debug(" {} >7d {}", next.getTime(), dates[LAST_7DAYS].getTime());
}
- if (views.size() == 0) {
- view = new AvailabilityView();
- view.setEndpoint(ep);
- _db.insert(view);
-
- } else {
- view = views.get(0);
+ if (start > dates[LAST_31DAYS].getTimeInMillis()) {
+ update(last31DaysStats, res);
+ log.debug(" {} >31d {}", next.getTime(), dates[LAST_31DAYS].getTime());
+ }
+ if (start > dates[THIS_WEEK].getTimeInMillis()) {
+ update(thisWeekStats, res);
+ log.debug(" {} >week {}", next.getTime(), dates[THIS_WEEK].getTime());
}
- return view;
+ }
+
+ // Update the views
+ EPViewAvailability epav = epview.getAvailability();
+
+ double last24HouerMean = 0;
+ if (!Double.isNaN(last24HoursStats.getMean())) last24HouerMean = last24HoursStats.getMean();
+ epav.setUptimeLast24h(last24HouerMean);
+ aview.setUptimeLast24h(last24HouerMean);
+
+ boolean upNow = ares.getIsAvailable();
+ aview.setUpNow(upNow);
+ epav.setUpNow(upNow);
+
+ double last7dayMean = 0;
+ if (!Double.isNaN(last7DaysStats.getMean())) last7dayMean = last7DaysStats.getMean();
+ aview.setUptimeLast7d(last7dayMean);
+ epav.setUptimeLast7d(last7dayMean);
+
+ double thisweek = 0D;
+ if (!Double.isNaN(thisWeekStats.getMean())) {
+ thisweek = thisWeekStats.getMean();
+ }
+
+ long key = dates[THIS_WEEK].getTimeInMillis();
+ boolean exists = false;
+ for (EPViewAvailabilityDataPoint dd : epav.getData().getValues()) {
+ // System.out.println(dd.getX()+" =?= "+key);
+ if (dd.getX() == key) {
+ exists = true;
+ dd.setY(thisweek);
+ }
+ }
+ // System.out.println(exists);
+ if (!exists) {
+ epav.getData().getValues().add(new EPViewAvailabilityDataPoint(key, thisweek));
+ log.debug("Add new week: " + key);
+ }
+
+ // if(thisweek<1D && thisweek>0D){
+ // System.out.println("Hello");
+ // }
+
+ double last31dayMean = 0;
+ if (!Double.isNaN(last31DaysStats.getMean())) last31dayMean = last31DaysStats.getMean();
+ epav.setUptimeLast31d(last31dayMean);
+
+ // update overallUp
+ int runs = epav.getTestRuns();
+ Double mean = epav.getUptimeOverall() * runs;
+ if (mean == null) mean = 0D;
+ if (upNow) mean += 1;
+ epav.setTestRuns(runs + 1);
+ epav.setUptimeOverall(mean / (double) (runs + 1));
+
+ log.debug(" [AView] {}", aview);
+ log.debug(" [EPView] {}", epview);
+ aview.setLastUpdate(ares.getEndpointResult().getEnd());
+
+ boolean succ = false;
+ succ = _db.update(aview);
+ succ = _db.update(epview);
+
+ // System.err.println("AView (after)="+aview);
+ // System.err.println("EPView (after)="+epview);
+
+ return succ;
+ } catch (Exception e) {
+ log.warn("[EXEC] {}", e);
+ }
+ return false;
+ }
+
+ private void update(SummaryStatistics stats, AResult res) {
+ if (res.getIsAvailable()) {
+ stats.addValue(1);
+ } else stats.addValue(0);
+ }
+
+ private AvailabilityView getView(Endpoint ep) {
+ AvailabilityView view = null;
+ List views =
+ _db.getResults(ep, AvailabilityView.class, AvailabilityView.SCHEMA$);
+ if (views.size() != 1) {
+ log.warn("We have {} AvailabilityView, expected was 1", views.size());
}
+ if (views.size() == 0) {
+ view = new AvailabilityView();
+ view.setEndpoint(ep);
+ _db.insert(view);
- // private Calendar[] getDates(long time) {
- // Calendar now = Calendar.getInstance();
- // now.setTimeInMillis(time);
- //
- // Calendar lastHour = (Calendar) now.clone();
- // lastHour.add(Calendar.HOUR, -1);
- // //testing
- // //lastHour.add(Calendar.MINUTE, -2);
- //
- // Calendar last24Hour = (Calendar) now.clone();
- //// last24Hour.add(Calendar.HOUR, -24);
- // last24Hour.add(Calendar.MINUTE, -6);
- //
- // Calendar last7Days = (Calendar) now.clone();
- // // last7Days.add(Calendar.DAY_OF_YEAR, -7);
- // last7Days.add(Calendar.MINUTE, -12);
- //
- //
- // Calendar last31Days = (Calendar) now.clone();
- // // last31Days.add(Calendar.DAY_OF_YEAR, -31);
- // last31Days.add(Calendar.MINUTE, -18);
- //
- //
- // Calendar thisweek = Calendar.getInstance();
- // // thisweek.set(Calendar.YEAR, now.get(Calendar.YEAR));
- // // thisweek.set(Calendar.WEEK_OF_YEAR, now.get(Calendar.WEEK_OF_YEAR));
- // thisweek.set(Calendar.YEAR, now.get(Calendar.YEAR));
- // thisweek.set(Calendar.DAY_OF_YEAR, now.get(Calendar.DAY_OF_YEAR));
- // thisweek.set(Calendar.HOUR_OF_DAY, now.get(Calendar.HOUR_OF_DAY));
- // thisweek.set(Calendar.MINUTE, (now.get(Calendar.MINUTE)/10)*10);
- //
- //
- //
- // Calendar [] c = new Calendar[5];
- // c[LAST_HOUR]=lastHour;
- // c[LAST_24HOURS]=last24Hour;
- // c[LAST_7DAYS]= last7Days;
- // c[LAST_31DAYS] = last31Days;
- // c[THIS_WEEK] = thisweek;
- //// System.out.println("[DATES] from "+now.getTime()+" last1h:"+lastHour.getTime()+"
- // last24h:"+last24Hour.getTime());
- //// System.out.println(thisweek.getTime());
- // return c;
- // }
+ } else {
+ view = views.get(0);
+ }
+ return view;
+ }
+
+ // private Calendar[] getDates(long time) {
+ // Calendar now = Calendar.getInstance();
+ // now.setTimeInMillis(time);
+ //
+ // Calendar lastHour = (Calendar) now.clone();
+ // lastHour.add(Calendar.HOUR, -1);
+ // //testing
+ // //lastHour.add(Calendar.MINUTE, -2);
+ //
+ // Calendar last24Hour = (Calendar) now.clone();
+ //// last24Hour.add(Calendar.HOUR, -24);
+ // last24Hour.add(Calendar.MINUTE, -6);
+ //
+ // Calendar last7Days = (Calendar) now.clone();
+ // // last7Days.add(Calendar.DAY_OF_YEAR, -7);
+ // last7Days.add(Calendar.MINUTE, -12);
+ //
+ //
+ // Calendar last31Days = (Calendar) now.clone();
+ // // last31Days.add(Calendar.DAY_OF_YEAR, -31);
+ // last31Days.add(Calendar.MINUTE, -18);
+ //
+ //
+ // Calendar thisweek = Calendar.getInstance();
+ // // thisweek.set(Calendar.YEAR, now.get(Calendar.YEAR));
+ // // thisweek.set(Calendar.WEEK_OF_YEAR, now.get(Calendar.WEEK_OF_YEAR));
+ // thisweek.set(Calendar.YEAR, now.get(Calendar.YEAR));
+ // thisweek.set(Calendar.DAY_OF_YEAR, now.get(Calendar.DAY_OF_YEAR));
+ // thisweek.set(Calendar.HOUR_OF_DAY, now.get(Calendar.HOUR_OF_DAY));
+ // thisweek.set(Calendar.MINUTE, (now.get(Calendar.MINUTE)/10)*10);
+ //
+ //
+ //
+ // Calendar [] c = new Calendar[5];
+ // c[LAST_HOUR]=lastHour;
+ // c[LAST_24HOURS]=last24Hour;
+ // c[LAST_7DAYS]= last7Days;
+ // c[LAST_31DAYS] = last31Days;
+ // c[THIS_WEEK] = thisweek;
+ //// System.out.println("[DATES] from "+now.getTime()+" last1h:"+lastHour.getTime()+"
+ // last24h:"+last24Hour.getTime());
+ //// System.out.println(thisweek.getTime());
+ // return c;
+ // }
}
@@ -236,40 +235,40 @@ private AvailabilityView getView(Endpoint ep) {
* @author umbrichj
*/
class DateCalculator {
- Calendar[] getDates(long time) {
- Calendar now = Calendar.getInstance();
- now.setTimeInMillis(time);
-
- Calendar lastHour = (Calendar) now.clone();
- lastHour.add(Calendar.HOUR, -1);
-
- Calendar last24Hour = (Calendar) now.clone();
- last24Hour.add(Calendar.HOUR, -24);
-
- Calendar last7Days = (Calendar) now.clone();
- last7Days.add(Calendar.DAY_OF_YEAR, -7);
-
- Calendar last31Days = (Calendar) now.clone();
- last31Days.add(Calendar.DAY_OF_YEAR, -31);
-
- Calendar thisweek = Calendar.getInstance();
- // thisweek.add(Calendar.DAY_OF_YEAR, -1);
-
- thisweek.set(Calendar.YEAR, now.get(Calendar.YEAR));
- thisweek.set(Calendar.WEEK_OF_YEAR, now.get(Calendar.WEEK_OF_YEAR));
- thisweek.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY);
- thisweek.set(Calendar.HOUR_OF_DAY, 0);
- thisweek.set(Calendar.MINUTE, 0);
- thisweek.set(Calendar.SECOND, 0);
- thisweek.set(Calendar.MILLISECOND, 0);
- //
-
- Calendar[] c = new Calendar[5];
- c[AAnalyser.LAST_HOUR] = lastHour;
- c[AAnalyser.LAST_24HOURS] = last24Hour;
- c[AAnalyser.LAST_7DAYS] = last7Days;
- c[AAnalyser.LAST_31DAYS] = last31Days;
- c[AAnalyser.THIS_WEEK] = thisweek;
- return c;
- }
+ Calendar[] getDates(long time) {
+ Calendar now = Calendar.getInstance();
+ now.setTimeInMillis(time);
+
+ Calendar lastHour = (Calendar) now.clone();
+ lastHour.add(Calendar.HOUR, -1);
+
+ Calendar last24Hour = (Calendar) now.clone();
+ last24Hour.add(Calendar.HOUR, -24);
+
+ Calendar last7Days = (Calendar) now.clone();
+ last7Days.add(Calendar.DAY_OF_YEAR, -7);
+
+ Calendar last31Days = (Calendar) now.clone();
+ last31Days.add(Calendar.DAY_OF_YEAR, -31);
+
+ Calendar thisweek = Calendar.getInstance();
+ // thisweek.add(Calendar.DAY_OF_YEAR, -1);
+
+ thisweek.set(Calendar.YEAR, now.get(Calendar.YEAR));
+ thisweek.set(Calendar.WEEK_OF_YEAR, now.get(Calendar.WEEK_OF_YEAR));
+ thisweek.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY);
+ thisweek.set(Calendar.HOUR_OF_DAY, 0);
+ thisweek.set(Calendar.MINUTE, 0);
+ thisweek.set(Calendar.SECOND, 0);
+ thisweek.set(Calendar.MILLISECOND, 0);
+ //
+
+ Calendar[] c = new Calendar[5];
+ c[AAnalyser.LAST_HOUR] = lastHour;
+ c[AAnalyser.LAST_24HOURS] = last24Hour;
+ c[AAnalyser.LAST_7DAYS] = last7Days;
+ c[AAnalyser.LAST_31DAYS] = last31Days;
+ c[AAnalyser.THIS_WEEK] = thisweek;
+ return c;
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/AEvol.java b/backend/src/main/java/sparqles/analytics/AEvol.java
index 351bfadb..085b800f 100644
--- a/backend/src/main/java/sparqles/analytics/AEvol.java
+++ b/backend/src/main/java/sparqles/analytics/AEvol.java
@@ -1,6 +1,5 @@
package sparqles.analytics;
-import com.google.gson.Gson;
import com.mongodb.MongoClient;
import java.io.BufferedReader;
import java.io.IOException;
@@ -8,124 +7,144 @@
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Collection;
+import java.util.TimeZone;
import org.jongo.Jongo;
import org.jongo.MongoCollection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sparqles.avro.Endpoint;
import sparqles.core.SPARQLESProperties;
import sparqles.paper.objects.AMonth;
-import sparqles.paper.objects.AvailEpFromList;
import sparqles.utils.MongoDBManager;
public class AEvol {
-
- public AEvol(String[] args) {
- try {
- Gson gson = new Gson();
-
- // check if there is any stat to run or if it is up to date
- // open connection to mongodb aEvol collection
- Jongo jongo =
- new Jongo(
- new MongoClient(
- SPARQLESProperties.getDB_HOST()
- + ":"
- + SPARQLESProperties.getDB_PORT())
- .getDB(SPARQLESProperties.getDB_NAME()));
- MongoCollection amonthsColl = jongo.getCollection(MongoDBManager.COLL_AMONTHS);
-
- // get last month
- AMonth lastMonth = amonthsColl.findOne().orderBy("{date: -1}").as(AMonth.class);
- // check that lastMonth is from a different month than the current one
- Calendar cal = Calendar.getInstance();
- cal.setTime(lastMonth.getDate());
- Calendar calNow = Calendar.getInstance();
-
- // in case there is at least a full new month to process
- cal.add(Calendar.MONTH, 1);
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
- while (calNow.get(Calendar.MONTH) > cal.get(Calendar.MONTH)) {
- // get the end of the month
- Calendar calEnd = (Calendar) cal.clone();
- calEnd.add(Calendar.MONTH, 1);
- System.out.println(
- "Computing month aggregation from date ["
- + sdf.format(cal.getTime())
- + " to "
- + sdf.format(calEnd.getTime())
- + "[");
-
- // read the list of endpoints
- String json = readUrl("http://sparqles.ai.wu.ac.at/api/endpoint/list");
- AvailEpFromList[] epArray = gson.fromJson(json, AvailEpFromList[].class);
- MongoCollection atasksColl = jongo.getCollection(MongoDBManager.COLL_AVAIL);
- // System.out.println(atasksColl.count("{'endpointResult.start': {$gt : #}}",
- // cal.getTimeInMillis()));
-
- AMonth newMonth = new AMonth();
- newMonth.setDate(cal.getTime());
-
- // for each endpoint in the list (remove ghosts from the picture), get it's period
- // availability and add this ton the result object
- for (int i = 0; i < epArray.length; i++) {
-
- // get number of avail and unavail tests
- long nbAvail =
- atasksColl.count(
- "{'endpointResult.endpoint.uri': '"
- + epArray[i].getUri()
- + "', 'isAvailable':true, 'endpointResult.start': {$gte"
- + " : "
- + cal.getTimeInMillis()
- + ", $lt : "
- + calEnd.getTimeInMillis()
- + "}}}");
- long nbUnavail =
- atasksColl.count(
- "{'endpointResult.endpoint.uri': '"
- + epArray[i].getUri()
- + "', 'isAvailable':false, 'endpointResult.start':"
- + " {$gte : "
- + cal.getTimeInMillis()
- + ", $lt : "
- + calEnd.getTimeInMillis()
- + "}}}");
- // System.out.println(nbAvail+"\t"+nbUnavail+"\t"+epArray[i].getUri());
- newMonth.addEndpoint(nbAvail, nbUnavail);
- }
-
- // add the new month to the collection
- amonthsColl.insert(newMonth);
-
- // increment the month to process
- cal.add(Calendar.MONTH, 1);
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
+ private static final Logger log = LoggerFactory.getLogger(AEvol.class);
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ new AEvol(args);
+ }
+
+ public AEvol(String[] args) {
+ SPARQLESProperties.init(new java.io.File("src/main/resources/sparqles_docker.properties"));
+
+ // read the list of endpoints
+ MongoDBManager dbm = new MongoDBManager();
+ try {
+ recalculateMonthly(dbm);
+ } catch (Exception e) {
+ log.error("Error while recalculating monthly data", e);
}
+ }
+
+ public static void recalculateMonthly(MongoDBManager dbm) {
+ try {
+ Collection eps = dbm.get(Endpoint.class, Endpoint.SCHEMA$);
+
+ // check if there is any stat to run or if it is up to date
+ // open connection to mongodb aEvol collection
+ Jongo jongo =
+ new Jongo(
+ new MongoClient(
+ SPARQLESProperties.getDB_HOST() + ":" + SPARQLESProperties.getDB_PORT())
+ .getDB(SPARQLESProperties.getDB_NAME()));
+ MongoCollection amonthsColl = jongo.getCollection(MongoDBManager.COLL_AMONTHS);
+ // get last month
+ AMonth lastMonth = amonthsColl.findOne().orderBy("{date: -1}").as(AMonth.class);
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ Calendar calNow = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ calNow.set(Calendar.DAY_OF_MONTH, 1);
+ calNow.set(Calendar.HOUR, 0);
+ calNow.set(Calendar.MINUTE, 0);
+ calNow.set(Calendar.SECOND, 0);
+ calNow.add(Calendar.MONTH, -1);
+ if (lastMonth == null) {
+ cal.setTimeInMillis((dbm.getFirstAvailabitlityTime() / 1000) * 1000);
+ cal.set(Calendar.DAY_OF_MONTH, 1);
+ cal.set(Calendar.HOUR, 0);
+ cal.set(Calendar.MINUTE, 0);
+ cal.set(Calendar.SECOND, 0);
+ } else {
+ cal.setTime(lastMonth.getDate());
+ cal.add(Calendar.MONTH, 1);
+ }
+
+ // in case there is at least a full new month to process
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+ while (calNow.compareTo(cal) >= 0) {
+ // get the end of the month
+ Calendar calEnd = (Calendar) cal.clone();
+ calEnd.add(Calendar.MONTH, 1);
+ log.debug(
+ "Computing month aggregation from date ["
+ + sdf.format(cal.getTime())
+ + " to "
+ + sdf.format(calEnd.getTime())
+ + "[");
+
+ // String json = readUrl("https://sparqles.demo.openlinksw.com/api/endpoint/lis");
+ // AvailEpFromList[] epArray = gson.fromJson(json, AvailEpFromList[].class);
+ MongoCollection atasksColl = jongo.getCollection(MongoDBManager.COLL_AVAIL);
+ // System.out.println(atasksColl.count("{'endpointResult.start': {$gt : #}}",
+ // cal.getTimeInMillis()));
+
+ AMonth newMonth = new AMonth();
+ newMonth.setDate(cal.getTime());
+
+ // for each endpoint in the collection
+ for (Endpoint e : eps) {
+ // get number of avail and unavail tests
+ long nbAvail =
+ atasksColl.count(
+ "{'endpointResult.endpoint.uri': '"
+ + e.getUri()
+ + "', 'isAvailable':true, 'endpointResult.start': {$gte : "
+ + cal.getTimeInMillis()
+ + ", $lt : "
+ + calEnd.getTimeInMillis()
+ + "}}}");
+ long nbUnavail =
+ atasksColl.count(
+ "{'endpointResult.endpoint.uri': '"
+ + e.getUri()
+ + "', 'isAvailable':false, 'endpointResult.start': {$gte : "
+ + cal.getTimeInMillis()
+ + ", $lt : "
+ + calEnd.getTimeInMillis()
+ + "}}}");
+ newMonth.addEndpoint(nbAvail, nbUnavail);
+ }
- /**
- * @param args
- */
- public static void main(String[] args) {
- new AEvol(args);
+ // add the new month to the collection
+ amonthsColl.insert(newMonth);
+
+ // increment the month to process
+ cal.add(Calendar.MONTH, 1);
+ }
+ log.debug("Recalculating availability monthly COMPLETE");
+ } catch (IOException e) {
+ log.info("Exception while processing availability monthly (IO)", e);
+ } catch (Exception e) {
+ log.info("Exception while processing availability monthly (unknown)", e);
}
-
- private static String readUrl(String urlString) throws Exception {
- BufferedReader reader = null;
- try {
- URL url = new URL(urlString);
- reader = new BufferedReader(new InputStreamReader(url.openStream()));
- StringBuffer buffer = new StringBuffer();
- int read;
- char[] chars = new char[1024];
- while ((read = reader.read(chars)) != -1) buffer.append(chars, 0, read);
-
- return buffer.toString();
- } finally {
- if (reader != null) reader.close();
- }
+ }
+
+ private static String readUrl(String urlString) throws Exception {
+ BufferedReader reader = null;
+ try {
+ URL url = new URL(urlString);
+ reader = new BufferedReader(new InputStreamReader(url.openStream()));
+ StringBuffer buffer = new StringBuffer();
+ int read;
+ char[] chars = new char[1024];
+ while ((read = reader.read(chars)) != -1) buffer.append(chars, 0, read);
+
+ return buffer.toString();
+ } finally {
+ if (reader != null) reader.close();
}
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/AnalyserInit.java b/backend/src/main/java/sparqles/analytics/AnalyserInit.java
index 81504429..c4e6582e 100644
--- a/backend/src/main/java/sparqles/analytics/AnalyserInit.java
+++ b/backend/src/main/java/sparqles/analytics/AnalyserInit.java
@@ -7,6 +7,7 @@
import org.slf4j.LoggerFactory;
import sparqles.avro.Endpoint;
import sparqles.avro.availability.AResult;
+import sparqles.avro.calculation.CResult;
import sparqles.avro.discovery.DResult;
import sparqles.avro.features.FResult;
import sparqles.avro.performance.PResult;
@@ -14,154 +15,180 @@
public class AnalyserInit {
- private static final Logger log = LoggerFactory.getLogger(AnalyserInit.class);
+ private static final Logger log = LoggerFactory.getLogger(AnalyserInit.class);
- private MongoDBManager _db;
+ private MongoDBManager _db;
- private boolean _onlyLast;
+ private boolean _onlyLast;
- public AnalyserInit(MongoDBManager db) {
- this(db, false);
- }
+ public AnalyserInit(MongoDBManager db) {
+ this(db, false);
+ }
- public AnalyserInit(MongoDBManager db, boolean onlyLast) {
- _db = db;
- _onlyLast = onlyLast;
- }
+ public AnalyserInit(MongoDBManager db, boolean onlyLast) {
+ _db = db;
+ _onlyLast = onlyLast;
+ }
- /**
- * Computes the aggregated statistics for the Availability task
- *
- * @param ep
- */
- public void run() {
-
- List eps = _db.get(Endpoint.class, Endpoint.SCHEMA$);
- AAnalyser a = new AAnalyser(_db);
- PAnalyser p = new PAnalyser(_db);
- DAnalyser d = new DAnalyser(_db);
- FAnalyser f = new FAnalyser(_db);
-
- log.info("Analysing {} endpoints", eps.size());
- for (Endpoint ep : eps) {
- log.info("ANALYSE {}", ep.getUri());
-
- availability(ep, a);
- discoverability(ep, d);
- interoperability(ep, f);
- performance(ep, p);
- }
- }
+ /** Computes the aggregated statistics for the Availability task */
+ public void run() {
- private void discoverability(Endpoint ep, DAnalyser d) {
- TreeSet res =
- new TreeSet(
- new Comparator() {
- public int compare(DResult o1, DResult o2) {
- int diff =
- o1.getEndpointResult()
- .getStart()
- .compareTo(o2.getEndpointResult().getStart());
- return diff;
- }
- });
-
- List epRes = _db.getResults(ep, DResult.class, DResult.SCHEMA$);
- for (DResult epres : epRes) {
- res.add(epres);
- }
- log.info("Analyse {} Performance results", epRes.size());
- if (_onlyLast && epRes.size() != 0) {
- d.analyse(res.last());
- } else {
- for (DResult ares : res) {
- d.analyse(ares);
- }
- }
- log.info("ANALYSE DISCOVERABILITY {} and {}", ep, epRes.size());
- }
+ List eps = _db.get(Endpoint.class, Endpoint.SCHEMA$);
+ AAnalyser a = new AAnalyser(_db);
+ PAnalyser p = new PAnalyser(_db);
+ DAnalyser d = new DAnalyser(_db);
+ FAnalyser f = new FAnalyser(_db);
+ CAnalyser c = new CAnalyser(_db);
- private void performance(Endpoint ep, PAnalyser p) {
- TreeSet res =
- new TreeSet(
- new Comparator() {
- public int compare(PResult o1, PResult o2) {
- int diff =
- o1.getEndpointResult()
- .getStart()
- .compareTo(o2.getEndpointResult().getStart());
- return diff;
- }
- });
-
- List epRes = _db.getResults(ep, PResult.class, PResult.SCHEMA$);
- for (PResult epres : epRes) {
- res.add(epres);
- }
- log.info("Analyse {} Performance results", epRes.size());
- if (_onlyLast && epRes.size() != 0) {
- p.analyse(res.last());
- } else {
- for (PResult ares : res) {
- p.analyse(ares);
- }
- }
- log.info("ANALYSE PERFORMANCE {} and {}", ep, epRes.size());
- }
+ log.info("Analysing {} endpoints", eps.size());
+ for (Endpoint ep : eps) {
+ log.info("ANALYSE {}", ep.getUri());
- private void interoperability(Endpoint ep, FAnalyser f) {
- TreeSet res =
- new TreeSet(
- new Comparator() {
- public int compare(FResult o1, FResult o2) {
- int diff =
- o1.getEndpointResult()
- .getStart()
- .compareTo(o2.getEndpointResult().getStart());
- return diff;
- }
- });
-
- List epRes = _db.getResults(ep, FResult.class, FResult.SCHEMA$);
- for (FResult epres : epRes) {
- res.add(epres);
- }
- log.info("Analyse {} Interoperability results", epRes.size());
- if (_onlyLast && epRes.size() != 0) {
- f.analyse(res.last());
- } else {
- for (FResult ares : res) {
- f.analyse(ares);
- }
- }
- log.info("ANALYSE INTEROPERABILITY {} and {}", ep, epRes.size());
+ availability(ep, a);
+ discoverability(ep, d);
+ interoperability(ep, f);
+ performance(ep, p);
+ calculation(ep, c);
}
-
- private void availability(Endpoint ep, AAnalyser a) {
-
- TreeSet res =
- new TreeSet(
- new Comparator() {
- public int compare(AResult o1, AResult o2) {
- int diff =
- o1.getEndpointResult()
- .getStart()
- .compareTo(o2.getEndpointResult().getStart());
- return diff;
- }
- });
-
- List epRes = _db.getResults(ep, AResult.class, AResult.SCHEMA$);
- for (AResult epres : epRes) {
- res.add(epres);
- }
- if (_onlyLast && epRes.size() != 0) {
- a.analyse(res.last());
- } else {
- for (AResult ares : res) {
- a.analyse(ares);
- }
- }
- log.info("ANALYSE AVAILABILITY {} and {}", ep.getUri(), epRes.size());
+ }
+
+ private void discoverability(Endpoint ep, DAnalyser d) {
+ TreeSet res =
+ new TreeSet(
+ new Comparator() {
+ public int compare(DResult o1, DResult o2) {
+ int diff =
+ Comparator.comparingLong(
+ (DResult value) -> value.getEndpointResult().getStart())
+ .compare(o1, o2);
+ return diff;
+ }
+ });
+
+ List epRes = _db.getResults(ep, DResult.class, DResult.SCHEMA$);
+ for (DResult epres : epRes) {
+ res.add(epres);
+ }
+ log.info("Analyse {} Performance results", epRes.size());
+ if (_onlyLast && epRes.size() != 0) {
+ d.analyse(res.last());
+ } else {
+ for (DResult ares : res) {
+ d.analyse(ares);
+ }
+ }
+ log.info("ANALYSE DISCOVERABILITY {} and {}", ep, epRes.size());
+ }
+
+ private void performance(Endpoint ep, PAnalyser p) {
+ TreeSet res =
+ new TreeSet(
+ new Comparator() {
+ public int compare(PResult o1, PResult o2) {
+ int diff =
+ Comparator.comparingLong(
+ (PResult value) -> value.getEndpointResult().getStart())
+ .compare(o1, o2);
+ return diff;
+ }
+ });
+
+ List epRes = _db.getResults(ep, PResult.class, PResult.SCHEMA$);
+ for (PResult epres : epRes) {
+ res.add(epres);
+ }
+ log.info("Analyse {} Performance results", epRes.size());
+ if (_onlyLast && epRes.size() != 0) {
+ p.analyse(res.last());
+ } else {
+ for (PResult ares : res) {
+ p.analyse(ares);
+ }
+ }
+ log.info("ANALYSE PERFORMANCE {} and {}", ep, epRes.size());
+ }
+
+ private void interoperability(Endpoint ep, FAnalyser f) {
+ TreeSet res =
+ new TreeSet(
+ new Comparator() {
+ public int compare(FResult o1, FResult o2) {
+ int diff =
+ Comparator.comparingLong(
+ (FResult value) -> value.getEndpointResult().getStart())
+ .compare(o1, o2);
+ return diff;
+ }
+ });
+
+ List epRes = _db.getResults(ep, FResult.class, FResult.SCHEMA$);
+ for (FResult epres : epRes) {
+ res.add(epres);
+ }
+ log.info("Analyse {} Interoperability results", epRes.size());
+ if (_onlyLast && epRes.size() != 0) {
+ f.analyse(res.last());
+ } else {
+ for (FResult ares : res) {
+ f.analyse(ares);
+ }
+ }
+ log.info("ANALYSE INTEROPERABILITY {} and {}", ep, epRes.size());
+ }
+
+ private void availability(Endpoint ep, AAnalyser a) {
+
+ TreeSet res =
+ new TreeSet(
+ new Comparator() {
+ public int compare(AResult o1, AResult o2) {
+ int diff =
+ Comparator.comparingLong(
+ (AResult value) -> value.getEndpointResult().getStart())
+ .compare(o1, o2);
+ return diff;
+ }
+ });
+
+ List epRes = _db.getResults(ep, AResult.class, AResult.SCHEMA$);
+ for (AResult epres : epRes) {
+ res.add(epres);
+ }
+ if (_onlyLast && epRes.size() != 0) {
+ a.analyse(res.last());
+ } else {
+ for (AResult ares : res) {
+ a.analyse(ares);
+ }
+ }
+ log.info("ANALYSE AVAILABILITY {} and {}", ep.getUri(), epRes.size());
+ }
+
+ private void calculation(Endpoint ep, CAnalyser c) {
+
+ TreeSet res =
+ new TreeSet(
+ new Comparator() {
+ public int compare(CResult o1, CResult o2) {
+ int diff =
+ Comparator.comparingLong(
+ (CResult value) -> value.getEndpointResult().getStart())
+ .compare(o1, o2);
+ return diff;
+ }
+ });
+
+ List epRes = _db.getResults(ep, CResult.class, CResult.SCHEMA$);
+ for (CResult epres : epRes) {
+ res.add(epres);
+ }
+ if (_onlyLast && epRes.size() != 0) {
+ c.analyse(res.last());
+ } else {
+ for (CResult cres : res) {
+ c.analyse(cres);
+ }
}
+ log.info("ANALYSE CALCULATION {} and {}", ep.getUri(), epRes.size());
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/Analytics.java b/backend/src/main/java/sparqles/analytics/Analytics.java
index 8f11e249..95911aeb 100644
--- a/backend/src/main/java/sparqles/analytics/Analytics.java
+++ b/backend/src/main/java/sparqles/analytics/Analytics.java
@@ -10,12 +10,17 @@
import sparqles.avro.analytics.EPViewAvailability;
import sparqles.avro.analytics.EPViewAvailabilityData;
import sparqles.avro.analytics.EPViewAvailabilityDataPoint;
+import sparqles.avro.analytics.EPViewCalculation;
import sparqles.avro.analytics.EPViewDiscoverability;
import sparqles.avro.analytics.EPViewDiscoverabilityData;
import sparqles.avro.analytics.EPViewInteroperability;
import sparqles.avro.analytics.EPViewInteroperabilityData;
import sparqles.avro.analytics.EPViewPerformance;
import sparqles.avro.analytics.EPViewPerformanceData;
+import sparqles.avro.availability.AResult;
+import sparqles.avro.discovery.DResult;
+import sparqles.avro.features.FResult;
+import sparqles.avro.performance.PResult;
import sparqles.utils.MongoDBManager;
/**
@@ -26,63 +31,80 @@
*/
public abstract class Analytics {
- private static Logger Log = LoggerFactory.getLogger(Analytics.class);
+ private static Logger log = LoggerFactory.getLogger(Analytics.class);
- protected final MongoDBManager _db;
+ protected final MongoDBManager _db;
- public Analytics(MongoDBManager db) {
- _db = db;
+ public Analytics(MongoDBManager db) {
+ _db = db;
+ }
+
+ protected EPView getEPView(Endpoint ep) {
+ EPView view = null;
+ List views = _db.getResults(ep, EPView.class, EPView.SCHEMA$);
+ if (views.size() != 1) {
+ log.warn("We have {} EPView, expected was 1", views.size());
}
+ if (views.size() == 0) {
+ view = new EPView();
+ view.setEndpoint(ep);
- protected EPView getEPView(Endpoint ep) {
- EPView view = null;
- List views = _db.getResults(ep, EPView.class, EPView.SCHEMA$);
- if (views.size() != 1) {
- Log.warn("We have {} EPView, expected was 1", views.size());
- }
- if (views.size() == 0) {
- view = new EPView();
- view.setEndpoint(ep);
+ EPViewAvailability av = new EPViewAvailability();
+ view.setAvailability(av);
+ EPViewAvailabilityData data = new EPViewAvailabilityData();
+ av.setData(data);
+ data.setKey("Availability");
+ data.setValues(new ArrayList());
- EPViewAvailability av = new EPViewAvailability();
- view.setAvailability(av);
- EPViewAvailabilityData data = new EPViewAvailabilityData();
- av.setData(data);
- data.setKey("Availability");
- data.setValues(new ArrayList());
+ EPViewPerformance p = new EPViewPerformance();
+ ArrayList askdata = new ArrayList();
+ ArrayList joindata = new ArrayList();
- EPViewPerformance p = new EPViewPerformance();
- ArrayList askdata = new ArrayList();
- ArrayList joindata = new ArrayList();
+ p.setAsk(askdata);
+ p.setJoin(joindata);
- p.setAsk(askdata);
- p.setJoin(joindata);
+ EPViewInteroperability iv = new EPViewInteroperability();
+ iv.setSPARQL11Features(new ArrayList());
+ iv.setSPARQL1Features(new ArrayList());
- EPViewInteroperability iv = new EPViewInteroperability();
- iv.setSPARQL11Features(new ArrayList());
- iv.setSPARQL1Features(new ArrayList());
+ view.setPerformance(p);
+ view.setAvailability(av);
+ view.setInteroperability(iv);
- view.setPerformance(p);
- view.setAvailability(av);
- view.setInteroperability(iv);
+ EPViewDiscoverability dv =
+ new EPViewDiscoverability(
+ "",
+ new ArrayList(),
+ new ArrayList());
+ view.setDiscoverability(dv);
- EPViewDiscoverability dv =
- new EPViewDiscoverability(
- "",
- new ArrayList(),
- new ArrayList());
- view.setDiscoverability(dv);
+ EPViewCalculation cv =
+ new EPViewCalculation(
+ -1l,
+ -1l,
+ -1l,
+ -1l,
+ -1l,
+ -1l,
+ new java.util.ArrayList(),
+ "",
+ false,
+ "",
+ false,
+ -1.0,
+ -1.0);
+ view.setCalculation(cv);
- _db.insert(view);
- } else {
- view = views.get(0);
- }
- return view;
+ _db.insert(view);
+ } else {
+ view = views.get(0);
}
+ return view;
+ }
- /**
- * @param result - the result to analyse
- * @return true in case of success, false otherwise
- */
- public abstract boolean analyse(V result);
+ /**
+ * @param result - the result to analyse
+ * @return true in case of success, false otherwise
+ */
+ public abstract boolean analyse(V result);
}
diff --git a/backend/src/main/java/sparqles/analytics/CAnalyser.java b/backend/src/main/java/sparqles/analytics/CAnalyser.java
new file mode 100644
index 00000000..92c2a80b
--- /dev/null
+++ b/backend/src/main/java/sparqles/analytics/CAnalyser.java
@@ -0,0 +1,95 @@
+package sparqles.analytics;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sparqles.avro.Endpoint;
+import sparqles.avro.analytics.CalculationView;
+import sparqles.avro.analytics.EPView;
+import sparqles.avro.analytics.EPViewCalculation;
+import sparqles.avro.calculation.CResult;
+import sparqles.utils.MongoDBManager;
+
+public class CAnalyser extends Analytics {
+ private static final Logger log = LoggerFactory.getLogger(CAnalyser.class);
+
+ public CAnalyser(MongoDBManager db) {
+ super(db);
+ }
+
+ @Override
+ public boolean analyse(CResult pres) {
+ log.info("[Analytics] {}", pres.getEndpointResult().getEndpoint());
+
+ Endpoint ep = pres.getEndpointResult().getEndpoint();
+
+ EPView epview = getEPView(ep);
+ CalculationView cview = getView(ep);
+
+ if (pres.getVoIDPart()) {
+ cview.setVoIDPart(true);
+ cview.setVoID(false);
+ } else {
+ cview.setVoID(!pres.getVoID().toString().equals(""));
+ cview.setVoIDPart(false);
+ }
+ if (pres.getSDPart()) {
+ cview.setSDPart(true);
+ cview.setSD(false);
+ } else {
+ cview.setSD(!pres.getSD().toString().equals(""));
+ cview.setSDPart(false);
+ }
+ cview.setCoherence(pres.getCoherence());
+ cview.setRS(pres.getRS());
+
+ cview.setLastUpdate(pres.getEndpointResult().getEnd());
+
+ EPViewCalculation cepview = epview.getCalculation();
+ cepview.setTriples(pres.getTriples());
+ cepview.setEntities(pres.getEntities());
+ cepview.setClasses(pres.getClasses());
+ cepview.setProperties(pres.getProperties());
+ cepview.setDistinctSubjects(pres.getDistinctSubjects());
+ cepview.setDistinctObjects(pres.getDistinctObjects());
+ cepview.setExampleResources(pres.getExampleResources());
+ cepview.setVoID(pres.getVoID());
+ cepview.setVoIDPart(pres.getVoIDPart());
+ cepview.setSD(pres.getSD());
+ cepview.setSDPart(pres.getSDPart());
+ cepview.setCoherence(pres.getCoherence());
+ cepview.setRS(pres.getRS());
+
+ _db.update(cview);
+ _db.update(epview);
+
+ return true;
+ }
+
+ private CalculationView getView(Endpoint ep) {
+ CalculationView view = null;
+ List views = new ArrayList();
+ if (_db != null) {
+ views = _db.getResults(ep, CalculationView.class, CalculationView.SCHEMA$);
+ }
+ if (views.size() != 1) {
+ log.warn("We have {} CalculationView, expected was 1", views.size());
+ }
+ if (views.size() == 0) {
+ view = new CalculationView();
+ view.setEndpoint(ep);
+ view.setSD(false);
+ view.setSDPart(false);
+ view.setVoID(false);
+ view.setVoIDPart(false);
+ view.setCoherence(-1.0);
+ view.setRS(-1.0);
+ view.setLastUpdate(-1L);
+ if (_db != null) _db.insert(view);
+ } else {
+ view = views.get(0);
+ }
+ return view;
+ }
+}
diff --git a/backend/src/main/java/sparqles/analytics/DAnalyser.java b/backend/src/main/java/sparqles/analytics/DAnalyser.java
index ed073452..1b089dcb 100644
--- a/backend/src/main/java/sparqles/analytics/DAnalyser.java
+++ b/backend/src/main/java/sparqles/analytics/DAnalyser.java
@@ -16,202 +16,198 @@
import sparqles.utils.MongoDBManager;
public class DAnalyser extends Analytics {
- private static final Logger log = LoggerFactory.getLogger(DAnalyser.class);
+ private static final Logger log = LoggerFactory.getLogger(DAnalyser.class);
- public DAnalyser(MongoDBManager db) {
- super(db);
- }
+ public DAnalyser(MongoDBManager db) {
+ super(db);
+ }
- @Override
- public boolean analyse(DResult pres) {
- log.info("[Analytics] {}", pres);
-
- Endpoint ep = pres.getEndpointResult().getEndpoint();
-
- DiscoverabilityView dview = getView(ep);
- EPView epview = getEPView(ep);
-
- List lvoid = new ArrayList();
- List lsd = new ArrayList();
-
- String serverName = "missing";
- for (DGETInfo info : pres.getDescriptionFiles()) {
- if (info.getOperation().toString().equals(DTask.EPURL)) {
- if (!info.getResponseServer().toString().equalsIgnoreCase("missing")) {
- serverName = info.getResponseServer().toString();
- }
-
- EPViewDiscoverabilityData d =
- new EPViewDiscoverabilityData("HTTP Get", info.getVoiDpreds().size() != 0);
- lvoid.add(d);
-
- d =
- new EPViewDiscoverabilityData(
- "HTTP Get", info.getSPARQLDESCpreds().size() != 0);
- lsd.add(d);
- }
- if (info.getOperation().toString().equalsIgnoreCase("wellknown")) {
- if (!info.getResponseServer().toString().equalsIgnoreCase("missing")) {
- serverName = info.getResponseServer().toString();
- }
-
- EPViewDiscoverabilityData d =
- new EPViewDiscoverabilityData(
- "/.well-known/void", info.getVoiDpreds().size() != 0);
- lvoid.add(d);
-
- d =
- new EPViewDiscoverabilityData(
- "/.well-known/void", info.getSPARQLDESCpreds().size() != 0);
- lsd.add(d);
- }
- if (info.getSPARQLDESCpreds().size() > 0) {
- dview.setSD(true);
- }
- if (info.getVoiDpreds().size() > 0) {
- dview.setVoID(true);
- }
- }
- log.info("Setting server name to {}", serverName);
- dview.setServerName(serverName);
- EPViewDiscoverability depview = epview.getDiscoverability();
+ @Override
+ public boolean analyse(DResult pres) {
+ log.info("[Analytics] {}", pres);
- depview.setServerName(dview.getServerName());
- depview.setVoIDDescription(lvoid);
+ Endpoint ep = pres.getEndpointResult().getEndpoint();
- for (QueryInfo info : pres.getQueryInfo()) {
+ DiscoverabilityView dview = getView(ep);
+ EPView epview = getEPView(ep);
- if (info.getOperation().equals("query-self")) {
- EPViewDiscoverabilityData d =
- new EPViewDiscoverabilityData(
- "SPARQL Endpoint content", info.getResults().size() != 0);
- lvoid.add(d);
- }
+ List lvoid = new ArrayList();
+ List lsd = new ArrayList();
+
+ String serverName = "missing";
+ for (DGETInfo info : pres.getDescriptionFiles()) {
+ if (info.getOperation().toString().equals(DTask.EPURL)) {
+ if (!info.getResponseServer().toString().equalsIgnoreCase("missing")) {
+ serverName = info.getResponseServer().toString();
}
- //
- //
-
- // d = new EPViewDiscoverabilityData("HTTP Get",
- // pres.getGetResult().getSPARQLDESCterms()!=0);
- // l.add(d);
- // d = new EPViewDiscoverabilityData("SPARQL Endpoint content",
- // pres.getVoidResult().getSPARQLFile().size()!=0);
- // l.add(d);
-
- // depview.setSDDescription(l);
-
- dview.setLastUpdate(pres.getEndpointResult().getEnd());
-
- _db.update(dview);
- _db.update(epview);
- return true;
-
- // SummaryStatistics askStatsCold = new SummaryStatistics();
- // SummaryStatistics askStatsWarm = new SummaryStatistics();
- // SummaryStatistics joinStatsCold = new SummaryStatistics();
- // SummaryStatistics joinStatsWarm = new SummaryStatistics();
- //
- // //prepare eppview data
- // EPViewPerformance eppview = epview.getPerformance();
- // EPViewPerformanceData askCold = new EPViewPerformanceData("Cold ASK Tests","#1f77b4",
- // new
- // ArrayList());
- // EPViewPerformanceData askWarm = new EPViewPerformanceData("WARM ASK Tests","#2ca02c",
- // new
- // ArrayList());
- // EPViewPerformanceData joinCold = new EPViewPerformanceData("Cold JOIN Tests","#1f77b4",
- // new
- // ArrayList());
- // EPViewPerformanceData joinWarm = new EPViewPerformanceData("Warm JOIN Tests","#2ca02c",
- // new
- // ArrayList());
- //
- // ArrayList askdata= new ArrayList();
- // askdata.add(askCold);
- // askdata.add(askWarm);
- // ArrayList joindata= new ArrayList();
- // joindata.add(joinCold);
- // joindata.add(joinWarm);
- //
- // eppview.setAsk(askdata);
- // eppview.setJoin(joindata);
- //
- //
- // Map map = pres.getResults();
- // int limit =0 ;
- //
- // for(Entry ent: map.entrySet()){
- // PSingleResult res = ent.getValue();
- // if(ent.getKey().toString().startsWith("ASK")){
- // askStatsCold.addValue(res.getCold().getClosetime()/(double)1000);
- // askStatsWarm.addValue(res.getWarm().getClosetime()/(double)1000);
- //
- // String key = ent.getKey().toString().replaceAll("ASK", "").toLowerCase();
- //
- //
- // askCold.getData().add(new
- // EPViewPerformanceDataValues(key,res.getCold().getClosetime()/(double)1000));
- // askWarm.getData().add(new
- // EPViewPerformanceDataValues(key,res.getWarm().getClosetime()/(double)1000));
- // }else if(ent.getKey().toString().startsWith("JOIN")){
- // joinStatsCold.addValue(res.getCold().getClosetime()/(double)1000);
- // joinStatsWarm.addValue(res.getCold().getClosetime()/(double)1000);
- //
- // String key = ent.getKey().toString().replaceAll("JOIN", "").toLowerCase();
- //
- // joinCold.getData().add(new
- // EPViewPerformanceDataValues(key,res.getCold().getClosetime()/(double)1000));
- // joinWarm.getData().add(new
- // EPViewPerformanceDataValues(key,res.getWarm().getClosetime()/(double)1000));
- // }else if(ent.getKey().toString().startsWith("LIMIT")){
- // int sol = res.getCold().getSolutions();
- // if(Math.max(limit, sol)==sol){
- // limit = sol;
- // }
- // sol = res.getWarm().getSolutions();
- // if(Math.max(limit, sol)==sol){
- // limit = sol;
- // }
- // }
- // }
- //
- //
- // //Update pview data
- // pview.setAskMeanCold(askStatsCold.getMean());
- // pview.setAskMeanWarm(askStatsWarm.getMean());
- // pview.setJoinMeanCold(joinStatsCold.getMean());
- // pview.setJoinMeanWarm(joinStatsWarm.getMean());
- //
- //
- // System.out.println(pview);
- // System.out.println(epview);
- // _db.update(pview);
- // _db.update(epview);
- //
- // return true;
- }
+ EPViewDiscoverabilityData d =
+ new EPViewDiscoverabilityData("HTTP Get", info.getVoiDpreds().size() != 0);
+ lvoid.add(d);
- private DiscoverabilityView getView(Endpoint ep) {
- DiscoverabilityView view = null;
- List views = new ArrayList();
- if (_db != null) {
- views = _db.getResults(ep, DiscoverabilityView.class, DiscoverabilityView.SCHEMA$);
+ d = new EPViewDiscoverabilityData("HTTP Get", info.getSPARQLDESCpreds().size() != 0);
+ lsd.add(d);
+ }
+ if (info.getOperation().toString().equalsIgnoreCase("wellknown")) {
+ if (!info.getResponseServer().toString().equalsIgnoreCase("missing")) {
+ serverName = info.getResponseServer().toString();
}
- if (views.size() != 1) {
- log.warn("We have {} FeatureView, expected was 1", views.size());
- }
- if (views.size() == 0) {
- view = new DiscoverabilityView();
- view.setEndpoint(ep);
- view.setSD(false);
- view.setVoID(false);
- view.setServerName("missing");
- view.setLastUpdate(-1L);
- if (_db != null) _db.insert(view);
- } else {
- view = views.get(0);
- }
- return view;
+
+ EPViewDiscoverabilityData d =
+ new EPViewDiscoverabilityData("/.well-known/void", info.getVoiDpreds().size() != 0);
+ lvoid.add(d);
+
+ d =
+ new EPViewDiscoverabilityData(
+ "/.well-known/void", info.getSPARQLDESCpreds().size() != 0);
+ lsd.add(d);
+ }
+ if (info.getSPARQLDESCpreds().size() > 0) {
+ dview.setSD(true);
+ }
+ if (info.getVoiDpreds().size() > 0) {
+ dview.setVoID(true);
+ }
+ }
+ log.info("Setting server name to {}", serverName);
+ dview.setServerName(serverName);
+ EPViewDiscoverability depview = epview.getDiscoverability();
+
+ depview.setServerName(dview.getServerName());
+ depview.setVoIDDescription(lvoid);
+
+ for (QueryInfo info : pres.getQueryInfo()) {
+
+ if (info.getOperation().equals("query-self")) {
+ EPViewDiscoverabilityData d =
+ new EPViewDiscoverabilityData("SPARQL Endpoint content", info.getResults().size() != 0);
+ lvoid.add(d);
+ }
+ }
+
+ //
+ //
+
+ // d = new EPViewDiscoverabilityData("HTTP Get",
+ // pres.getGetResult().getSPARQLDESCterms()!=0);
+ // l.add(d);
+ // d = new EPViewDiscoverabilityData("SPARQL Endpoint content",
+ // pres.getVoidResult().getSPARQLFile().size()!=0);
+ // l.add(d);
+
+ // depview.setSDDescription(l);
+
+ dview.setLastUpdate(pres.getEndpointResult().getEnd());
+
+ _db.update(dview);
+ _db.update(epview);
+ return true;
+
+ // SummaryStatistics askStatsCold = new SummaryStatistics();
+ // SummaryStatistics askStatsWarm = new SummaryStatistics();
+ // SummaryStatistics joinStatsCold = new SummaryStatistics();
+ // SummaryStatistics joinStatsWarm = new SummaryStatistics();
+ //
+ // //prepare eppview data
+ // EPViewPerformance eppview = epview.getPerformance();
+ // EPViewPerformanceData askCold = new EPViewPerformanceData("Cold ASK Tests","#1f77b4",
+ // new
+ // ArrayList());
+ // EPViewPerformanceData askWarm = new EPViewPerformanceData("WARM ASK Tests","#2ca02c",
+ // new
+ // ArrayList());
+ // EPViewPerformanceData joinCold = new EPViewPerformanceData("Cold JOIN Tests","#1f77b4",
+ // new
+ // ArrayList());
+ // EPViewPerformanceData joinWarm = new EPViewPerformanceData("Warm JOIN Tests","#2ca02c",
+ // new
+ // ArrayList());
+ //
+ // ArrayList askdata= new ArrayList();
+ // askdata.add(askCold);
+ // askdata.add(askWarm);
+ // ArrayList joindata= new ArrayList();
+ // joindata.add(joinCold);
+ // joindata.add(joinWarm);
+ //
+ // eppview.setAsk(askdata);
+ // eppview.setJoin(joindata);
+ //
+ //
+ // Map map = pres.getResults();
+ // int limit =0 ;
+ //
+ // for(Entry ent: map.entrySet()){
+ // PSingleResult res = ent.getValue();
+ // if(ent.getKey().toString().startsWith("ASK")){
+ // askStatsCold.addValue(res.getCold().getClosetime()/(double)1000);
+ // askStatsWarm.addValue(res.getWarm().getClosetime()/(double)1000);
+ //
+ // String key = ent.getKey().toString().replaceAll("ASK", "").toLowerCase();
+ //
+ //
+ // askCold.getData().add(new
+ // EPViewPerformanceDataValues(key,res.getCold().getClosetime()/(double)1000));
+ // askWarm.getData().add(new
+ // EPViewPerformanceDataValues(key,res.getWarm().getClosetime()/(double)1000));
+ // }else if(ent.getKey().toString().startsWith("JOIN")){
+ // joinStatsCold.addValue(res.getCold().getClosetime()/(double)1000);
+ // joinStatsWarm.addValue(res.getCold().getClosetime()/(double)1000);
+ //
+ // String key = ent.getKey().toString().replaceAll("JOIN", "").toLowerCase();
+ //
+ // joinCold.getData().add(new
+ // EPViewPerformanceDataValues(key,res.getCold().getClosetime()/(double)1000));
+ // joinWarm.getData().add(new
+ // EPViewPerformanceDataValues(key,res.getWarm().getClosetime()/(double)1000));
+ // }else if(ent.getKey().toString().startsWith("LIMIT")){
+ // int sol = res.getCold().getSolutions();
+ // if(Math.max(limit, sol)==sol){
+ // limit = sol;
+ // }
+ // sol = res.getWarm().getSolutions();
+ // if(Math.max(limit, sol)==sol){
+ // limit = sol;
+ // }
+ // }
+ // }
+ //
+ //
+ // //Update pview data
+ // pview.setAskMeanCold(askStatsCold.getMean());
+ // pview.setAskMeanWarm(askStatsWarm.getMean());
+ // pview.setJoinMeanCold(joinStatsCold.getMean());
+ // pview.setJoinMeanWarm(joinStatsWarm.getMean());
+ //
+ //
+ // System.out.println(pview);
+ // System.out.println(epview);
+ // _db.update(pview);
+ // _db.update(epview);
+ //
+ // return true;
+ }
+
+ private DiscoverabilityView getView(Endpoint ep) {
+ DiscoverabilityView view = null;
+ List views = new ArrayList();
+ if (_db != null) {
+ views = _db.getResults(ep, DiscoverabilityView.class, DiscoverabilityView.SCHEMA$);
+ }
+ if (views.size() != 1) {
+ log.warn("We have {} FeatureView, expected was 1", views.size());
+ }
+ if (views.size() == 0) {
+ view = new DiscoverabilityView();
+ view.setEndpoint(ep);
+ view.setSD(false);
+ view.setVoID(false);
+ view.setServerName("missing");
+ view.setLastUpdate(-1L);
+ if (_db != null) _db.insert(view);
+ } else {
+ view = views.get(0);
}
+ return view;
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/EndpointComparator.java b/backend/src/main/java/sparqles/analytics/EndpointComparator.java
index 32aad11e..1b0f16a7 100644
--- a/backend/src/main/java/sparqles/analytics/EndpointComparator.java
+++ b/backend/src/main/java/sparqles/analytics/EndpointComparator.java
@@ -5,20 +5,20 @@
public class EndpointComparator implements Comparator {
- @Override
- public int compare(Endpoint o1, Endpoint o2) {
- int diff = o1.getUri().toString().compareToIgnoreCase(o2.getUri().toString());
+ @Override
+ public int compare(Endpoint o1, Endpoint o2) {
+ int diff = o1.getUri().toString().compareToIgnoreCase(o2.getUri().toString());
- // if(diff == 0)
- // diff= o1.getDatasets().size()- o2.getDatasets().size();
+ // if(diff == 0)
+ // diff= o1.getDatasets().size()- o2.getDatasets().size();
- // if(diff == 0){
- // for(Dataset d: o1.getDatasets()){
- // if(!o2.getDatasets().contains(d)){
- // return -1;
- // }
- // }
- // }
- return diff;
- }
+ // if(diff == 0){
+ // for(Dataset d: o1.getDatasets()){
+ // if(!o2.getDatasets().contains(d)){
+ // return -1;
+ // }
+ // }
+ // }
+ return diff;
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/FAnalyser.java b/backend/src/main/java/sparqles/analytics/FAnalyser.java
index ef7e80f7..9ae49c76 100644
--- a/backend/src/main/java/sparqles/analytics/FAnalyser.java
+++ b/backend/src/main/java/sparqles/analytics/FAnalyser.java
@@ -16,82 +16,80 @@
import sparqles.utils.MongoDBManager;
public class FAnalyser extends Analytics {
- private static final Logger log = LoggerFactory.getLogger(FAnalyser.class);
+ private static final Logger log = LoggerFactory.getLogger(FAnalyser.class);
- public FAnalyser(MongoDBManager db) {
- super(db);
- }
+ public FAnalyser(MongoDBManager db) {
+ super(db);
+ }
- @Override
- public boolean analyse(FResult pres) {
- log.trace("[Analytics] {}", pres);
-
- Endpoint ep = pres.getEndpointResult().getEndpoint();
-
- InteroperabilityView fview = getView(ep);
- EPView epview = getEPView(ep);
-
- List sparql1Feat = new ArrayList();
- List sparql11Feat = new ArrayList();
-
- int SPARQL1 = 0, SPARQL11 = 0;
- for (Entry ent : pres.getResults().entrySet()) {
- String key = ent.getKey().toString();
- Run run = ent.getValue().getRun();
-
- String q = SpecificFTask.valueOf(key).toString().toLowerCase();
- if (key.contains("SPARQL1_")) {
- q = q.replaceAll("sparql10/", "").replace(".rq", "");
- EPViewInteroperabilityData t =
- new EPViewInteroperabilityData(q, false, run.getException());
-
- if (run.getException() == null) {
- SPARQL1++;
- t.setValue(true);
- }
- sparql1Feat.add(t);
- } else if (key.contains("SPARQL11_")) {
- q = q.replaceAll("sparql11/", "").replace(".rq", "");
- EPViewInteroperabilityData t =
- new EPViewInteroperabilityData(q, false, run.getException());
-
- if (run.getException() == null) {
- SPARQL11++;
- t.setValue(true);
- }
- sparql11Feat.add(t);
- }
- }
+ @Override
+ public boolean analyse(FResult pres) {
+ log.trace("[Analytics] {}", pres);
- fview.setNbCompliantSPARQL1Features(SPARQL1);
- fview.setNbCompliantSPARQL11Features(SPARQL11);
- epview.getInteroperability().setSPARQL1Features(sparql1Feat);
- epview.getInteroperability().setSPARQL11Features(sparql11Feat);
+ Endpoint ep = pres.getEndpointResult().getEndpoint();
- fview.setLastUpdate(pres.getEndpointResult().getEnd());
+ InteroperabilityView fview = getView(ep);
+ EPView epview = getEPView(ep);
- _db.update(fview);
- _db.update(epview);
- return true;
- }
+ List sparql1Feat = new ArrayList();
+ List sparql11Feat = new ArrayList();
- private InteroperabilityView getView(Endpoint ep) {
- InteroperabilityView view = null;
- List views =
- _db.getResults(ep, InteroperabilityView.class, InteroperabilityView.SCHEMA$);
- if (views.size() != 1) {
- log.warn("We have {} FeatureView, expected was 1", views.size());
+ int SPARQL1 = 0, SPARQL11 = 0;
+ for (Entry ent : pres.getResults().entrySet()) {
+ String key = ent.getKey().toString();
+ Run run = ent.getValue().getRun();
+
+ String q = SpecificFTask.valueOf(key).toString().toLowerCase();
+ if (key.contains("SPARQL1_")) {
+ q = q.replaceAll("sparql10/", "").replace(".rq", "");
+ EPViewInteroperabilityData t = new EPViewInteroperabilityData(q, false, run.getException());
+
+ if (run.getException() == null) {
+ SPARQL1++;
+ t.setValue(true);
}
- if (views.size() == 0) {
- view = new InteroperabilityView();
- view.setEndpoint(ep);
- view.setNbCompliantSPARQL11Features(-1);
- view.setNbCompliantSPARQL1Features(-1);
- _db.insert(view);
-
- } else {
- view = views.get(0);
+ sparql1Feat.add(t);
+ } else if (key.contains("SPARQL11_")) {
+ q = q.replaceAll("sparql11/", "").replace(".rq", "");
+ EPViewInteroperabilityData t = new EPViewInteroperabilityData(q, false, run.getException());
+
+ if (run.getException() == null) {
+ SPARQL11++;
+ t.setValue(true);
}
- return view;
+ sparql11Feat.add(t);
+ }
+ }
+
+ fview.setNbCompliantSPARQL1Features(SPARQL1);
+ fview.setNbCompliantSPARQL11Features(SPARQL11);
+ epview.getInteroperability().setSPARQL1Features(sparql1Feat);
+ epview.getInteroperability().setSPARQL11Features(sparql11Feat);
+
+ fview.setLastUpdate(pres.getEndpointResult().getEnd());
+
+ _db.update(fview);
+ _db.update(epview);
+ return true;
+ }
+
+ private InteroperabilityView getView(Endpoint ep) {
+ InteroperabilityView view = null;
+ List views =
+ _db.getResults(ep, InteroperabilityView.class, InteroperabilityView.SCHEMA$);
+ if (views.size() != 1) {
+ log.warn("We have {} FeatureView, expected was 1", views.size());
+ }
+ if (views.size() == 0) {
+ view = new InteroperabilityView();
+ view.setEndpoint(ep);
+ view.setNbCompliantSPARQL11Features(-1);
+ view.setNbCompliantSPARQL1Features(-1);
+ _db.insert(view);
+
+ } else {
+ view = views.get(0);
}
+ return view;
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/IndexViewAnalytics.java b/backend/src/main/java/sparqles/analytics/IndexViewAnalytics.java
index cf4314d6..dbb6dc70 100644
--- a/backend/src/main/java/sparqles/analytics/IndexViewAnalytics.java
+++ b/backend/src/main/java/sparqles/analytics/IndexViewAnalytics.java
@@ -1,15 +1,8 @@
package sparqles.analytics;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.TreeMap;
-import java.util.TreeSet;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -17,6 +10,7 @@
import sparqles.avro.analytics.EPView;
import sparqles.avro.analytics.EPViewAvailability;
import sparqles.avro.analytics.EPViewAvailabilityDataPoint;
+import sparqles.avro.analytics.EPViewCalculation;
import sparqles.avro.analytics.EPViewDiscoverability;
import sparqles.avro.analytics.EPViewDiscoverabilityData;
import sparqles.avro.analytics.EPViewInteroperability;
@@ -26,6 +20,9 @@
import sparqles.avro.analytics.EPViewPerformanceDataValues;
import sparqles.avro.analytics.Index;
import sparqles.avro.analytics.IndexAvailabilityDataPoint;
+import sparqles.avro.analytics.IndexViewCalculation;
+import sparqles.avro.analytics.IndexViewCalculationData;
+import sparqles.avro.analytics.IndexViewCalculationDataValues;
import sparqles.avro.analytics.IndexViewDiscoverability;
import sparqles.avro.analytics.IndexViewDiscoverabilityData;
import sparqles.avro.analytics.IndexViewDiscoverabilityDataValues;
@@ -35,550 +32,702 @@
import sparqles.avro.analytics.IndexViewPerformance;
import sparqles.avro.analytics.IndexViewPerformanceData;
import sparqles.avro.analytics.IndexViewPerformanceDataValues;
-import sparqles.core.CONSTANTS;
+import sparqles.core.SPARQLESProperties;
import sparqles.core.Task;
import sparqles.utils.MongoDBManager;
public class IndexViewAnalytics implements Task {
- private static final Logger log = LoggerFactory.getLogger(IndexViewAnalytics.class);
- final int askCold = 0, askWarm = 1, joinCold = 2, joinWarm = 3;
- final int sparql1_solMods = 0,
- sparql1_com = 1,
- sparql1_graph = 2,
- sparql11_agg = 3,
- sparql11_filter = 4,
- sparql11_other = 5;
- private MongoDBManager _dbm;
-
- @Override
- public Index call() throws Exception {
-
- // get the index view
- Collection idxs = _dbm.get(Index.class, Index.SCHEMA$);
- Index idx = null;
- if (idxs.size() == 0) {
- idx = createIndex();
- _dbm.insert(idx);
- } else if (idxs.size() > 1) {
- // log.warn("Too many results");
- } else idx = idxs.iterator().next();
-
- // get epview
- Collection epviews = _dbm.get(EPView.class, EPView.SCHEMA$);
- log.info("Found {} idx views and {} epviews", idxs.size(), epviews.size());
-
- // Prepare aggregated analytics
- Map weekHist = new HashMap();
-
- SummaryStatistics[] perfStats = {
- new SummaryStatistics(),
- new SummaryStatistics(),
- new SummaryStatistics(),
- new SummaryStatistics()
- };
-
- SimpleHistogram[] interStats = {
- new SimpleHistogram(),
- new SimpleHistogram(),
- new SimpleHistogram(),
- new SimpleHistogram(),
- new SimpleHistogram(),
- new SimpleHistogram()
- };
-
- Count[] discoStats = {new Count(), new Count()};
-
- // iterate over all epviews and analyse them
- for (EPView epv : epviews) {
- System.err.println(epv);
- // analyse availability
- analyseAvailability(epv.getAvailability(), weekHist);
-
- // analyse performance
- analysePerformance(epv.getPerformance(), perfStats);
-
- // analyse interoperability
- analyseInteroperability(epv.getInteroperability(), interStats);
-
- // analyse interoperability
- analyseDiscoverability(epv.getDiscoverability(), discoStats);
- }
-
- // update the index view
- updateAvailabilityStats(idx, weekHist);
+ private static final Logger log = LoggerFactory.getLogger(IndexViewAnalytics.class);
+
+ private MongoDBManager _dbm;
+ final int askCold = 0, askWarm = 1, joinCold = 2, joinWarm = 3;
+ final int sparql1_solMods = 0,
+ sparql1_com = 1,
+ sparql1_graph = 2,
+ sparql11_agg = 3,
+ sparql11_filter = 4,
+ sparql11_other = 5;
+
+ @Override
+ public Index call() throws Exception {
+
+ // get the index view
+ Collection idxs = _dbm.get(Index.class, Index.SCHEMA$);
+ Index idx = null;
+ if (idxs == null || idxs.isEmpty()) {
+ idx = createIndex();
+ _dbm.insert(idx);
+ } else if (idxs.size() > 1) {
+ log.error("Too many results");
+ } else {
+ idx = idxs.iterator().next();
+ }
- // update performance stats
- updatePerformanceStats(idx, perfStats);
+ // get epview
+ Collection epviews = _dbm.get(EPView.class, EPView.SCHEMA$);
+ log.info("Found {} idx views and {} epviews", idxs.size(), epviews.size());
- // update interoperability stats
- updateInteroperability(idx, interStats);
+ // Prepare aggregated analytics
+ Map weekHist = new HashMap();
- updateDiscoverability(idx, discoStats);
+ SummaryStatistics[] perfStats = {
+ new SummaryStatistics(),
+ new SummaryStatistics(),
+ new SummaryStatistics(),
+ new SummaryStatistics()
+ };
+ var thresholdStats = new DescriptiveStatistics();
- log.info("Updated view {}", idx);
- _dbm.update(idx);
+ SimpleHistogram[] interStats = {
+ new SimpleHistogram(),
+ new SimpleHistogram(),
+ new SimpleHistogram(),
+ new SimpleHistogram(),
+ new SimpleHistogram(),
+ new SimpleHistogram()
+ };
- return idx;
- }
+ Count[] discoStats = {new Count(), new Count()};
- private void analyseDiscoverability(
- EPViewDiscoverability discoverability, Count[] discoStats) {
- discoStats[1].add(discoverability.getServerName().toString());
-
- boolean sd = false, voidd = false;
- if (discoverability.getSDDescription().size() != 0) {
- for (EPViewDiscoverabilityData d : discoverability.getSDDescription()) {
- if (d.getValue()) {
- discoStats[0].add("sd");
- sd = true;
- break;
- }
- }
- }
- if (discoverability.getVoIDDescription().size() != 0) {
- for (EPViewDiscoverabilityData d : discoverability.getVoIDDescription()) {
- if (d.getValue()) {
- discoStats[0].add("void");
- voidd = true;
- break;
- }
- }
- }
- if (!voidd && !sd) {
- discoStats[0].add("no");
- }
+ Count[] calcStats = {new Count(), new Count(), new Count()};
- discoStats[0].add("total");
- }
+ // analyse availability
+ recalculateAvailabilityMonthly();
- private void updateDiscoverability(Index idx, Count[] object) {
-
- IndexViewDiscoverability iv = idx.getDiscoverability();
- Count server = object[1];
-
- List l = new ArrayList();
- List lv =
- new ArrayList();
-
- TreeSet set =
- new TreeSet(
- new Comparator() {
-
- @Override
- public int compare(
- IndexViewDiscoverabilityDataValues o1,
- IndexViewDiscoverabilityDataValues o2) {
- int diff = o1.getValue().compareTo(o2.getValue());
- if (diff == 0) diff = -1;
- return diff;
- }
- });
-
- for (String k : server.keySet()) {
- set.add(
- new IndexViewDiscoverabilityDataValues(
- k, server.get(k) / (double) server.getTotal()));
- }
+ // iterate over all epviews and analyse them
+ for (EPView epv : epviews) {
+ log.trace("EPView: {}", epv);
+ analyseAvailability(epv.getAvailability(), weekHist);
- for (IndexViewDiscoverabilityDataValues d : set.descendingSet()) {
- lv.add(d);
- }
- l.add(new IndexViewDiscoverabilityData("Server Names", lv));
- iv.setServerName(l);
-
- Count stats = object[0];
- int v = 0;
- if (stats.containsKey("no")) {
- v = stats.get("no");
- iv.setNoDescription(v / (double) stats.get("total"));
- } else iv.setNoDescription(0D);
-
- v = stats.getOrDefault("sd", 0);
-
- Integer totalVal = stats.get("total");
- if (totalVal != null) {
- iv.setSDDescription(v / (double) totalVal);
-
- v = stats.getOrDefault("void", 0);
- iv.setVoIDDescription(v / (double) totalVal);
- } else {
- log.error("Total value is missing");
- iv.setSDDescription(-1.0);
- iv.setVoIDDescription(-1.0);
- }
- }
+ // analyse performance
+ analysePerformance(epv.getPerformance(), perfStats, thresholdStats);
- private void updateInteroperability(Index idx, SimpleHistogram[] interStats) {
- IndexViewInteroperability iv = idx.getInteroperability();
+ // analyse interoperability
+ analyseInteroperability(epv.getInteroperability(), interStats);
- List v = new ArrayList();
- iv.setData(v);
+ // analyse discoverability
+ analyseDiscoverability(epv.getDiscoverability(), discoStats);
- v.add(updateSPARQL1(interStats));
- v.add(updateSPARQL11(interStats));
+ // analyse calculation
+ analyseCalculation(epv.getCalculation(), calcStats);
}
- private IndexViewInterData updateSPARQL11(SimpleHistogram[] interStats) {
- IndexViewInterData ivd = new IndexViewInterData();
- ivd.setColor("#2ca02c");
- ivd.setKey("SPARQL 1.1");
-
- ArrayList v = new ArrayList();
- // sparql1 mod
- double perc =
- interStats[sparql11_agg].bin[3] / (double) interStats[sparql11_agg].sampleSize;
- v.add(new IndexViewInterDataValues("Aggregate", perc));
-
- // sparql1 com
- perc = interStats[sparql11_filter].bin[3] / (double) interStats[sparql11_filter].sampleSize;
- v.add(new IndexViewInterDataValues("Filter", perc));
-
- // sparql1 graph
- perc = interStats[sparql11_other].bin[3] / (double) interStats[sparql11_other].sampleSize;
- v.add(new IndexViewInterDataValues("Other", perc));
-
- ivd.setData(v);
+ // update the index view
+ updateAvailabilityStats(idx, weekHist);
+
+ // update performance stats
+ updatePerformanceStats(idx, perfStats, thresholdStats);
+
+ // update interoperability stats
+ updateInteroperability(idx, interStats);
+
+ // update discoverability stats
+ updateDiscoverability(idx, discoStats);
+
+ // update calculation stats
+ updateCalculation(idx, calcStats);
+
+ log.info("Updated view {}", idx);
+ _dbm.update(idx);
+
+ return idx;
+ }
+
+ private void recalculateAvailabilityMonthly() {
+ log.info("Recalculating availability monthly");
+ AEvol.recalculateMonthly(_dbm);
+ }
+
+ private void analyseCalculation(EPViewCalculation calculation, Count[] calcStats) {
+ double coherence = calculation.getCoherence();
+ if (coherence < 0)
+ ; // DO NOTHING
+ else if (coherence < 0.25) calcStats[1].add("[0.00-0.25[");
+ else if (coherence < 0.5) calcStats[1].add("[0.25-0.50[");
+ else if (coherence < 0.75) calcStats[1].add("[0.50-0.75[");
+ else if (coherence < 0.95) calcStats[1].add("[0.75-0.95[");
+ else calcStats[1].add("[0.95-1.00]");
+
+ double rs = calculation.getRS();
+ if (rs < 0)
+ ; // DO NOTHING
+ else if (rs < 10) calcStats[2].add("[0-10[");
+ else if (rs < 100) calcStats[2].add("[10-100[");
+ else if (rs < 1000) calcStats[2].add("[100-1000[");
+ else if (rs < 10000) calcStats[2].add("[1000-10000[");
+ else calcStats[2].add("[10000-)");
+
+ if (!calculation.getVoID().toString().equals("")) calcStats[0].add("VoID");
+ if (calculation.getVoIDPart()) calcStats[0].add("VoIDPart");
+ if (!calculation.getSD().toString().equals("")) calcStats[0].add("SD");
+ if (calculation.getSDPart()) calcStats[0].add("SDPart");
+ if (calculation.getCoherence() >= 0) calcStats[0].add("Coherence");
+ if (calculation.getRS() >= 0) calcStats[0].add("RS");
+
+ calcStats[0].add("total");
+ }
+
+ private void updateCalculation(Index idx, Count[] object) {
+
+ IndexViewCalculation iv = idx.getCalculation();
+
+ Count coherence = object[1];
+
+ List l1 = new ArrayList();
+ List lv1 = new ArrayList();
+
+ TreeSet set1 =
+ new TreeSet(
+ new Comparator() {
+
+ @Override
+ public int compare(
+ IndexViewCalculationDataValues o1, IndexViewCalculationDataValues o2) {
+ int diff =
+ Comparator.comparingDouble(
+ (IndexViewCalculationDataValues value) -> value.getValue())
+ .compare(o1, o2);
+ if (diff == 0) diff = -1;
+ return diff;
+ }
+ });
+
+ for (String k : coherence.keySet()) {
+ set1.add(
+ new IndexViewCalculationDataValues(k, coherence.get(k) / (double) coherence.getTotal()));
+ }
- return ivd;
+ for (IndexViewCalculationDataValues d : set1.descendingSet()) {
+ lv1.add(d);
}
+ l1.add(new IndexViewCalculationData("Coherence", lv1));
- private IndexViewInterData updateSPARQL1(SimpleHistogram[] interStats) {
- IndexViewInterData ivd = new IndexViewInterData();
- ivd.setColor("#1f77b4");
- ivd.setKey("SPARQL 1.0");
+ iv.setCoherences(l1);
- ArrayList v = new ArrayList();
- // sparql1 mod
- double perc =
- interStats[sparql1_solMods].bin[3]
- / (double) interStats[sparql1_solMods].sampleSize;
- v.add(new IndexViewInterDataValues("Solution Modifiers", perc));
+ Count rs = object[2];
- // sparql1 com
- perc = interStats[sparql1_com].bin[3] / (double) interStats[sparql1_com].sampleSize;
- v.add(new IndexViewInterDataValues("Common Operators and Filters", perc));
+ List l2 = new ArrayList();
+ List lv2 = new ArrayList();
- // sparql1 graph
- perc = interStats[sparql1_graph].bin[3] / (double) interStats[sparql1_graph].sampleSize;
- v.add(new IndexViewInterDataValues("Graph and other", perc));
+ TreeSet set2 =
+ new TreeSet(
+ new Comparator() {
- ivd.setData(v);
+ @Override
+ public int compare(
+ IndexViewCalculationDataValues o1, IndexViewCalculationDataValues o2) {
+ int diff =
+ Comparator.comparingDouble(
+ (IndexViewCalculationDataValues value) -> value.getValue())
+ .compare(o1, o2);
+ if (diff == 0) diff = -1;
+ return diff;
+ }
+ });
- return ivd;
+ for (String k : rs.keySet()) {
+ set2.add(new IndexViewCalculationDataValues(k, rs.get(k) / (double) rs.getTotal()));
}
- private void analyseInteroperability(
- EPViewInteroperability interoperability, SimpleHistogram[] interStats) {
- boolean[] all = new boolean[6];
- Arrays.fill(all, true);
-
- for (EPViewInteroperabilityData d : interoperability.getSPARQL1Features()) {
-
- String l = d.getLabel().toString();
-
- boolean bv = d.getValue();
- /*
- SEL[.]*ORDERBY*OFFSET
- SEL[.]*ORDERBY-ASC
- SEL[.]*ORDERBY-DESC
- SEL[.]*ORDERBY
- SEL-DISTINCT[.]
- SEL-REDUCED[.]
- */
- if (l.contains("orderby") || l.contains("distinct") || l.contains("reduced")) {
- all[sparql1_solMods] = all[sparql1_solMods] && bv;
- }
-
- /*
- SEL[.]
- SEL[JOIN]
- SEL[OPT]
- SEL[UNION]
- -- matches fil --
- SEL[FIL(!BOUND)]
- SEL[FIL(BLANK)]
- SEL[FIL(BOOL)]
- SEL[FIL(IRI)]
- SEL[FIL(NUM)]
- SEL[FIL(REGEX)]
- SEL[FIL(REGEX-i)]
- SEL[FIL(STR)]
- SEL[BNODE] -> bnode
- SEL[EMPTY] -> empty
- */
- else if (l.contains("fil")
- || l.contains("bnode")
- || l.contains("empty")
- || l.contains("union")
- || l.contains("opt")
- || l.contains("join")
- || l.contains("sel[.]")) {
-
- all[sparql1_com] = all[sparql1_com] && bv;
-
- }
- /*
- SEL[FROM]
- SEL[GRAPH]
- SEL[GRAPH;JOIN]
- SEL[GRAPH;UNION]
- CON[.]
- CON[JOIN]
- CON[OPT]
- ASK[.]
- */
- else if (l.contains("graph")
- || l.contains("con")
- || l.contains("ask")
- || l.contains("from")) {
- all[sparql1_graph] = all[sparql1_graph] && bv;
- } else {
- log.info("Could not match {}", l);
- }
- }
- for (EPViewInteroperabilityData d : interoperability.getSPARQL11Features()) {
- String l = d.getLabel().toString();
- boolean bv = d.getValue();
- /*
- Aggregate
- SEL[AVG]*GROUPBY
- SEL[AVG]
- SEL[COUNT]*GROUPBY
- SEL[MAX]
- SEL[MIN]
- SEL[MINUS]
- SEL[SUM]
- */
- if (l.contains("avg")
- || l.contains("count")
- || l.contains("max")
- || l.contains("min")
- || l.contains("minus")
- || l.contains("sum")) {
- all[sparql11_agg] = all[sparql11_agg] && bv;
- }
- /*
- Filter
- SEL[FIL(!EXISTS)]
- SEL[FIL(ABS)]
- SEL[FIL(CONTAINS)]
- SEL[FIL(EXISTS)]
- SEL[FIL(START)]
- */
- else if (l.contains("fil") || l.contains("distinct") || l.contains("reduced")) {
- all[sparql11_filter] = all[sparql11_filter] && bv;
- }
- /*
- Other
- ASK[FIL(!IN)]
- CON-[.]
- SEL[BIND]
- SEL[PATHS]
- SEL[SERVICE]
- SEL[SUBQ]
- SEL[SUBQ;GRAPH]
- SEL[VALUES]
- */
- else if (l.contains("ask")
- || l.contains("con")
- || l.contains("bind")
- || l.contains("paths")
- || l.contains("service")
- || l.contains("subq")
- || l.contains("values")) {
- all[sparql11_other] = all[sparql11_other] && bv;
- } else {
- log.info("Could not match {}", l);
- }
- }
-
- boolean update = false;
- for (int i = 0; i < all.length; i++) {
- update = update || all[i];
+ for (IndexViewCalculationDataValues d : set2.descendingSet()) {
+ lv2.add(d);
+ }
+ l2.add(new IndexViewCalculationData("RS", lv2));
+
+ iv.setRss(l2);
+
+ Count stats = object[0];
+ double totalVal = (double) stats.get("total");
+ iv.setVoID(stats.get("VoID") / totalVal);
+ iv.setVoIDPart(stats.get("VoIDPart") / totalVal);
+ iv.setSD(stats.get("SD") / totalVal);
+ iv.setSDPart(stats.get("SDPart") / totalVal);
+ iv.setCoherence(stats.get("Coherence") / totalVal);
+ iv.setRS(stats.get("RS") / totalVal);
+ }
+
+ private void analyseDiscoverability(
+ EPViewDiscoverability discoverability, Count[] discoStats) {
+ discoStats[1].add(discoverability.getServerName().toString());
+
+ boolean sd = false, voidd = false;
+ if (discoverability.getSDDescription().size() != 0) {
+ for (EPViewDiscoverabilityData d : discoverability.getSDDescription()) {
+ if (d.getValue()) {
+ discoStats[0].add("sd");
+ sd = true;
+ break;
}
-
- if (update) {
- for (int i = 0; i < all.length; i++) {
- if (all[i]) interStats[i].add(1D);
- else interStats[i].add(0D);
- }
+ }
+ }
+ if (discoverability.getVoIDDescription().size() != 0) {
+ for (EPViewDiscoverabilityData d : discoverability.getVoIDDescription()) {
+ if (d.getValue()) {
+ discoStats[0].add("void");
+ voidd = true;
+ break;
}
- // System.out.println(Arrays.toString(interStats));
-
+ }
+ }
+ if (!voidd && !sd) {
+ discoStats[0].add("no");
}
- private void analysePerformance(EPViewPerformance performance, SummaryStatistics[] perfStats) {
- update(performance.getAsk(), perfStats[askCold], perfStats[askWarm]);
- update(performance.getJoin(), perfStats[joinCold], perfStats[joinWarm]);
+ discoStats[0].add("total");
+ }
+
+ private void updateDiscoverability(Index idx, Count[] object) {
+
+ IndexViewDiscoverability iv = idx.getDiscoverability();
+ Count server = object[1];
+
+ List l = new ArrayList();
+ List lv =
+ new ArrayList();
+
+ TreeSet set =
+ new TreeSet(
+ new Comparator() {
+
+ @Override
+ public int compare(
+ IndexViewDiscoverabilityDataValues o1, IndexViewDiscoverabilityDataValues o2) {
+ int diff =
+ Comparator.comparingDouble(
+ (IndexViewDiscoverabilityDataValues value) -> value.getValue())
+ .compare(o1, o2);
+ if (diff == 0) diff = -1;
+ return diff;
+ }
+ });
+
+ for (String k : server.keySet()) {
+ set.add(
+ new IndexViewDiscoverabilityDataValues(k, server.get(k) / (double) server.getTotal()));
}
- private void analyseAvailability(
- EPViewAvailability availability, Map weekHist) {
- for (EPViewAvailabilityDataPoint value : availability.getData().getValues()) {
- update(value, weekHist);
- }
+ for (IndexViewDiscoverabilityDataValues d : set.descendingSet()) {
+ lv.add(d);
+ }
+ l.add(new IndexViewDiscoverabilityData("Server Names", lv));
+ iv.setServerName(l);
+
+ Count stats = object[0];
+ int v = 0;
+ if (stats.containsKey("no")) {
+ v = stats.get("no");
+ iv.setNoDescription(v / (double) stats.get("total"));
+ } else iv.setNoDescription(0D);
+
+ v = stats.getOrDefault("sd", 0);
+
+ Integer totalVal = stats.get("total");
+ if (totalVal != null) {
+ iv.setSDDescription(v / (double) totalVal);
+
+ v = stats.getOrDefault("void", 0);
+ iv.setVoIDDescription(v / (double) totalVal);
+ } else {
+ log.error("Total value is missing");
+ iv.setSDDescription(-1.0);
+ iv.setVoIDDescription(-1.0);
+ }
+ }
+
+ private void updateInteroperability(Index idx, SimpleHistogram[] interStats) {
+ IndexViewInteroperability iv = idx.getInteroperability();
+
+ List v = new ArrayList();
+ iv.setData(v);
+
+ v.add(updateSPARQL1(interStats));
+ v.add(updateSPARQL11(interStats));
+ }
+
+ private IndexViewInterData updateSPARQL11(SimpleHistogram[] interStats) {
+ IndexViewInterData ivd = new IndexViewInterData();
+ ivd.setColor("#2ca02c");
+ ivd.setKey("SPARQL 1.1");
+
+ ArrayList v = new ArrayList();
+ // sparql1 mod
+ double perc = interStats[sparql11_agg].bin[3] / (double) interStats[sparql11_agg].sampleSize;
+ v.add(new IndexViewInterDataValues("Aggregate", perc));
+
+ // sparql1 com
+ perc = interStats[sparql11_filter].bin[3] / (double) interStats[sparql11_filter].sampleSize;
+ v.add(new IndexViewInterDataValues("Filter", perc));
+
+ // sparql1 graph
+ perc = interStats[sparql11_other].bin[3] / (double) interStats[sparql11_other].sampleSize;
+ v.add(new IndexViewInterDataValues("Other", perc));
+
+ ivd.setData(v);
+
+ return ivd;
+ }
+
+ private IndexViewInterData updateSPARQL1(SimpleHistogram[] interStats) {
+ IndexViewInterData ivd = new IndexViewInterData();
+ ivd.setColor("#1f77b4");
+ ivd.setKey("SPARQL 1.0");
+
+ ArrayList v = new ArrayList();
+ // sparql1 mod
+ double perc =
+ interStats[sparql1_solMods].bin[3] / (double) interStats[sparql1_solMods].sampleSize;
+ v.add(new IndexViewInterDataValues("Solution Modifiers", perc));
+
+ // sparql1 com
+ perc = interStats[sparql1_com].bin[3] / (double) interStats[sparql1_com].sampleSize;
+ v.add(new IndexViewInterDataValues("Common Operators and Filters", perc));
+
+ // sparql1 graph
+ perc = interStats[sparql1_graph].bin[3] / (double) interStats[sparql1_graph].sampleSize;
+ v.add(new IndexViewInterDataValues("Graph and other", perc));
+
+ ivd.setData(v);
+
+ return ivd;
+ }
+
+ private void analyseInteroperability(
+ EPViewInteroperability interoperability, SimpleHistogram[] interStats) {
+ boolean[] all = new boolean[6];
+ Arrays.fill(all, true);
+
+ for (EPViewInteroperabilityData d : interoperability.getSPARQL1Features()) {
+
+ String l = d.getLabel().toString();
+
+ boolean bv = d.getValue();
+ /*
+ SEL[.]*ORDERBY*OFFSET
+ SEL[.]*ORDERBY-ASC
+ SEL[.]*ORDERBY-DESC
+ SEL[.]*ORDERBY
+ SEL-DISTINCT[.]
+ SEL-REDUCED[.]
+ */
+ if (l.contains("orderby") || l.contains("distinct") || l.contains("reduced")) {
+ all[sparql1_solMods] = all[sparql1_solMods] && bv;
+ }
+
+ /*
+ SEL[.]
+ SEL[JOIN]
+ SEL[OPT]
+ SEL[UNION]
+ -- matches fil --
+ SEL[FIL(!BOUND)]
+ SEL[FIL(BLANK)]
+ SEL[FIL(BOOL)]
+ SEL[FIL(IRI)]
+ SEL[FIL(NUM)]
+ SEL[FIL(REGEX)]
+ SEL[FIL(REGEX-i)]
+ SEL[FIL(STR)]
+ SEL[BNODE] -> bnode
+ SEL[EMPTY] -> empty
+ */
+ else if (l.contains("fil")
+ || l.contains("bnode")
+ || l.contains("empty")
+ || l.contains("union")
+ || l.contains("opt")
+ || l.contains("join")
+ || l.contains("sel[.]")) {
+
+ all[sparql1_com] = all[sparql1_com] && bv;
+
+ }
+ /*
+ SEL[FROM]
+ SEL[GRAPH]
+ SEL[GRAPH;JOIN]
+ SEL[GRAPH;UNION]
+ CON[.]
+ CON[JOIN]
+ CON[OPT]
+ ASK[.]
+ */
+ else if (l.contains("graph")
+ || l.contains("con")
+ || l.contains("ask")
+ || l.contains("from")) {
+ all[sparql1_graph] = all[sparql1_graph] && bv;
+ } else {
+ log.info("Could not match {}", l);
+ }
+ }
+ for (EPViewInteroperabilityData d : interoperability.getSPARQL11Features()) {
+ String l = d.getLabel().toString();
+ boolean bv = d.getValue();
+ /*
+ Aggregate
+ SEL[AVG]*GROUPBY
+ SEL[AVG]
+ SEL[COUNT]*GROUPBY
+ SEL[MAX]
+ SEL[MIN]
+ SEL[MINUS]
+ SEL[SUM]
+ */
+ if (l.contains("avg")
+ || l.contains("count")
+ || l.contains("max")
+ || l.contains("min")
+ || l.contains("minus")
+ || l.contains("sum")) {
+ all[sparql11_agg] = all[sparql11_agg] && bv;
+ }
+ /*
+ Filter
+ SEL[FIL(!EXISTS)]
+ SEL[FIL(ABS)]
+ SEL[FIL(CONTAINS)]
+ SEL[FIL(EXISTS)]
+ SEL[FIL(START)]
+ */
+ else if (l.contains("fil") || l.contains("distinct") || l.contains("reduced")) {
+ all[sparql11_filter] = all[sparql11_filter] && bv;
+ }
+ /*
+ Other
+ ASK[FIL(!IN)]
+ CON-[.]
+ SEL[BIND]
+ SEL[PATHS]
+ SEL[SERVICE]
+ SEL[SUBQ]
+ SEL[SUBQ;GRAPH]
+ SEL[VALUES]
+ */
+ else if (l.contains("ask")
+ || l.contains("con")
+ || l.contains("bind")
+ || l.contains("paths")
+ || l.contains("service")
+ || l.contains("subq")
+ || l.contains("values")) {
+ all[sparql11_other] = all[sparql11_other] && bv;
+ } else {
+ log.info("Could not match {}", l);
+ }
}
- private void updatePerformanceStats(Index idx, SummaryStatistics[] perfStats) {
- ArrayList data = new ArrayList();
- List l = new ArrayList();
- IndexViewPerformanceData cold = new IndexViewPerformanceData("Cold Tests", "#1f77b4", l);
- l = new ArrayList();
- IndexViewPerformanceData warm = new IndexViewPerformanceData("Warm Tests", "#2ca02c", l);
- data.add(cold);
- data.add(warm);
-
- double v = (perfStats[askCold].getN() == 0) ? -1D : perfStats[askCold].getMean();
- cold.getData().add(new IndexViewPerformanceDataValues("Average ASK", v));
- v = (perfStats[joinCold].getN() == 0) ? -1D : perfStats[joinCold].getMean();
- cold.getData().add(new IndexViewPerformanceDataValues("Average JOIN", v));
-
- v = (perfStats[askWarm].getN() == 0) ? -1D : perfStats[askWarm].getMean();
- warm.getData().add(new IndexViewPerformanceDataValues("Average ASK", v));
- v = (perfStats[joinWarm].getN() == 0) ? -1D : perfStats[joinWarm].getMean();
- warm.getData().add(new IndexViewPerformanceDataValues("Average JOIN", v));
-
- idx.getPerformance().setThreshold(-1L);
- idx.getPerformance().setData(data);
+ boolean update = false;
+ for (int i = 0; i < all.length; i++) {
+ update = update || all[i];
}
- private void updateAvailabilityStats(Index idx, Map weekHist) {
- List aidxs = idx.getAvailability();
- // update availability stats
- for (Entry week : weekHist.entrySet()) {
- SimpleHistogram sh = week.getValue();
-
- int total = sh.sampleSize;
-
- for (AvailabilityIndex aidx : aidxs) {
- int value = 0;
- if (aidx.getKey().equals("[0;5]")) value = sh.bin[0];
- if (aidx.getKey().equals("]5;90]")) value = sh.bin[1];
- if (aidx.getKey().equals("]90;95]")) value = sh.bin[2];
- if (aidx.getKey().equals("]95;100]")) value = sh.bin[3];
-
- boolean exists = false;
- for (IndexAvailabilityDataPoint i : aidx.getValues()) {
- if (i.getX().equals(week.getKey())) {
- exists = true;
- i.setY(value / (double) total);
- }
- }
- if (!exists)
- aidx.getValues()
- .add(
- new IndexAvailabilityDataPoint(
- week.getKey(), value / (double) total));
- }
- }
+ if (update) {
+ for (int i = 0; i < all.length; i++) {
+ if (all[i]) interStats[i].add(1D);
+ else interStats[i].add(0D);
+ }
+ }
+ // System.out.println(Arrays.toString(interStats));
+
+ }
+
+ private void analysePerformance(
+ EPViewPerformance performance,
+ SummaryStatistics[] perfStats,
+ DescriptiveStatistics thresholdStats) {
+ update(performance.getAsk(), perfStats[askCold], perfStats[askWarm]);
+ update(performance.getJoin(), perfStats[joinCold], perfStats[joinWarm]);
+ updateThreshold(performance.getThreshold(), thresholdStats);
+ }
+
+ private void updateThreshold(long threshold, DescriptiveStatistics thresholdStats) {
+ log.debug("Found threshold: {}", threshold);
+ // 100002 is the LIMIT used to detect a threshold
+ if (threshold > 0 && threshold < 100_002) {
+ thresholdStats.addValue(threshold);
}
+ }
- private void update(
- List results, SummaryStatistics cold, SummaryStatistics warm) {
- for (EPViewPerformanceData pdata : results) {
- if (pdata.getKey().toString().contains("Cold")) {
- for (EPViewPerformanceDataValues v : pdata.getData()) {
- cold.addValue(v.getValue());
- }
- }
- if (pdata.getKey().toString().contains("Warm")) {
- for (EPViewPerformanceDataValues v : pdata.getData()) {
- warm.addValue(v.getValue());
- }
- }
+ private void analyseAvailability(
+ EPViewAvailability availability, Map weekHist) {
+ for (EPViewAvailabilityDataPoint value : availability.getData().getValues()) {
+ update(value, weekHist);
+ }
+ }
+
+ private void updatePerformanceStats(
+ Index idx, SummaryStatistics[] perfStats, DescriptiveStatistics thresholdStats) {
+ ArrayList data = new ArrayList();
+ List l = new ArrayList();
+ IndexViewPerformanceData cold = new IndexViewPerformanceData("Cold Tests", "#1f77b4", l);
+ l = new ArrayList();
+ IndexViewPerformanceData warm = new IndexViewPerformanceData("Warm Tests", "#2ca02c", l);
+ data.add(cold);
+ data.add(warm);
+
+ double v = (perfStats[askCold].getN() == 0) ? -1D : perfStats[askCold].getMean();
+ cold.getData().add(new IndexViewPerformanceDataValues("Average ASK", v));
+ v = (perfStats[joinCold].getN() == 0) ? -1D : perfStats[joinCold].getMean();
+ cold.getData().add(new IndexViewPerformanceDataValues("Average JOIN", v));
+
+ v = (perfStats[askWarm].getN() == 0) ? -1D : perfStats[askWarm].getMean();
+ warm.getData().add(new IndexViewPerformanceDataValues("Average ASK", v));
+ v = (perfStats[joinWarm].getN() == 0) ? -1D : perfStats[joinWarm].getMean();
+ warm.getData().add(new IndexViewPerformanceDataValues("Average JOIN", v));
+
+ // median
+ var medianThresh = (long) thresholdStats.getPercentile(50);
+ log.debug("Median thresh: {}", medianThresh);
+ idx.getPerformance().setThreshold(medianThresh);
+ idx.getPerformance().setData(data);
+ }
+
+ private void updateAvailabilityStats(Index idx, Map weekHist) {
+ List aidxs = idx.getAvailability();
+ // update availability stats
+ for (Entry week : weekHist.entrySet()) {
+ SimpleHistogram sh = week.getValue();
+
+ int total = sh.sampleSize;
+
+ for (AvailabilityIndex aidx : aidxs) {
+ int value = 0;
+ if (aidx.getKey().equals("[0;5]")) value = sh.bin[0];
+ if (aidx.getKey().equals("]5;90]")) value = sh.bin[1];
+ if (aidx.getKey().equals("]90;95]")) value = sh.bin[2];
+ if (aidx.getKey().equals("]95;100]")) value = sh.bin[3];
+
+ boolean exists = false;
+ for (IndexAvailabilityDataPoint i : aidx.getValues()) {
+ if (i.getX().equals(week.getKey())) {
+ exists = true;
+ i.setY(value / (double) total);
+ }
}
+ if (!exists)
+ aidx.getValues()
+ .add(new IndexAvailabilityDataPoint(week.getKey(), value / (double) total));
+ }
}
+ }
- private Index createIndex() {
- Index idx = new Index();
- idx.setEndpoint(CONSTANTS.SPARQLES);
+ private void update(
+ List results, SummaryStatistics cold, SummaryStatistics warm) {
- AvailabilityIndex aidx =
- new AvailabilityIndex("[0;5]", new ArrayList());
- List aidxs = new ArrayList();
-
- aidxs.add(aidx);
-
- aidx = new AvailabilityIndex("]5;90]", new ArrayList());
- aidxs.add(aidx);
-
- aidx = new AvailabilityIndex("]90;95]", new ArrayList());
- aidxs.add(aidx);
-
- aidx = new AvailabilityIndex("]95;100]", new ArrayList());
- aidxs.add(aidx);
-
- idx.setAvailability(aidxs);
+ for (EPViewPerformanceData pdata : results) {
+ if (pdata.getKey().toString().contains("Cold")) {
+ for (EPViewPerformanceDataValues v : pdata.getData()) {
+ cold.addValue(v.getValue());
+ }
+ }
+ if (pdata.getKey().toString().contains("Warm")) {
+ for (EPViewPerformanceDataValues v : pdata.getData()) {
+ warm.addValue(v.getValue());
+ }
+ }
+ }
+ }
+
+ private Index createIndex() {
+ Index idx = new Index();
+ idx.setEndpoint(SPARQLESProperties.getSparqlesEndpoint());
+
+ AvailabilityIndex aidx =
+ new AvailabilityIndex("[0;5]", new ArrayList());
+ List aidxs = new ArrayList();
+
+ aidxs.add(aidx);
+
+ aidx = new AvailabilityIndex("]5;90]", new ArrayList());
+ aidxs.add(aidx);
+
+ aidx = new AvailabilityIndex("]90;95]", new ArrayList());
+ aidxs.add(aidx);
+
+ aidx = new AvailabilityIndex("]95;100]", new ArrayList());
+ aidxs.add(aidx);
+
+ idx.setAvailability(aidxs);
+
+ IndexViewPerformance idxp = new IndexViewPerformance();
+ idxp.setThreshold(-1L);
+ idxp.setData(new ArrayList());
+ idx.setPerformance(idxp);
+
+ IndexViewInteroperability idxi = new IndexViewInteroperability();
+ idxi.setData(new ArrayList());
+ idx.setInteroperability(idxi);
+
+ IndexViewDiscoverability idxd = new IndexViewDiscoverability();
+ idxd.setNoDescription(-1D);
+ idxd.setSDDescription(-1D);
+ idxd.setVoIDDescription(-1D);
+ idxd.setServerName(new ArrayList());
+ idx.setDiscoverability(idxd);
+
+ IndexViewCalculation idxc = new IndexViewCalculation();
+ idxc.setVoID(-1D);
+ idxc.setVoIDPart(-1D);
+ idxc.setSD(-1D);
+ idxc.setSDPart(-1D);
+ idxc.setCoherence(-1D);
+ idxc.setRS(-1D);
+ idxc.setCoherences(new ArrayList());
+ idxc.setRss(new ArrayList());
+ idx.setCalculation(idxc);
+
+ return idx;
+ }
+
+ private void update(EPViewAvailabilityDataPoint value, Map weekHist) {
+ String key = "" + value.getX();
+ SimpleHistogram sh = weekHist.get(key);
+ if (sh == null) {
+ sh = new SimpleHistogram();
+ weekHist.put(key, sh);
+ }
+ sh.add(value.getY());
+ }
- IndexViewPerformance idxp = new IndexViewPerformance();
- idxp.setThreshold(-1L);
- idxp.setData(new ArrayList());
- idx.setPerformance(idxp);
+ @Override
+ public void setDBManager(MongoDBManager dbm) {
+ _dbm = dbm;
+ }
- IndexViewInteroperability idxi = new IndexViewInteroperability();
- idxi.setData(new ArrayList());
- idx.setInteroperability(idxi);
+ class SimpleHistogram {
- IndexViewDiscoverability idxd = new IndexViewDiscoverability();
- idxd.setNoDescription(-1D);
- idxd.setSDDescription(-1D);
- idxd.setVoIDDescription(-1D);
- idxd.setServerName(new ArrayList());
- idx.setDiscoverability(idxd);
+ int sampleSize = 0;
+ int[] bin = {0, 0, 0, 0};
- return idx;
- }
+ public void add(Double d) {
+ if (d <= 0.05) bin[0]++;
+ else if (0.05 < d && d <= 0.9) bin[1]++;
+ else if (0.9 < d && d <= 0.95) bin[2]++;
+ else if (0.95 < d && d <= 1) bin[3]++;
- private void update(EPViewAvailabilityDataPoint value, Map weekHist) {
- String key = "" + value.getX();
- SimpleHistogram sh = weekHist.get(key);
- if (sh == null) {
- sh = new SimpleHistogram();
- weekHist.put(key, sh);
- }
- sh.add(value.getY());
+ sampleSize++;
}
@Override
- public void setDBManager(MongoDBManager dbm) {
- _dbm = dbm;
+ public String toString() {
+ return Arrays.toString(bin) + ":" + sampleSize;
}
+ }
- class SimpleHistogram {
+ class Count extends TreeMap {
+ int sampleSize = 0;
- int sampleSize = 0;
- int[] bin = {0, 0, 0, 0};
-
- public void add(Double d) {
- if (d <= 0.05) bin[0]++;
- else if (0.05 < d && d <= 0.9) bin[1]++;
- else if (0.9 < d && d <= 0.95) bin[2]++;
- else if (0.95 < d && d <= 1) bin[3]++;
-
- sampleSize++;
- }
+ public void add(T t) {
+ if (this.containsKey(t)) {
+ this.put(t, this.get(t) + 1);
+ } else this.put(t, 1);
- @Override
- public String toString() {
- return Arrays.toString(bin) + ":" + sampleSize;
- }
+ sampleSize++;
}
- class Count extends TreeMap {
- int sampleSize = 0;
-
- public void add(T t) {
- if (this.containsKey(t)) {
- this.put(t, this.get(t) + 1);
- } else this.put(t, 1);
-
- sampleSize++;
- }
-
- public double getTotal() {
- // TODO Auto-generated method stub
- return sampleSize;
- }
+ public double getTotal() {
+ return sampleSize;
}
+ }
}
diff --git a/backend/src/main/java/sparqles/analytics/PAnalyser.java b/backend/src/main/java/sparqles/analytics/PAnalyser.java
index e65cf8a7..1d5ab046 100644
--- a/backend/src/main/java/sparqles/analytics/PAnalyser.java
+++ b/backend/src/main/java/sparqles/analytics/PAnalyser.java
@@ -18,143 +18,148 @@
import sparqles.utils.MongoDBManager;
public class PAnalyser extends Analytics {
- private static final Logger log = LoggerFactory.getLogger(PAnalyser.class);
-
- public PAnalyser(MongoDBManager db) {
- super(db);
+ private static final Logger log = LoggerFactory.getLogger(PAnalyser.class);
+
+ public PAnalyser(MongoDBManager db) {
+ super(db);
+ }
+
+ @Override
+ public boolean analyse(PResult pres) {
+ Endpoint ep = pres.getEndpointResult().getEndpoint();
+
+ log.info("Analyzing PResult for endpoint={}", ep.getUri());
+ log.debug("Analyse {}", pres);
+
+ PerformanceView pview = getView(ep);
+ EPView epview = getEPView(ep);
+
+ SummaryStatistics askStatsCold = new SummaryStatistics();
+ SummaryStatistics askStatsWarm = new SummaryStatistics();
+ SummaryStatistics joinStatsCold = new SummaryStatistics();
+ SummaryStatistics joinStatsWarm = new SummaryStatistics();
+
+ // prepare eppview data
+ EPViewPerformance eppview = epview.getPerformance();
+ EPViewPerformanceData askCold =
+ new EPViewPerformanceData(
+ "Cold ASK Tests", "#1f77b4", new ArrayList());
+ EPViewPerformanceData askWarm =
+ new EPViewPerformanceData(
+ "Warm ASK Tests", "#2ca02c", new ArrayList());
+ EPViewPerformanceData joinCold =
+ new EPViewPerformanceData(
+ "Cold JOIN Tests", "#1f77b4", new ArrayList());
+ EPViewPerformanceData joinWarm =
+ new EPViewPerformanceData(
+ "Warm JOIN Tests", "#2ca02c", new ArrayList());
+
+ ArrayList askdata = new ArrayList();
+ askdata.add(askCold);
+ askdata.add(askWarm);
+ ArrayList joindata = new ArrayList();
+ joindata.add(joinCold);
+ joindata.add(joinWarm);
+
+ eppview.setAsk(askdata);
+ eppview.setJoin(joindata);
+
+ Map map = pres.getResults();
+ long limit = 0;
+
+ for (Entry ent : map.entrySet()) {
+ PSingleResult res = ent.getValue();
+
+ if (ent.getKey().toString().startsWith("ASK")) {
+
+ askStatsCold.addValue(res.getCold().getClosetime() / (double) 1000);
+ askStatsWarm.addValue(res.getWarm().getClosetime() / (double) 1000);
+
+ String key = ent.getKey().toString().replaceAll("ASK", "").toLowerCase();
+
+ askCold
+ .getData()
+ .add(
+ new EPViewPerformanceDataValues(
+ key,
+ res.getCold().getClosetime() / (double) 1000,
+ res.getCold().getException()));
+ askWarm
+ .getData()
+ .add(
+ new EPViewPerformanceDataValues(
+ key,
+ res.getWarm().getClosetime() / (double) 1000,
+ res.getWarm().getException()));
+ } else if (ent.getKey().toString().startsWith("JOIN")) {
+ joinStatsCold.addValue(res.getCold().getClosetime() / (double) 1000);
+ joinStatsWarm.addValue(res.getWarm().getClosetime() / (double) 1000);
+
+ String key = ent.getKey().toString().replaceAll("JOIN", "").toLowerCase();
+
+ joinCold
+ .getData()
+ .add(
+ new EPViewPerformanceDataValues(
+ key,
+ res.getCold().getClosetime() / (double) 1000,
+ res.getCold().getException()));
+ joinWarm
+ .getData()
+ .add(
+ new EPViewPerformanceDataValues(
+ key,
+ res.getWarm().getClosetime() / (double) 1000,
+ res.getWarm().getException()));
+ } else if (ent.getKey().toString().startsWith("LIMIT")) {
+ int sol = res.getCold().getSolutions();
+ if (Math.max(limit, sol) == sol) {
+ limit = sol;
+ }
+ sol = res.getWarm().getSolutions();
+ if (Math.max(limit, sol) == sol) {
+ limit = sol;
+ }
+ }
}
+ eppview.setThreshold(limit);
+ pview.setThreshold(limit);
- @Override
- public boolean analyse(PResult pres) {
- log.info("Analyse {}", pres);
-
- Endpoint ep = pres.getEndpointResult().getEndpoint();
-
- PerformanceView pview = getView(ep);
- EPView epview = getEPView(ep);
-
- SummaryStatistics askStatsCold = new SummaryStatistics();
- SummaryStatistics askStatsWarm = new SummaryStatistics();
- SummaryStatistics joinStatsCold = new SummaryStatistics();
- SummaryStatistics joinStatsWarm = new SummaryStatistics();
-
- // prepare eppview data
- EPViewPerformance eppview = epview.getPerformance();
- EPViewPerformanceData askCold =
- new EPViewPerformanceData(
- "Cold ASK Tests", "#1f77b4", new ArrayList());
- EPViewPerformanceData askWarm =
- new EPViewPerformanceData(
- "Warm ASK Tests", "#2ca02c", new ArrayList());
- EPViewPerformanceData joinCold =
- new EPViewPerformanceData(
- "Cold JOIN Tests", "#1f77b4", new ArrayList());
- EPViewPerformanceData joinWarm =
- new EPViewPerformanceData(
- "Warm JOIN Tests", "#2ca02c", new ArrayList