From 1396ecef20feaff02e205ec08b36fbc891a87233 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 11:20:41 +0000 Subject: [PATCH 01/12] Adding platforms to reproducibility healthcheck We now have automated reproducibility testing on 3 more platforms, so we're checking their results as part of the daily healthcheck. Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 5cc595e65..956a67d95 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -48,9 +48,9 @@ def getPlatformReproTestMap() { def platformReproTestMap = [x64Linux: ["special.system", "Rebuild_Same_JDK_Reproducibility_Test"], x64Windows: ["dev.system", "Rebuild_Same_JDK_Reproducibility_Test_win"], x64Mac: ["NA", ""], - ppc64leLinux: ["NA", ""], - aarch64Linux: ["NA", ""], - aarch64Mac: ["NA", ""] + ppc64leLinux: ["special.system", "Rebuild_Same_JDK_Reproducibility_Test"], + aarch64Linux: ["special.system", "Rebuild_Same_JDK_Reproducibility_Test"], + aarch64Mac: ["dev.system", "Rebuild_Same_JDK_Reproducibility_Test_Mac"] ] return platformReproTestMap } From cc67396b969601e7e7dadd02de723793ef0a850c Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 14:24:02 +0000 Subject: [PATCH 02/12] Adding tolerance for decimal point Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 956a67d95..a760cae07 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -412,9 +412,9 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR if ( testOutput.contains("Running test "+reproTestName) ) { platformResult = "???% - ${reproTestName} ran but failed to produce a percentage. Test Link: " + testJob.buildUrl // Now we know the test ran, - def matcherObject = testOutput =~ /ReproduciblePercent = [0-9]+ %/ + def matcherObject = testOutput =~ /ReproduciblePercent = [0-9]+\.?[0-9]* %/ if ( matcherObject ) { - platformResult = matcherObject[0] =~ /[0-9]+ %/ + platformResult = matcherObject[0] =~ /[0-9]+\.?[0-9]* %/ } } } @@ -431,12 +431,13 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR results[jdkVersion][1].each{key, value -> if (value.equals("NA")) { naCount++ - } else if ( (value ==~ /^[0-9]+ %/) ) { - overallAverage += (value =~ /^[0-9]+/)[0] as Integer + } else if ( (value ==~ /^[0-9]+\.?[0-9]* %/) ) { + overallAverage += (value =~ /^[0-9]+\.?[0-9]*/)[0] as Double } // else do nothing, as we presume non-integer and non-NA values are 0. } - overallAverage = overallAverage == 0 ? 0 : overallAverage.intdiv(results[jdkVersion][1].size() - naCount) + overallAverage = overallAverage == 0 ? 0 : overallAverage / (results[jdkVersion][1].size() - naCount) + overallAverage = 0 ? 0 : Math.round(overallAverage * 100) / 100 results[jdkVersion][0] = overallAverage+" %" } } From 5282f3fd3fd4e99b411a8bbf99e430404e283cca Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 14:24:28 +0000 Subject: [PATCH 03/12] Typo fix --- tools/nightly_build_and_test_stats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index a760cae07..9c11edad4 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -424,7 +424,7 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR results[jdkVersion][1][onePlatform] = platformResult } - // Now we have the percentages for each platform, we canculate the jdkVersion-specific average. + // Now we have the percentages for each platform, we calculate the jdkVersion-specific average. def overallAverage = 0 // Ignoring the platforms where the test is not available yet. def naCount = 0 From e99db47a6ad3a6c497e1159f3d49950e63e0d31b Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 14:28:36 +0000 Subject: [PATCH 04/12] Syntax fix Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 9c11edad4..183871e8d 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -425,7 +425,7 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR } // Now we have the percentages for each platform, we calculate the jdkVersion-specific average. - def overallAverage = 0 + Double overallAverage = 0.0 // Ignoring the platforms where the test is not available yet. def naCount = 0 results[jdkVersion][1].each{key, value -> From fc48ccd69120000579d61b7b02285d7279c31c5b Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 15:16:38 +0000 Subject: [PATCH 05/12] Fixing the regex Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 183871e8d..fde6c7be3 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -412,9 +412,9 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR if ( testOutput.contains("Running test "+reproTestName) ) { platformResult = "???% - ${reproTestName} ran but failed to produce a percentage. Test Link: " + testJob.buildUrl // Now we know the test ran, - def matcherObject = testOutput =~ /ReproduciblePercent = [0-9]+\.?[0-9]* %/ + def matcherObject = testOutput =~ /ReproduciblePercent = (100|[0-9][0-9]?\.?[0-9]?[0-9]?) %/ if ( matcherObject ) { - platformResult = matcherObject[0] =~ /[0-9]+\.?[0-9]* %/ + println((matcherObject[0] =~ /(100|[0-9][0-9]?\.?[0-9]?[0-9]?) %/)[0][0]) } } } From cb1df5b74848afd55f52cb045e040c67ce8b52c3 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 15:21:27 +0000 Subject: [PATCH 06/12] Fixing variable content loss Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index fde6c7be3..8707eaf38 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -414,7 +414,7 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR // Now we know the test ran, def matcherObject = testOutput =~ /ReproduciblePercent = (100|[0-9][0-9]?\.?[0-9]?[0-9]?) %/ if ( matcherObject ) { - println((matcherObject[0] =~ /(100|[0-9][0-9]?\.?[0-9]?[0-9]?) %/)[0][0]) + platformResult = ((matcherObject[0] =~ /(100|[0-9][0-9]?\.?[0-9]?[0-9]?) %/)[0][0]) } } } @@ -431,7 +431,7 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR results[jdkVersion][1].each{key, value -> if (value.equals("NA")) { naCount++ - } else if ( (value ==~ /^[0-9]+\.?[0-9]* %/) ) { + } else if ( value ==~ /^[0-9]+\.?[0-9]* %/ ) { overallAverage += (value =~ /^[0-9]+\.?[0-9]*/)[0] as Double } // else do nothing, as we presume non-integer and non-NA values are 0. From f3754fb8f08aee05d091d971cdd2f3ed131c338b Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 15:25:51 +0000 Subject: [PATCH 07/12] Fixing another regex Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 8707eaf38..0edc29c98 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -822,7 +822,7 @@ node('worker') { reproducibleBuilds[featureRelease][1].each{ key, value -> if (!value.equals("NA")) { echo key+": "+value - if(value ==~ /[0-9]+ %/) { + if(value ==~ /[0-9]+\.?[0=9]* %/) { summaryOfRepros+=" "+key+"("+value+")," } else { summaryOfRepros+=" "+key+"(?%)," From 87c50784c0ff5b735fd6c137af9b7995dcb96b16 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 1 Nov 2024 15:33:17 +0000 Subject: [PATCH 08/12] Typo fix Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 0edc29c98..79bce703b 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -822,7 +822,7 @@ node('worker') { reproducibleBuilds[featureRelease][1].each{ key, value -> if (!value.equals("NA")) { echo key+": "+value - if(value ==~ /[0-9]+\.?[0=9]* %/) { + if(value ==~ /[0-9]+\.?[0-9]* %/) { summaryOfRepros+=" "+key+"("+value+")," } else { summaryOfRepros+=" "+key+"(?%)," From 5b7a51a820231ebb22e6b53f97d8813166b36c96 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 4 Nov 2024 16:30:04 +0000 Subject: [PATCH 09/12] Fixing Linter concerns Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 79bce703b..c7f1ee5a5 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -15,6 +15,7 @@ limitations under the License. /* groovylint-disable NestedBlockDepth */ import groovy.json.JsonSlurper +import java.math.MathContext; import java.time.LocalDateTime import java.time.Instant import java.time.ZoneId @@ -425,19 +426,21 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR } // Now we have the percentages for each platform, we calculate the jdkVersion-specific average. - Double overallAverage = 0.0 + BigDecimal overallAverage = 0.0 // Ignoring the platforms where the test is not available yet. def naCount = 0 results[jdkVersion][1].each{key, value -> if (value.equals("NA")) { naCount++ } else if ( value ==~ /^[0-9]+\.?[0-9]* %/ ) { - overallAverage += (value =~ /^[0-9]+\.?[0-9]*/)[0] as Double + overallAverage += (value =~ /^[0-9]+\.?[0-9]*/)[0] as BigDecimal } // else do nothing, as we presume non-integer and non-NA values are 0. } - overallAverage = overallAverage == 0 ? 0 : overallAverage / (results[jdkVersion][1].size() - naCount) - overallAverage = 0 ? 0 : Math.round(overallAverage * 100) / 100 + if (overallAverage != 0) { + overallAverage = overallAverage / (results[jdkVersion][1].size() - naCount) + } + overallAverage.round(new MathContext(3)) results[jdkVersion][0] = overallAverage+" %" } } From 109b6c433fa8a5358fd9f72364a4492e683f6b99 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 4 Nov 2024 16:42:35 +0000 Subject: [PATCH 10/12] Alternative rounding method to solve Jenkins' MathContext aversion Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index c7f1ee5a5..ecd3cf763 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -440,7 +440,7 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR if (overallAverage != 0) { overallAverage = overallAverage / (results[jdkVersion][1].size() - naCount) } - overallAverage.round(new MathContext(3)) + overallAverage = (overallAverage * 100).toBigInteger() / 100 results[jdkVersion][0] = overallAverage+" %" } } From 7e33cad66962879309c86fa4ec26aec8e1b8f794 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 4 Nov 2024 17:00:59 +0000 Subject: [PATCH 11/12] Changing method of rounding down Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index ecd3cf763..616147178 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -440,8 +440,8 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR if (overallAverage != 0) { overallAverage = overallAverage / (results[jdkVersion][1].size() - naCount) } - overallAverage = (overallAverage * 100).toBigInteger() / 100 - results[jdkVersion][0] = overallAverage+" %" + // This reduces the output to 2 decimal places. + results[jdkVersion][0] = ((overallAverage.stripTrailingZeros().toString() + "00") =~ /[0-9]+\.?[0-9]?[0-9]?/)[0]+" %" } } From f8dc66ad4f26411ce8c84e1385379847b751a3d6 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 4 Nov 2024 17:04:04 +0000 Subject: [PATCH 12/12] Removing banned method Signed-off-by: Adam Farley --- tools/nightly_build_and_test_stats.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nightly_build_and_test_stats.groovy b/tools/nightly_build_and_test_stats.groovy index 616147178..e927bd962 100644 --- a/tools/nightly_build_and_test_stats.groovy +++ b/tools/nightly_build_and_test_stats.groovy @@ -441,7 +441,7 @@ def getReproducibilityPercentage(String jdkVersion, String trssId, String trssUR overallAverage = overallAverage / (results[jdkVersion][1].size() - naCount) } // This reduces the output to 2 decimal places. - results[jdkVersion][0] = ((overallAverage.stripTrailingZeros().toString() + "00") =~ /[0-9]+\.?[0-9]?[0-9]?/)[0]+" %" + results[jdkVersion][0] = ((overallAverage.toString()) =~ /[0-9]+\.?[0-9]?[0-9]?/)[0]+" %" } }