From 9fd75c950d77b00569d63008ec40bcde67f9c5ce Mon Sep 17 00:00:00 2001 From: Prashant Pandey <84911643+suddendust@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:35:17 +0530 Subject: [PATCH] Build Profiles for aarch64 (#13648) * Build profiles for aarch64 * Add license header * Checkstyle * Download netty libs using the right OS classifier * Revert "Download netty libs using the right OS classifier" This reverts commit d77d166ecbdb7eed722f839d04696f36476818e2. * SkipTests on ARM progamatically --- .../sql/parsers/rewriter/ClpRewriterTest.java | 9 ++++ .../java/org/apache/pinot/util/TestUtils.java | 8 +++ .../function/ClpTransformFunctionsTest.java | 4 ++ .../CLPEncodingRealtimeIntegrationTest.java | 3 ++ .../clplog/CLPLogRecordExtractorTest.java | 12 +++++ .../segment/creator/DictionariesTest.java | 8 +++ .../creator/CLPForwardIndexCreatorTest.java | 3 ++ .../mutable/CLPMutableForwardIndexTest.java | 4 ++ .../spi/memory/PinotLArrayByteBufferTest.java | 5 ++ pom.xml | 52 +++++++++++++++++++ 10 files changed, 108 insertions(+) diff --git a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java index 987f4e449bbb..0121e4605f92 100644 --- a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java @@ -30,6 +30,8 @@ import org.apache.pinot.common.request.PinotQuery; import org.apache.pinot.sql.parsers.CalciteSqlParser; import org.apache.pinot.sql.parsers.SqlCompilationException; +import org.apache.pinot.util.TestUtils; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; @@ -40,6 +42,13 @@ public class ClpRewriterTest { private static final QueryRewriter _QUERY_REWRITER = new ClpRewriter(); + @BeforeClass + public void setup() { + //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet: + // https://github.com/y-scope/clp-ffi-java/issues/46 + TestUtils.ensureArchitectureIsNotARM(); + } + @Test public void testCLPDecodeRewrite() { // clpDecode rewrite from column group to individual columns diff --git a/pinot-common/src/test/java/org/apache/pinot/util/TestUtils.java b/pinot-common/src/test/java/org/apache/pinot/util/TestUtils.java index a126a043b136..f93291a3dea1 100644 --- a/pinot-common/src/test/java/org/apache/pinot/util/TestUtils.java +++ b/pinot-common/src/test/java/org/apache/pinot/util/TestUtils.java @@ -26,10 +26,12 @@ import java.time.Instant; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import net.openhft.chronicle.core.Jvm; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; +import org.testng.SkipException; /** @@ -200,6 +202,12 @@ public static T waitForResult(SupplierWithException supplier, long timeou throw new IllegalStateException("Failed to return result in " + timeoutMs + "ms"); } + public static void ensureArchitectureIsNotARM() { + if (Jvm.isArm()) { + throw new SkipException("Skipping test as the underlying architecture is ARM"); + } + } + @FunctionalInterface public interface SupplierWithException { T get() diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ClpTransformFunctionsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ClpTransformFunctionsTest.java index 67c1a6ef0890..6c7b0b6ba727 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ClpTransformFunctionsTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ClpTransformFunctionsTest.java @@ -52,6 +52,7 @@ import org.apache.pinot.spi.exception.BadQueryRequestException; import org.apache.pinot.spi.utils.ReadMode; import org.apache.pinot.spi.utils.builder.TableConfigBuilder; +import org.apache.pinot.util.TestUtils; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeTest; @@ -87,6 +88,9 @@ public class ClpTransformFunctionsTest { @BeforeClass public void setup() throws Exception { + //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet: + // https://github.com/y-scope/clp-ffi-java/issues/46 + TestUtils.ensureArchitectureIsNotARM(); // Setup the schema and table config Schema schema = new Schema.SchemaBuilder() .addSingleValueDimension(LOGTYPE_COLUMN, FieldSpec.DataType.STRING) diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CLPEncodingRealtimeIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CLPEncodingRealtimeIntegrationTest.java index d6b1ed119525..b4434c1f507b 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CLPEncodingRealtimeIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/CLPEncodingRealtimeIntegrationTest.java @@ -40,6 +40,9 @@ public class CLPEncodingRealtimeIntegrationTest extends BaseClusterIntegrationTe @BeforeClass public void setUp() throws Exception { + //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet: + // https://github.com/y-scope/clp-ffi-java/issues/46 + TestUtils.ensureArchitectureIsNotARM(); TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir); _avroFiles = unpackAvroData(_tempDir); diff --git a/pinot-plugins/pinot-input-format/pinot-clp-log/src/test/java/org/apache/pinot/plugin/inputformat/clplog/CLPLogRecordExtractorTest.java b/pinot-plugins/pinot-input-format/pinot-clp-log/src/test/java/org/apache/pinot/plugin/inputformat/clplog/CLPLogRecordExtractorTest.java index 72058be3451e..32d0caa0f9c3 100644 --- a/pinot-plugins/pinot-input-format/pinot-clp-log/src/test/java/org/apache/pinot/plugin/inputformat/clplog/CLPLogRecordExtractorTest.java +++ b/pinot-plugins/pinot-input-format/pinot-clp-log/src/test/java/org/apache/pinot/plugin/inputformat/clplog/CLPLogRecordExtractorTest.java @@ -26,8 +26,11 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import net.openhft.chronicle.core.Jvm; import org.apache.pinot.spi.data.readers.GenericRow; import org.apache.pinot.sql.parsers.rewriter.ClpRewriter; +import org.testng.SkipException; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import static org.apache.pinot.plugin.inputformat.clplog.CLPLogRecordExtractorConfig.FIELDS_FOR_CLP_ENCODING_CONFIG_KEY; @@ -50,6 +53,15 @@ public class CLPLogRecordExtractorTest { private static final String _MESSAGE_2_FIELD_VALUE = "Stopped job_123 on node-987: 3 cores, 6 threads and " + "22.0% memory used."; + @BeforeClass + public void setup() { + //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet: + // https://github.com/y-scope/clp-ffi-java/issues/46 + if (Jvm.isArm()) { + throw new SkipException("Skipping test as the underlying architecture is ARM"); + } + } + @Test public void testCLPEncoding() { Map props = new HashMap<>(); diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java index a678be426984..5a3226e58716 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import net.openhft.chronicle.core.Jvm; import org.apache.avro.Schema.Field; import org.apache.avro.file.DataFileStream; import org.apache.avro.generic.GenericRecord; @@ -68,6 +69,7 @@ import org.apache.pinot.spi.utils.ReadMode; import org.apache.pinot.util.TestUtils; import org.testng.Assert; +import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -537,6 +539,12 @@ private AbstractColumnStatisticsCollector buildStatsCollector(String column, Dat @Test public void clpStatsCollectorTest() { + //skip this test if the underlying architecture is aarch64. See: https://github.com/y-scope/clp-ffi-java/issues/46. + // Note that an x86 JVM might be running in emulated mode on macs. In that case, this will return a false. This is + // okay as in that case the x86 version of CLP would be used. + if (Jvm.isArm()) { + throw new SkipException("CLP is not supported yet on arm/aarch. Skipping this test"); + } Schema schema = new Schema(); schema.addField(new DimensionFieldSpec("column1", DataType.STRING, true)); List fieldConfigList = new ArrayList<>(); diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java index c97a1d318fb3..cd2d0aade79a 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java @@ -51,6 +51,9 @@ public class CLPForwardIndexCreatorTest { @BeforeClass public void setUp() throws Exception { + //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet: + // https://github.com/y-scope/clp-ffi-java/issues/46 + TestUtils.ensureArchitectureIsNotARM(); TestUtils.ensureDirectoriesExistAndEmpty(TEMP_DIR); } diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/mutable/CLPMutableForwardIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/mutable/CLPMutableForwardIndexTest.java index a926b9a396ad..ff7b6e80c66a 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/mutable/CLPMutableForwardIndexTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/mutable/CLPMutableForwardIndexTest.java @@ -27,6 +27,7 @@ import org.apache.pinot.segment.local.segment.creator.impl.stats.StringColumnPreIndexStatsCollector; import org.apache.pinot.segment.spi.memory.PinotDataBufferMemoryManager; import org.apache.pinot.spi.data.FieldSpec; +import org.apache.pinot.util.TestUtils; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -38,6 +39,9 @@ public class CLPMutableForwardIndexTest { @BeforeClass public void setUp() { + //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet: + // https://github.com/y-scope/clp-ffi-java/issues/46 + TestUtils.ensureArchitectureIsNotARM(); _memoryManager = new DirectMemoryManager(VarByteSVMutableForwardIndexTest.class.getName()); } diff --git a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java index 5c343fb7ef45..5ba1ddc2fabb 100644 --- a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java +++ b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java @@ -18,6 +18,7 @@ */ package org.apache.pinot.segment.spi.memory; +import net.openhft.chronicle.core.Jvm; import org.apache.pinot.segment.spi.utils.JavaVersion; import org.testng.SkipException; import org.testng.annotations.BeforeClass; @@ -35,6 +36,10 @@ protected boolean prioritizeByteBuffer() { @BeforeClass public void abortOnModernJava() { + //larray isn't supported on Mac/aarch64 + if (Jvm.isMacArm()) { + throw new SkipException("Skipping LArray tests because they cannot run on Mac/aarch64"); + } if (JavaVersion.VERSION > 15) { throw new SkipException("Skipping LArray tests because they cannot run in Java " + JavaVersion.VERSION); } diff --git a/pom.xml b/pom.xml index 033e90f398bb..53659e374798 100644 --- a/pom.xml +++ b/pom.xml @@ -249,6 +249,58 @@ + + + linux-aarch64 + + + + linux + aarch64 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + aarch64 + + + + + + + + + osx-aarch64 + + + + mac os x + aarch64 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + + aarch64 + + + + + + github-actions