Build Profiles for aarch64 (apache#13648)

* Build profiles for aarch64 * Add license header * Checkstyle * Download netty libs using the right OS classifier * Revert "Download netty libs using the right OS classifier" This reverts commit d77d166. * SkipTests on ARM progamatically
soumitra-st · Jul 26, 2024 · 9fd75c9 · 9fd75c9
1 parent 8dd5b50
commit 9fd75c9
Show file tree

Hide file tree

Showing 10 changed files with 108 additions and 0 deletions.
diff --git a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java
@@ -30,6 +30,8 @@
 import org.apache.pinot.common.request.PinotQuery;
 import org.apache.pinot.sql.parsers.CalciteSqlParser;
 import org.apache.pinot.sql.parsers.SqlCompilationException;
+import org.apache.pinot.util.TestUtils;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import static org.testng.Assert.assertEquals;
@@ -40,6 +42,13 @@
 public class ClpRewriterTest {
   private static final QueryRewriter _QUERY_REWRITER = new ClpRewriter();
 
+  @BeforeClass
+  public void setup() {
+    //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet:
+    // https://github.com/y-scope/clp-ffi-java/issues/46
+    TestUtils.ensureArchitectureIsNotARM();
+  }
+
   @Test
   public void testCLPDecodeRewrite() {
     // clpDecode rewrite from column group to individual columns

diff --git a/pinot-common/src/test/java/org/apache/pinot/util/TestUtils.java b/pinot-common/src/test/java/org/apache/pinot/util/TestUtils.java
@@ -26,10 +26,12 @@
 import java.time.Instant;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import net.openhft.chronicle.core.Jvm;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
+import org.testng.SkipException;
 
 
 /**
@@ -200,6 +202,12 @@ public static <T> T waitForResult(SupplierWithException<T> supplier, long timeou
     throw new IllegalStateException("Failed to return result in " + timeoutMs + "ms");
   }
 
+  public static void ensureArchitectureIsNotARM() {
+    if (Jvm.isArm()) {
+      throw new SkipException("Skipping test as the underlying architecture is ARM");
+    }
+  }
+
   @FunctionalInterface
   public interface SupplierWithException<T> {
     T get()

diff --git a/...est/java/org/apache/pinot/core/operator/transform/function/ClpTransformFunctionsTest.java b/...est/java/org/apache/pinot/core/operator/transform/function/ClpTransformFunctionsTest.java
@@ -52,6 +52,7 @@
 import org.apache.pinot.spi.exception.BadQueryRequestException;
 import org.apache.pinot.spi.utils.ReadMode;
 import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.util.TestUtils;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeTest;
@@ -87,6 +88,9 @@ public class ClpTransformFunctionsTest {
   @BeforeClass
   public void setup()
       throws Exception {
+    //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet:
+    // https://github.com/y-scope/clp-ffi-java/issues/46
+    TestUtils.ensureArchitectureIsNotARM();
     // Setup the schema and table config
     Schema schema = new Schema.SchemaBuilder()
         .addSingleValueDimension(LOGTYPE_COLUMN, FieldSpec.DataType.STRING)

diff --git a/.../src/test/java/org/apache/pinot/integration/tests/CLPEncodingRealtimeIntegrationTest.java b/.../src/test/java/org/apache/pinot/integration/tests/CLPEncodingRealtimeIntegrationTest.java
@@ -40,6 +40,9 @@ public class CLPEncodingRealtimeIntegrationTest extends BaseClusterIntegrationTe
   @BeforeClass
   public void setUp()
       throws Exception {
+    //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet:
+    // https://github.com/y-scope/clp-ffi-java/issues/46
+    TestUtils.ensureArchitectureIsNotARM();
     TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir);
     _avroFiles = unpackAvroData(_tempDir);
 

diff --git a/...g/src/test/java/org/apache/pinot/plugin/inputformat/clplog/CLPLogRecordExtractorTest.java b/...g/src/test/java/org/apache/pinot/plugin/inputformat/clplog/CLPLogRecordExtractorTest.java
@@ -26,8 +26,11 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import net.openhft.chronicle.core.Jvm;
 import org.apache.pinot.spi.data.readers.GenericRow;
 import org.apache.pinot.sql.parsers.rewriter.ClpRewriter;
+import org.testng.SkipException;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import static org.apache.pinot.plugin.inputformat.clplog.CLPLogRecordExtractorConfig.FIELDS_FOR_CLP_ENCODING_CONFIG_KEY;
@@ -50,6 +53,15 @@ public class CLPLogRecordExtractorTest {
   private static final String _MESSAGE_2_FIELD_VALUE = "Stopped job_123 on node-987: 3 cores, 6 threads and "
       + "22.0% memory used.";
 
+  @BeforeClass
+  public void setup() {
+    //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet:
+    // https://github.com/y-scope/clp-ffi-java/issues/46
+    if (Jvm.isArm()) {
+      throw new SkipException("Skipping test as the underlying architecture is ARM");
+    }
+  }
+
   @Test
   public void testCLPEncoding() {
     Map<String, String> props = new HashMap<>();

diff --git a/...-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java b/...-local/src/test/java/org/apache/pinot/segment/local/segment/creator/DictionariesTest.java
@@ -29,6 +29,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import net.openhft.chronicle.core.Jvm;
 import org.apache.avro.Schema.Field;
 import org.apache.avro.file.DataFileStream;
 import org.apache.avro.generic.GenericRecord;
@@ -68,6 +69,7 @@
 import org.apache.pinot.spi.utils.ReadMode;
 import org.apache.pinot.util.TestUtils;
 import org.testng.Assert;
+import org.testng.SkipException;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -537,6 +539,12 @@ private AbstractColumnStatisticsCollector buildStatsCollector(String column, Dat
 
   @Test
   public void clpStatsCollectorTest() {
+    //skip this test if the underlying architecture is aarch64. See: https://github.com/y-scope/clp-ffi-java/issues/46.
+    // Note that an x86 JVM might be running in emulated mode on macs. In that case, this will return a false. This is
+    // okay as in that case the x86 version of CLP would be used.
+    if (Jvm.isArm()) {
+      throw new SkipException("CLP is not supported yet on arm/aarch. Skipping this test");
+    }
     Schema schema = new Schema();
     schema.addField(new DimensionFieldSpec("column1", DataType.STRING, true));
     List<FieldConfig> fieldConfigList = new ArrayList<>();

diff --git a/...java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java b/...java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java
@@ -51,6 +51,9 @@ public class CLPForwardIndexCreatorTest {
   @BeforeClass
   public void setUp()
       throws Exception {
+    //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet:
+    // https://github.com/y-scope/clp-ffi-java/issues/46
+    TestUtils.ensureArchitectureIsNotARM();
     TestUtils.ensureDirectoriesExistAndEmpty(TEMP_DIR);
   }
 

diff --git a/.../apache/pinot/segment/local/segment/index/forward/mutable/CLPMutableForwardIndexTest.java b/.../apache/pinot/segment/local/segment/index/forward/mutable/CLPMutableForwardIndexTest.java
@@ -27,6 +27,7 @@
 import org.apache.pinot.segment.local.segment.creator.impl.stats.StringColumnPreIndexStatsCollector;
 import org.apache.pinot.segment.spi.memory.PinotDataBufferMemoryManager;
 import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.util.TestUtils;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -38,6 +39,9 @@ public class CLPMutableForwardIndexTest {
 
   @BeforeClass
   public void setUp() {
+    //skip this test if the underlying arch is aarch64 as CLP isn't supported on ARM yet:
+    // https://github.com/y-scope/clp-ffi-java/issues/46
+    TestUtils.ensureArchitectureIsNotARM();
     _memoryManager = new DirectMemoryManager(VarByteSVMutableForwardIndexTest.class.getName());
   }
 

diff --git a/...ment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java b/...ment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.segment.spi.memory;
 
+import net.openhft.chronicle.core.Jvm;
 import org.apache.pinot.segment.spi.utils.JavaVersion;
 import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
@@ -35,6 +36,10 @@ protected boolean prioritizeByteBuffer() {
 
   @BeforeClass
   public void abortOnModernJava() {
+    //larray isn't supported on Mac/aarch64
+    if (Jvm.isMacArm()) {
+      throw new SkipException("Skipping LArray tests because they cannot run on Mac/aarch64");
+    }
     if (JavaVersion.VERSION > 15) {
       throw new SkipException("Skipping LArray tests because they cannot run in Java " + JavaVersion.VERSION);
     }

diff --git a/pom.xml b/pom.xml
@@ -249,6 +249,58 @@
   </properties>
 
   <profiles>
+    <!--build profile for linux-aarch64. We exclude certain tests as they use runtime JNI bindings not supported for linux-aarch64-->
+    <profile>
+      <id>linux-aarch64</id>
+      <activation>
+        <!--os info is taken from the output of mvn -version-->
+        <os>
+          <name>linux</name>
+          <arch>aarch64</arch>
+        </os>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>${surefire.version}</version>
+            <configuration>
+              <systemPropertyVariables>
+                <os.arch>aarch64</os.arch>
+              </systemPropertyVariables>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <!--build profile for osx-aarch64. We exclude certain tests as they use runtime JNI bindings not supported for linux-aarch64-->
+    <profile>
+      <id>osx-aarch64</id>
+      <activation>
+        <!--os info is taken from the output of mvn -version-->
+        <os>
+          <name>mac os x</name>
+          <arch>aarch64</arch>
+        </os>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>${surefire.version}</version>
+            <configuration>
+              <excludes>
+              </excludes>
+              <systemPropertyVariables>
+                <os.arch>aarch64</os.arch>
+              </systemPropertyVariables>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>github-actions</id>
       <activation>