update longtail to 0.4.0

.github/workflows/create-release.yml

@@ -12,19 +12,26 @@ jobs:
 
     strategy:
       matrix:
+        os: [ubuntu-20.04, macos-latest, windows-latest]
+        arch: [amd64, arm64]
         include:
-        - os: ubuntu-20.04
-          target: "./cmd/longtail/longtail"
-          platform: linux
-          tag: "${GITHUB_REF_NAME}"
-        - os: macos-latest
-          target: "./cmd/longtail/longtail"
-          platform: macos
-          tag: "${GITHUB_REF_NAME}"
-        - os: windows-latest
-          target: "./cmd/longtail/longtail.exe"
-          platform: win32
-          tag: "${env:GITHUB_REF_NAME}"
+          - os: ubuntu-20.04
+            target: "./cmd/longtail/longtail"
+            platform: linux
+            tag: "${GITHUB_REF_NAME}"
+          - os: macos-latest
+            target: "./cmd/longtail/longtail"
+            platform: macos
+            tag: "${GITHUB_REF_NAME}"
+          - os: windows-latest
+            target: "./cmd/longtail/longtail.exe"
+            platform: win32
+            tag: "${env:GITHUB_REF_NAME}"
+        exclude:
+          - os: ubuntu-20.04
+            arch: arm64
+          - os: windows-latest
+            arch: arm64
 
     runs-on: ${{matrix.os}}
 
@@ -39,10 +46,14 @@ jobs:
 
       - name: Build
         run: |
+          echo "GOARCH: " $GOARCH
           pushd ./cmd/longtail
           echo matrix.tag ${{ matrix.tag }}
           go build -ldflags="-s -w -X 'github.com/DanEngelbrecht/golongtail/commands.BuildVersion=${{ matrix.tag }}'" .
           popd
+        env:
+          GOARCH: ${{matrix.arch}}
+          CGO_ENABLED: 1
 
       - name: build dist
         run: |
@@ -52,7 +63,7 @@ jobs:
       - name: Upload artifacts
         uses: actions/upload-artifact@master
         with:
-          name: dist-${{matrix.platform}}-x64
+          name: dist-${{matrix.platform}}-${{matrix.arch}}
           path: dist
 
   create-release:
@@ -68,42 +79,55 @@ jobs:
     - name: Download Linux artifacts
       uses: actions/download-artifact@v1
       with:
-        name: dist-linux-x64
+        name: dist-linux-amd64
         path: dist-linux-x64
 
-    - name: Download MacOs artifacts
+    - name: Download MacOs x64 artifacts
       uses: actions/download-artifact@v1
       with:
-        name: dist-macos-x64
+        name: dist-macos-amd64
         path: dist-macos-x64
 
+    - name: Download MacOs arm64 artifacts
+      uses: actions/download-artifact@v1
+      with:
+        name: dist-macos-arm64
+        path: dist-macos-arm64
+
     - name: Download Win32 artifacts
       uses: actions/download-artifact@v1
       with:
-        name: dist-win32-x64
+        name: dist-win32-amd64
         path: dist-win32-x64
 
     - name: Set execute permission
       run: |
         chmod +x dist-linux-x64/longtail
         chmod +x dist-macos-x64/longtail
+        chmod +x dist-macos-arm64/longtail
 
-    - name: Rename artifacts
+    - name: Copy executables
       run: |
         cp dist-linux-x64/longtail longtail-linux-x64
         cp dist-macos-x64/longtail longtail-macos-x64
         cp dist-win32-x64/longtail.exe longtail-win32-x64.exe
+        cp dist-macos-arm64/longtail longtail-macos-arm64
 
     - name: Zip Linux artifacts
       uses: montudor/[email protected]
       with:
         args: zip -qq -r ./linux-x64.zip ./dist-linux-x64
 
-    - name: Zip MacOS artifacts
+    - name: Zip MacOS x64 artifacts
       uses: montudor/[email protected]
       with:
         args: zip -qq -r ./macos-x64.zip ./dist-macos-x64
 
+    - name: Zip MacOS arm64 artifacts
+      uses: montudor/[email protected]
+      with:
+        args: zip -qq -r ./macos-arm64.zip ./dist-macos-arm64
+
     - name: Zip Win32 artifacts
       uses: montudor/[email protected]
       with:
@@ -142,6 +166,8 @@ jobs:
           win32-x64.zip
           linux-x64.zip
           macos-x64.zip
+          macos-arm64.zip
           longtail-linux-x64
           longtail-macos-x64
+          longtail-macos-arm64
           longtail-win32-x64.exe

.github/workflows/master-build.yml

@@ -11,6 +11,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, macos-latest, windows-latest]
+        arch: [amd64]
+        include:
+          - os: macos-latest
+            arch: arm64
 
     runs-on: ${{matrix.os}}
 
@@ -24,6 +28,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: test
+        if: ${{ matrix.arch == 'amd64' }}
         run: |
           pushd ./longtaillib
           go test .
@@ -37,9 +42,15 @@ jobs:
           pushd ./remotestore
           go test .
           popd
+        env:
+          GOARCH: ${{matrix.arch}}
+          CGO_ENABLED: 1
 
       - name: build cmd
         run: |
           pushd ./cmd/longtail
           go build .
           popd
+        env:
+          GOARCH: ${{matrix.arch}}
+          CGO_ENABLED: 1

.github/workflows/pr-build.yml

@@ -20,6 +20,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, macos-latest, windows-latest]
+        arch: [amd64]
+        include:
+          - os: macos-latest
+            arch: arm64
 
     runs-on: ${{matrix.os}}
 
@@ -33,6 +37,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: test
+        if: ${{ matrix.arch == 'amd64' }}
         run: |
           pushd ./longtaillib
           go test .
@@ -46,9 +51,15 @@ jobs:
           pushd ./remotestore
           go test .
           popd
+        env:
+          GOARCH: ${{matrix.arch}}
+          CGO_ENABLED: 1
 
       - name: build cmd
         run: |
           pushd ./cmd/longtail
           go build .
           popd
+        env:
+          GOARCH: ${{matrix.arch}}
+          CGO_ENABLED: 1

CHANGELOG.md

@@ -1,4 +1,9 @@
 ##
+- **NEW** added builds for Arm64 flavours
+  - meow hash option is not supported on Arm64 and will return 0 if called
+  - blake2 hash option is not supported on Arm64 and will return 0 if called
+  - `macos-arm64.zip` and `longtail-macos-arm64` artifacts are produced when creating a release
+- **UPDATED** Updated to longtail 0.4.0
 - **UPDATED** Updated all golang dependencies
 - **UPDATED** Updated Github Actions Linux runners to 20.04
 

longtaillib/longtail/include/src/longtail.h

@@ -1555,7 +1555,7 @@ struct Longtail_StoreIndex
     uint32_t* m_ChunkCount;             // Total number of chunks across all blocks - chunk hashes may occur more than once
     TLongtail_Hash* m_BlockHashes;      // [] m_BlockHashes is the hash of each block
     TLongtail_Hash* m_ChunkHashes;      // [] For each m_BlockChunkCount[n] there are n consecutive chunk hashes in m_ChunkHashes[]
-    uint32_t* m_BlockChunksOffsets;     // [] m_BlockChunksOffsets[n] is the offset in m_ChunkBlockCount[] and m_ChunkHashes[]
+    uint32_t* m_BlockChunksOffsets;     // [] m_BlockChunksOffsets[n] is the offset in m_ChunkSizes[] and m_ChunkHashes[]
     uint32_t* m_BlockChunkCounts;       // [] m_BlockChunkCounts[n] is number of chunks in block m_BlockHash[n]
     uint32_t* m_BlockTags;              // [] m_BlockTags is the tag for each block
     uint32_t* m_ChunkSizes;             // [] m_ChunkSizes is the size of each chunk
@@ -1632,7 +1632,19 @@ LONGTAIL_EXPORT int Longtail_ValidateStore(
  * @param[in] store_index   The source index to copy from
  * @return                  An initialized struct Longtail_StoreIndex copied from @p store_index, or 0 on bad parameters / out of memory
  */
-LONGTAIL_EXPORT struct Longtail_StoreIndex* Longtail_CopyStoreIndex(struct Longtail_StoreIndex* store_index);
+LONGTAIL_EXPORT struct Longtail_StoreIndex* Longtail_CopyStoreIndex(const struct Longtail_StoreIndex* store_index);
+
+/*! @brief Splits a Longtail_StoreIndex into multiple store indexes
+ *
+ * Splits a Longtail_StoreIndex into multiple store indexes with each having a max size of split_size.
+ *  *
+ * @param[in] store_index   The source index to split from
+ * @param[in] split_size    The maximum size each resulting store index may have
+ * @param[out] split_size   A pointer to a pointer that will contain a pointer to an array of store index pointers
+ * @param[out] out_count    A pointer that will contain the number of store indexes in the resulting out_store_indexes array
+ * @return                  Return code (errno style), zero on success.
+ */
+LONGTAIL_EXPORT int Longtail_SplitStoreIndex(struct Longtail_StoreIndex* store_index, size_t split_size, struct Longtail_StoreIndex*** out_store_indexes, uint64_t* out_count);
 
 /*! @brief Writes a struct Longtail_StoreIndex to a byte buffer.
  *

longtaillib/longtail_darwin_arm64.go

@@ -0,0 +1,4 @@
+package longtaillib
+
+// #cgo LDFLAGS: -L${SRCDIR}/longtail -llongtail_darwin_arm64 -lm
+import "C"

longtailstorelib/fsstore_unix_arm64.go

@@ -0,0 +1,75 @@
+// +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris
+
+package longtailstorelib
+
+import (
+	"fmt"
+	"syscall"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type Lock struct {
+	filename string
+	fd       int
+}
+
+func NewFileLock(filename string) *Lock {
+	return &Lock{filename: filename}
+}
+
+func (l *Lock) Lock() error {
+	const fname = "Lock.open"
+	if err := l.open(); err != nil {
+		return errors.Wrap(err, fname)
+	}
+	return syscall.Flock(l.fd, syscall.LOCK_EX)
+}
+
+func (l *Lock) Unlock() error {
+	const fname = "Lock.Unlock"
+	err := syscall.Close(l.fd)
+	if err != nil {
+		return errors.Wrap(err, fname)
+	}
+	return nil
+}
+
+func (l *Lock) LockWithTimeout(timeout time.Duration) error {
+	const fname = "Lock.LockWithTimeout"
+	err := l.open()
+	if err != nil {
+		return errors.Wrap(err, fname)
+	}
+	result := make(chan error)
+	cancel := make(chan struct{})
+	go func() {
+		err := syscall.Flock(l.fd, syscall.LOCK_EX)
+		select {
+		case <-cancel:
+			// Timed out, cleanup if necessary.
+			syscall.Flock(l.fd, syscall.LOCK_UN)
+			syscall.Close(l.fd)
+		case result <- err:
+		}
+	}()
+	select {
+	case err := <-result:
+		return errors.Wrap(err, fname)
+	case <-time.After(timeout):
+		close(cancel)
+		err := fmt.Errorf("Retry timed out for lock file `%s`, waited %s", l.filename, timeout.String())
+		return errors.Wrap(err, fname)
+	}
+}
+
+func (l *Lock) open() error {
+	const fname = "Lock.open"
+	fd, err := syscall.Open(l.filename, syscall.O_CREAT|syscall.O_RDONLY, 0600)
+	if err != nil {
+		return errors.Wrap(err, fname)
+	}
+	l.fd = fd
+	return nil
+}