initial multipart transfer

deploy/fly-io/.dockerignore

@@ -0,0 +1,2 @@
+*
+!start.sh

deploy/fly-io/Dockerfile

@@ -0,0 +1,10 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y curl
+RUN curl -fsSL https://d.juicefs.com/install | sh -
+RUN curl -fsSL https://tailscale.com/install.sh | sh -
+
+WORKDIR /app
+COPY . .
+
+ENTRYPOINT ["/app/start.sh"]

deploy/fly-io/fly.toml

@@ -0,0 +1,47 @@
+app = 'data-stage-beam-cloud'
+primary_region = 'iad'
+kill_signal = 'SIGTERM'
+
+[build]
+
+[deploy]
+  strategy = 'immediate'
+
+[env]
+  JFS_ADDRESS = '0.0.0.0:9000'
+
+[http_service]
+  internal_port = 9000
+  force_https = true
+  auto_start_machines = true
+  auto_stop_machines = 'off'
+
+  [http_service.concurrency]
+    type = 'requests'
+    hard_limit = 30000
+    soft_limit = 30000
+
+  [http_service.tls_options]
+    versions = ['TLSv1.2', 'TLSv1.3']
+    default_self_signed = false
+
+  [http_service.http_options]
+    idle_timeout = 900
+
+    [http_service.http_options.response]
+      pristine = true
+
+[checks]
+  [checks.app]
+    port = 9000
+    type = 'http'
+    interval = '15s'
+    timeout = '5s'
+    grace_period = '30s'
+    method = 'get'
+    path = '/minio/health/live'
+
+[[vm]]
+  cpus = 2
+  cpu_kind = 'shared'
+  memory = '2gb'

deploy/fly-io/start.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# tailscale
+tailscaled --port=41641 --socket=/var/run/tailscale/tailscaled.sock --state=/var/lib/tailscale/tailscaled.state &
+tailscale up --authkey=${TAILSCALE_AUTHKEY} --ssh --reset --accept-routes
+
+while [ -z "$(tailscale ip 2>/dev/null)" ]; do
+  echo "Waiting for Tailscale to fully establish a connection..."
+  sleep 1
+done
+
+# juicefs
+JUICEFS_ADDRESS=${JUICEFS_ADDRESS:-0.0.0.0:9000}
+JUICEFS_BLOCK_SIZE=${JUICEFS_ADDRESS:-4096}
+JUICEFS_BUFFER_SIZE=${JUICEFS_BUFFER_SIZE:-300}
+JUICEFS_CACHE_SIZE=${JUICEFS_CACHE_SIZE:-0}
+JUICEFS_PREFETCH=${JUICEFS_PREFETCH:-1}
+
+juicefs format \
+  --storage=s3 \
+  --bucket=${JUICEFS_BUCKET} \
+  --access-key=${JUICEFS_ACCESS_KEY} \
+  --secret-key=${JUICEFS_SECRET_KEY} \
+  --block-size=${JUICEFS_BLOCK_SIZE} \
+  --no-update \
+  ${JUICEFS_REDIS_URI} \
+  ${JUICEFS_NAME}
+
+juicefs gateway \
+  --storage="s3" \
+  --bucket=${JUICEFS_BUCKET} \
+  --buffer-size=${JUICEFS_BUFFER_SIZE} \
+  --cache-size=${JUICEFS_CACHE_SIZE} \
+  --prefetch=${JUICEFS_PREFETCH} \
+  --no-usage-report \
+  ${JUICEFS_REDIS_URI} \
+  ${JUICEFS_ADDRESS}

pkg/common/config.default.yaml

@@ -56,6 +56,11 @@ gateway:
     memory: 32768
     maxReplicas: 10
     maxGpuCount: 2
+fileService:
+  endpointUrl: https://data-stage-beam-cloud.fly.dev
+  bucketName: beta9-fs
+  accessKey:
+  secretKey:
 imageService:
   localCacheEnabled: true
   registryStore: local

pkg/gateway/gateway.proto

@@ -17,6 +17,12 @@ service GatewayService {
   rpc ReplaceObjectContent(ReplaceObjectContentRequest)
       returns (ReplaceObjectContentResponse) {}
 
+  // Multipart Upload
+  rpc CreatePresignedURL(CreatePresignedURLRequest) returns (CreatePresignedURLResponse) {}
+  rpc CreateMultipartUpload(CreateMultipartUploadRequest) returns (CreateMultipartUploadResponse) {}
+  rpc CompleteMultipartUpload(CompleteMultipartUploadRequest) returns (CompleteMultipartUploadResponse) {}
+  rpc AbortMultipartUpload(AbortMultipartUploadRequest) returns (AbortMultipartUploadResponse) {}
+
   // Containers
   rpc ListContainers(ListContainersRequest) returns (ListContainersResponse) {}
   rpc StopContainer(StopContainerRequest) returns (StopContainerResponse) {}
@@ -517,4 +523,83 @@ message ExportWorkspaceConfigResponse {
   string gateway_grpc_url = 3;
   int32 gateway_grpc_port = 4;
   string workspace_id = 5;
-}
+}
+
+message PresignedURLParams {
+  string upload_id = 1;
+  uint32 part_number = 2;
+
+  uint64 content_length = 3;
+  string content_type = 4;
+}
+
+enum PresignedURLMethod {
+  GetObject = 0;
+  PutObject = 1;
+  HeadObject = 2;
+  UploadPart = 3;
+}
+
+message CreatePresignedURLRequest {
+  string volume_name = 1;
+  string volume_path = 2;
+  uint32 expires = 3;
+
+  PresignedURLMethod method = 4;
+  PresignedURLParams params = 5;
+}
+
+message CreatePresignedURLResponse {
+  bool ok = 1;
+  string err_msg = 2;
+  string url = 3;
+}
+
+message CreateMultipartUploadRequest {
+  string volume_name = 1;
+  string volume_path = 2;
+  uint64 chunk_size = 3;
+  uint64 file_size = 4;
+}
+
+message FileUploadPart {
+  uint32 number = 1;
+  uint64 start = 2;
+  uint64 end = 3;
+  string url = 4;
+}
+
+message CreateMultipartUploadResponse {
+  bool ok = 1;
+  string err_msg = 2;
+  string upload_id = 3;
+  repeated FileUploadPart file_upload_parts = 4;
+}
+
+message CompletedPart {
+  uint32 number = 1;
+  string etag = 2;
+}
+
+message CompleteMultipartUploadRequest {
+  string upload_id = 1;
+  string volume_name = 2;
+  string volume_path = 3;
+  repeated CompletedPart completed_parts = 4;
+}
+
+message CompleteMultipartUploadResponse {
+  bool ok = 1;
+  string err_msg = 2;
+}
+
+message AbortMultipartUploadRequest {
+  string upload_id = 1;
+  string volume_name = 2;
+  string volume_path = 3;
+}
+
+message AbortMultipartUploadResponse {
+  bool ok = 1;
+  string err_msg = 2;
+}