From 56e0176905342ad35415d90464f4e2bcc904e31b Mon Sep 17 00:00:00 2001 From: postables Date: Fri, 17 Apr 2020 01:02:43 -0700 Subject: [PATCH 1/4] remove remaining upstream references --- Dockerfile | 2 +- Dockerfile.arm.release | 2 +- Dockerfile.arm64.release | 2 +- Dockerfile.release | 2 +- Dockerfile.simpleci | 4 ++-- Makefile | 2 +- SECURITY.md | 2 +- browser/package.json | 4 ++-- minio.spec | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index c939ae092c677..7407f2b280371 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV GO111MODULE on RUN \ apk add --no-cache git && \ - git clone https://github.com/minio/minio && cd minio && \ + git clone https://github.com/RTradeLtd/s3x && cd minio && \ go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" FROM alpine:3.10 diff --git a/Dockerfile.arm.release b/Dockerfile.arm.release index 3e1f64192ff3b..f56a39eab1ea9 100644 --- a/Dockerfile.arm.release +++ b/Dockerfile.arm.release @@ -8,7 +8,7 @@ ENV GO111MODULE on RUN \ apk add --no-cache git 'curl>7.61.0' && \ - git clone https://github.com/minio/minio && \ + git clone https://github.com/RTradeLtd/s3x && \ curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . FROM arm32v7/alpine:3.10 diff --git a/Dockerfile.arm64.release b/Dockerfile.arm64.release index 71188682102ac..697aaf68208a3 100644 --- a/Dockerfile.arm64.release +++ b/Dockerfile.arm64.release @@ -8,7 +8,7 @@ ENV GO111MODULE on RUN \ apk add --no-cache git 'curl>7.61.0' && \ - git clone https://github.com/minio/minio && \ + git clone https://github.com/RTradeLtd/s3x && \ curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . FROM arm64v8/alpine:3.10 diff --git a/Dockerfile.release b/Dockerfile.release index 1dd41bf99f431..6e01a30e6cbb4 100644 --- a/Dockerfile.release +++ b/Dockerfile.release @@ -6,7 +6,7 @@ ENV GO111MODULE on RUN \ apk add --no-cache git && \ - git clone https://github.com/minio/minio + git clone https://github.com/RTradeLtd/s3x FROM alpine:3.10 diff --git a/Dockerfile.simpleci b/Dockerfile.simpleci index f87b0c8726485..1a26036813aa2 100644 --- a/Dockerfile.simpleci +++ b/Dockerfile.simpleci @@ -3,8 +3,8 @@ #------------------------------------------------------------- FROM golang:1.13 -COPY . /go/src/github.com/minio/minio -WORKDIR /go/src/github.com/minio/minio +COPY . /go/src/github.com/RTradeLtd/s3x +WORKDIR /go/src/github.com/RTradeLtd/s3x RUN apt-get update && apt-get install -y jq ENV GO111MODULE=on diff --git a/Makefile b/Makefile index d5c0245ba664c..567a6fdfcf22f 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOARCH := $(shell go env GOARCH) GOOS := $(shell go env GOOS) VERSION ?= $(shell git describe --tags) -TAG ?= "minio/minio:$(VERSION)" +TAG ?= "RTradeLtd/s3x:$(VERSION)" BUILD_LDFLAGS := '$(LDFLAGS)' diff --git a/SECURITY.md b/SECURITY.md index 4cd47af90b7c6..7fd7c3aef2189 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ Whenever there is a security update you just need to upgrade to the latest versi ## Reporting a Vulnerability -All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories) +All security bugs in [RTradeLtd/s3x](https://github,com/RTradeLtd/s3x) (or other minio/* repositories) should be reported by email to security@min.io. Your email will be acknowledged within 48 hours, and you'll receive a more detailed response to your email within 72 hours indicating the next steps in handling your report. diff --git a/browser/package.json b/browser/package.json index 90f54b43afcaa..fa1551eceb493 100644 --- a/browser/package.json +++ b/browser/package.json @@ -19,14 +19,14 @@ }, "repository": { "type": "git", - "url": "https://github.com/minio/minio" + "url": "https://github.com/RTradeLtd/s3x" }, "author": "MinIO Inc", "license": "Apache-2.0", "bugs": { "url": "https://github.com/RTradeLtd/s3x/issues" }, - "homepage": "https://github.com/minio/minio", + "homepage": "https://github.com/RTradeLtd/s3x", "devDependencies": { "async": "^1.5.2", "babel-cli": "^6.26.0", diff --git a/minio.spec b/minio.spec index 1f03a297c3763..ca03917c18cad 100644 --- a/minio.spec +++ b/minio.spec @@ -1,6 +1,6 @@ %define tag RELEASE.2017-04-25T01-27-49Z %define subver %(echo %{tag} | sed -e 's/[^0-9]//g') -# git fetch https://github.com/minio/minio.git refs/tags/RELEASE.2017-02-16T01-47-30Z +# git fetch https://github.com/RTradeLtd/s3x.git refs/tags/RELEASE.2017-02-16T01-47-30Z # git rev-list -n 1 FETCH_HEAD %define commitid 83abb310b4ce3a0dfc6d7faf78e33cb6f9132cfe Summary: Cloud Storage Server. @@ -21,7 +21,7 @@ BuildRoot: %{tmpdir}/%{name}-%{version}-root-%(id -u -n) ## Go related tags. %define gobuild(o:) go build -ldflags "${LDFLAGS:-}" %{?**}; %define gopath %{_libdir}/golang -%define import_path github.com/minio/minio +%define import_path github.com/RTradeLtd/s3x %description MinIO is an object storage server released under Apache License v2.0. From 17c8bdc4054e95ef8440b9a406a8e319e829f3ab Mon Sep 17 00:00:00 2001 From: postables Date: Fri, 17 Apr 2020 13:20:55 -0700 Subject: [PATCH 2/4] fix docker build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 567a6fdfcf22f..ac3166d7209cf 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOARCH := $(shell go env GOARCH) GOOS := $(shell go env GOOS) VERSION ?= $(shell git describe --tags) -TAG ?= "RTradeLtd/s3x:$(VERSION)" +TAG ?= "rtradetech/s3x:$(VERSION)" BUILD_LDFLAGS := '$(LDFLAGS)' From 542534128c37e3f1aec43867c1ec646e9e70856f Mon Sep 17 00:00:00 2001 From: postables Date: Fri, 17 Apr 2020 13:22:43 -0700 Subject: [PATCH 3/4] update kubernetes to use correct docker version --- kubernetes_local.yml | 2 +- kubernetes_remote.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes_local.yml b/kubernetes_local.yml index 11403ed0d1a7c..7778943e7e14b 100644 --- a/kubernetes_local.yml +++ b/kubernetes_local.yml @@ -34,7 +34,7 @@ spec: claimName: workdir containers: - name: s3x - image: rtradetech/s3x-test:latest + image: rtradetech/s3x:latest args: ["gateway", "s3x", "--temporalx.endpoint", "localhost:9090", "--temporalx.insecure"] env: - name: MINIO_ACCESS_KEY diff --git a/kubernetes_remote.yml b/kubernetes_remote.yml index 7fa116c2abd81..69b08778e9094 100644 --- a/kubernetes_remote.yml +++ b/kubernetes_remote.yml @@ -34,7 +34,7 @@ spec: claimName: workdir containers: - name: s3x - image: rtradetech/s3x-test:latest + image: rtradetech/s3x:latest args: ["gateway", "s3x", "--temporalx.insecure"] env: - name: MINIO_ACCESS_KEY From f3bbb753c0642291b07d82471322de9a25671dd8 Mon Sep 17 00:00:00 2001 From: postables Date: Fri, 17 Apr 2020 14:04:20 -0700 Subject: [PATCH 4/4] update readme --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be52ae8848937..0d8dde0d17cfe 100644 --- a/README.md +++ b/README.md @@ -104,13 +104,19 @@ In order to enable the speed needed by S3 applications we used TemporalX to prov * Compared to go-ipfs and existing IPFS implementations it is **[fast](https://medium.com/temporal-cloud/temporalx-vs-go-ipfs-official-node-benchmarks-8457037a77cf)** * Built in data replication +The default setting of s3x is to use a publicly available TemporalX server, however this is only sufficient for demo/trial purposes and in production you'll want to be running a TemporalX server yourself. To reduce network delays, and latency decreasing performance of s3x, it is recommended that you run TemporalX on the same box as you are running s3x, however this is only recommendation and it is totally fine to use a remote TemporalX node. + ## Storage All data, that is all the objects, buckets, metadata, is stored on IPFS. Because IPFS uses hashes, and S3 uses arbitrary names for the buckets and objects, we perform bookkeeping with a "ledger". The actual "on-ipfs" structure of the objects, and buckets themselves is defined via protocol buffers, saved as an IPLD "raw" object type. They contain the data if there, as well as object and bucket metadata. -## Ledger +## Ledger Store + +The "ledger store" is an internal book keeper responsible for keeping a map of bucket and objects names, as well as the latest version of their IPFS CID. The ledger is what bridges TemporalX/IPFS to S3/MinIO, and is solely responsible for this `name->hash` map. All object/bucket data is stored on IPFS, with the only "off-IPFS" being this `name->hash` map, sometimes called "ledger store metadata". The default implementation is done using a badger v2 datastore using `dgraph-io/badger` sufficient for running single s3x node installations. + +There is a second available option for this ledger store that enables highly available s3x setupts with multiple nodes. By using a CRDT datastore ontop of a badger datastore, updates to the ledger state can be broadcasted between any s3x instance configured to listen to the same topic. One thing to note is that the performance of this will be slightly worse than the pure badger ledger store. -The "ledger" is an internal book keeper responsible for keeping track of the latest IPFS CID's that belong to each and every object, and bucket stored, and is currently implemented as a `dgraph-io/badger/v2` key-value datastore. +Please note that this doesn't give high availability for the actual bucket/object data. For this you would want to use TemporalX's built-in replication system. # Kubernetes