From 3a072379d9a21c111f84cba1b32532b1254493fd Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 10:21:15 +0200 Subject: [PATCH 1/8] Production-grade Dockerfile --- Dockerfile | 34 +++--------------- Dockerfile.dev | 70 +++++++++++++++++++++++++++++++++++++ tools/docker-network/run.sh | 4 +-- 3 files changed, 77 insertions(+), 31 deletions(-) create mode 100644 Dockerfile.dev diff --git a/Dockerfile b/Dockerfile index 4a69168eb..560c936dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ -ARG WITH_GO_WORK=0 # https://hub.docker.com/_/golang -FROM golang:1.21-bullseye AS base +FROM golang:1.21-bookworm AS build ARG BUILD_TAGS=rocksdb @@ -13,27 +12,7 @@ RUN mkdir /scratch /app WORKDIR /scratch -FROM base AS env-with-go-work-0 - -# Here we assume our build context is the parent directory of iota-core -COPY . ./iota-core - -# We don't want go.work files to interfere in this build environment -RUN rm -f /scratch/iota-core/go.work /scratch/iota-core/go.work.sum - -FROM base AS env-with-go-work-1 - -COPY ./iota-core ./iota-core -COPY ./iota.go ./iota.go -COPY ./hive.go ./hive.go -COPY ./inx/go ./inx/go -COPY ./inx-app ./inx-app -COPY ./go.work ./ -COPY ./go.work.sum ./ - -FROM env-with-go-work-${WITH_GO_WORK} AS build - -WORKDIR /scratch/iota-core +COPY . . # Ensure ca-certificates are up to date RUN update-ca-certificates @@ -42,8 +21,7 @@ ENV GOCACHE=/go/cache # Download go modules RUN --mount=type=cache,target=/go go mod download -# Do not verify modules if we have local modules coming from go.work -RUN --mount=type=cache,target=/go if [ "${WITH_GO_WORK}" = "0" ]; then go mod verify; fi +RUN --mount=type=cache,target=/go go mod verify # Build the binary RUN --mount=type=cache,target=/go go build -o /app/iota-core -tags="$BUILD_TAGS" -ldflags='-w -s' @@ -52,14 +30,12 @@ RUN --mount=type=cache,target=/go go build -o /app/iota-core -tags="$BUILD_TAGS" RUN cp ./config_defaults.json /app/config.json RUN cp ./peering.json /app/peering.json -RUN mkdir -p /app/data/peerdb - ############################ # Runtime Image ############################ -# https://console.cloud.google.com/gcr/images/distroless/global/cc-debian11 +# https://console.cloud.google.com/gcr/images/distroless/global/cc-debian12 # using distroless cc "nonroot" image, which includes everything in the base image (glibc, libssl and openssl) -FROM gcr.io/distroless/cc-debian11:nonroot +FROM gcr.io/distroless/cc-debian12:nonroot # Copy the app dir into distroless image COPY --chown=nonroot:nonroot --from=build /app /app diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000..4a69168eb --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,70 @@ +ARG WITH_GO_WORK=0 +# https://hub.docker.com/_/golang +FROM golang:1.21-bullseye AS base + +ARG BUILD_TAGS=rocksdb + +LABEL org.label-schema.description="IOTA core node" +LABEL org.label-schema.name="iotaledger/iota-core" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.vcs-url="https://github.com/iotaledger/iota-core" + +RUN mkdir /scratch /app + +WORKDIR /scratch + +FROM base AS env-with-go-work-0 + +# Here we assume our build context is the parent directory of iota-core +COPY . ./iota-core + +# We don't want go.work files to interfere in this build environment +RUN rm -f /scratch/iota-core/go.work /scratch/iota-core/go.work.sum + +FROM base AS env-with-go-work-1 + +COPY ./iota-core ./iota-core +COPY ./iota.go ./iota.go +COPY ./hive.go ./hive.go +COPY ./inx/go ./inx/go +COPY ./inx-app ./inx-app +COPY ./go.work ./ +COPY ./go.work.sum ./ + +FROM env-with-go-work-${WITH_GO_WORK} AS build + +WORKDIR /scratch/iota-core + +# Ensure ca-certificates are up to date +RUN update-ca-certificates + +ENV GOCACHE=/go/cache + +# Download go modules +RUN --mount=type=cache,target=/go go mod download +# Do not verify modules if we have local modules coming from go.work +RUN --mount=type=cache,target=/go if [ "${WITH_GO_WORK}" = "0" ]; then go mod verify; fi + +# Build the binary +RUN --mount=type=cache,target=/go go build -o /app/iota-core -tags="$BUILD_TAGS" -ldflags='-w -s' + +# Copy the assets +RUN cp ./config_defaults.json /app/config.json +RUN cp ./peering.json /app/peering.json + +RUN mkdir -p /app/data/peerdb + +############################ +# Runtime Image +############################ +# https://console.cloud.google.com/gcr/images/distroless/global/cc-debian11 +# using distroless cc "nonroot" image, which includes everything in the base image (glibc, libssl and openssl) +FROM gcr.io/distroless/cc-debian11:nonroot + +# Copy the app dir into distroless image +COPY --chown=nonroot:nonroot --from=build /app /app + +WORKDIR /app +USER nonroot + +ENTRYPOINT ["/app/iota-core"] diff --git a/tools/docker-network/run.sh b/tools/docker-network/run.sh index ba3703ae3..924fdce0f 100755 --- a/tools/docker-network/run.sh +++ b/tools/docker-network/run.sh @@ -26,12 +26,12 @@ echo "Build iota-core" # Setup necessary environment variables. export DOCKER_BUILD_CONTEXT="../../" -export DOCKERFILE_PATH="./Dockerfile" +export DOCKERFILE_PATH="./Dockerfile.dev" if [[ "$WITH_GO_WORK" -eq 1 ]] then export DOCKER_BUILD_CONTEXT="../../../" - export DOCKERFILE_PATH="./iota-core/Dockerfile" + export DOCKERFILE_PATH="./iota-core/Dockerfile.dev" fi # Allow docker compose to build and cache an image From 807b1af3c8249c53039866d5a6fdf096dfdaf23e Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 10:21:44 +0200 Subject: [PATCH 2/8] Recent genesis time for FeatureNet --- tools/genesis-snapshot/presets/presets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/genesis-snapshot/presets/presets.go b/tools/genesis-snapshot/presets/presets.go index d6627393d..87b841d7b 100644 --- a/tools/genesis-snapshot/presets/presets.go +++ b/tools/genesis-snapshot/presets/presets.go @@ -150,7 +150,7 @@ var Feature = []options.Option[snapshotcreator.Options]{ iotago.NewV3ProtocolParameters( iotago.WithNetworkOptions("feature", "rms"), iotago.WithSupplyOptions(10_000_000_000, 100, 1, 10, 100, 100, 100), - iotago.WithTimeProviderOptions(1689848996, 10, 13), + iotago.WithTimeProviderOptions(1697406181, 10, 13), iotago.WithLivenessOptions(30, 30, 10, 20, 30), // increase/decrease threshold = fraction * slotDurationInSeconds * schedulerRate iotago.WithCongestionControlOptions(500, 500, 500, 800000, 500000, 100000, 1000, 100), From 789489c2f5dd5a5b4a3e69050a0d10dc691c3ac2 Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:27:04 +0200 Subject: [PATCH 3/8] Featurenet GitHub action --- .github/workflows/feature-network-deploy.yml | 78 ++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/feature-network-deploy.yml diff --git a/.github/workflows/feature-network-deploy.yml b/.github/workflows/feature-network-deploy.yml new file mode 100644 index 000000000..4f5b665fc --- /dev/null +++ b/.github/workflows/feature-network-deploy.yml @@ -0,0 +1,78 @@ +name: Feature network deploy +on: + push: + branches: + - develop + workflow_dispatch: + inputs: + snapshotUrl: + description: 'Custom snapshot URL:' + required: false + default: "" + +jobs: + deploy: + environment: feature + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: 1 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: '${{ secrets.IOTALEDGER_DOCKER_USERNAME }}' + password: '${{ secrets.IOTALEDGER_DOCKER_PASSWORD }}' + + - name: Publish to Docker Hub + uses: docker/build-push-action@v3 + with: + tags: iotaledger/iota-core:feature + push: true + build-args: | + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + + - # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Install SSH Key + env: + SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} + run: | + mkdir -p ~/.ssh/ + echo "$SSH_DEPLOY_KEY" > ~/.ssh/id_ed25519 + sudo chmod 600 ~/.ssh/id_ed25519 + echo ' + Host * + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + ' > ~/.ssh/config + + - name: Ansible deploy + env: + CUSTOM_SNAPSHOT_URL: '${{ github.event.inputs.snapshotUrl }}' + DEFAULT_SNAPSHOT_URL: 'https://0x0.st/HJXh.bin' + NETWORK_ENVIRONMENT: '${{ secrets.NETWORK_ENVIRONMENT }}' + IOTA_CORE_DOCKER_IMAGE_REPO: 'iotaledger/iota-core' + IOTA_CORE_DOCKER_IMAGE_TAG: 'feature' + run: | + ./deploy/ansible/run.sh feature.yml From da35f62675b50bd608f6e0e7d5ab328cf495ab66 Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:27:21 +0200 Subject: [PATCH 4/8] Adapt Ansible code to download snapshot from URL --- deploy/ansible/roles/iota-core-node/tasks/main.yml | 6 +++--- deploy/ansible/run.sh | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/deploy/ansible/roles/iota-core-node/tasks/main.yml b/deploy/ansible/roles/iota-core-node/tasks/main.yml index 71b47945e..6cb8422be 100644 --- a/deploy/ansible/roles/iota-core-node/tasks/main.yml +++ b/deploy/ansible/roles/iota-core-node/tasks/main.yml @@ -9,9 +9,9 @@ state: directory mode: '0755' -- name: Copy snapshot file - copy: - src: "{{ snapshot_path }}" +- name: Download snapshot file + get_url: + url: "{{ customSnapshotUrl if customSnapshotUrl else defaultSnapshotUrl }}" dest: /opt/iota-core/snapshot.bin mode: '0644' diff --git a/deploy/ansible/run.sh b/deploy/ansible/run.sh index b5da2d43f..766e1e3df 100755 --- a/deploy/ansible/run.sh +++ b/deploy/ansible/run.sh @@ -1,3 +1,5 @@ +eval "$NETWORK_ENVIRONMENT" + export ANSIBLE_STRATEGY=free export ANSIBLE_PIPELINING=true export ANSIBLE_PERSISTENT_CONTROL_PATH_DIR="/tmp/" @@ -6,7 +8,8 @@ ARGS=("$@") ansible-playbook -u root -i deploy/ansible/hosts/"${1:-feature.yml}" \ --forks 20 --ssh-common-args "-o ControlMaster=auto -o ControlPersist=5m" \ --extra-vars \ -"snapshot_path=$SNAPSHOT_PATH +"customSnapshotUrl=$CUSTOM_SNAPSHOT_URL +defaultSnapshotUrl=$DEFAULT_SNAPSHOT_URL iota_core_docker_image_repo=$IOTA_CORE_DOCKER_IMAGE_REPO iota_core_docker_image_tag=$IOTA_CORE_DOCKER_IMAGE_TAG wireguard_server_private_key=$WIREGUARD_SERVER_PRIVKEY From 70566ef99093a6525c3acdd1321a0bee7ca48929 Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:48:18 +0200 Subject: [PATCH 5/8] Make action show-up in PR --- .github/workflows/feature-network-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/feature-network-deploy.yml b/.github/workflows/feature-network-deploy.yml index 4f5b665fc..5ef951574 100644 --- a/.github/workflows/feature-network-deploy.yml +++ b/.github/workflows/feature-network-deploy.yml @@ -1,5 +1,6 @@ name: Feature network deploy on: + pull_request: push: branches: - develop From 72c8da57b7f2295b7a5d7c6a62e2ff13d482658e Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:55:43 +0200 Subject: [PATCH 6/8] Remove one-time Action show --- .github/workflows/feature-network-deploy.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/feature-network-deploy.yml b/.github/workflows/feature-network-deploy.yml index 5ef951574..c4c71e425 100644 --- a/.github/workflows/feature-network-deploy.yml +++ b/.github/workflows/feature-network-deploy.yml @@ -1,6 +1,5 @@ name: Feature network deploy on: - pull_request: push: branches: - develop @@ -75,5 +74,4 @@ jobs: NETWORK_ENVIRONMENT: '${{ secrets.NETWORK_ENVIRONMENT }}' IOTA_CORE_DOCKER_IMAGE_REPO: 'iotaledger/iota-core' IOTA_CORE_DOCKER_IMAGE_TAG: 'feature' - run: | - ./deploy/ansible/run.sh feature.yml + run: ./deploy/ansible/run.sh feature.yml From 44c183598d570c1a99675ffec51c7492b7efa16e Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:16:34 +0200 Subject: [PATCH 7/8] Upgrade Debian base image for Dockerfile.dev --- Dockerfile.dev | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 4a69168eb..e9073b384 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,6 +1,6 @@ ARG WITH_GO_WORK=0 # https://hub.docker.com/_/golang -FROM golang:1.21-bullseye AS base +FROM golang:1.21-bookworm AS base ARG BUILD_TAGS=rocksdb @@ -57,9 +57,9 @@ RUN mkdir -p /app/data/peerdb ############################ # Runtime Image ############################ -# https://console.cloud.google.com/gcr/images/distroless/global/cc-debian11 +# https://console.cloud.google.com/gcr/images/distroless/global/cc-debian12 # using distroless cc "nonroot" image, which includes everything in the base image (glibc, libssl and openssl) -FROM gcr.io/distroless/cc-debian11:nonroot +FROM gcr.io/distroless/cc-debian12:nonroot # Copy the app dir into distroless image COPY --chown=nonroot:nonroot --from=build /app /app From 87a0a71b9dc5e648429c6d3b996860ffaf48253d Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:15:35 +0200 Subject: [PATCH 8/8] Remove Dockerfile caches --- Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 560c936dc..44a17fda1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,14 +17,12 @@ COPY . . # Ensure ca-certificates are up to date RUN update-ca-certificates -ENV GOCACHE=/go/cache - # Download go modules -RUN --mount=type=cache,target=/go go mod download -RUN --mount=type=cache,target=/go go mod verify +RUN go mod download +RUN go mod verify # Build the binary -RUN --mount=type=cache,target=/go go build -o /app/iota-core -tags="$BUILD_TAGS" -ldflags='-w -s' +RUN go build -o /app/iota-core -tags="$BUILD_TAGS" -ldflags='-w -s' # Copy the assets RUN cp ./config_defaults.json /app/config.json