From b8d2fbd800a7422a35f1c011238db1d8df5fa0a0 Mon Sep 17 00:00:00 2001 From: Orbital Date: Wed, 10 Jul 2024 16:47:42 -0500 Subject: [PATCH 1/2] cln: remove experimental-onion-messages cfg flag --- conf/cln/config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/cln/config b/conf/cln/config index 45716bd..48da8a2 100644 --- a/conf/cln/config +++ b/conf/cln/config @@ -8,4 +8,3 @@ bitcoin-rpcpassword=pass log-level=debug experimental-offers -experimental-onion-messages From 4f3200c7c3ba3c9b1c1a52f45de58757afa16104 Mon Sep 17 00:00:00 2001 From: Orbital Date: Wed, 10 Jul 2024 16:51:04 -0500 Subject: [PATCH 2/2] compose: add new cln docker build --- docker-compose.yml | 8 +- docker/cln/Dockerfile | 278 ++++++++++++++++++++ docker/cln/entrypoint.sh | 27 ++ docker/cln/plugins/clnrest/pyproject.toml | 22 ++ docker/cln/plugins/wss-proxy/pyproject.toml | 14 + 5 files changed, 345 insertions(+), 4 deletions(-) create mode 100644 docker/cln/Dockerfile create mode 100644 docker/cln/entrypoint.sh create mode 100644 docker/cln/plugins/clnrest/pyproject.toml create mode 100644 docker/cln/plugins/wss-proxy/pyproject.toml diff --git a/docker-compose.yml b/docker-compose.yml index 83d667c..378c738 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: ipv4_address: 172.30.1.3 cln1: - image: elementsproject/lightningd:v24.05 + build: ./docker/cln restart: unless-stopped depends_on: - bitcoind @@ -103,7 +103,7 @@ services: ipv4_address: 172.30.2.2 cln2: - image: elementsproject/lightningd:v24.05 + build: ./docker/cln restart: unless-stopped depends_on: - bitcoind @@ -146,7 +146,7 @@ services: - "ldknode2:/root/.ldknode" cln3: - image: elementsproject/lightningd:v24.05 + build: ./docker/cln restart: unless-stopped depends_on: - bitcoind @@ -262,4 +262,4 @@ networks: ipam: driver: default config: - - subnet: 172.30.0.0/16 \ No newline at end of file + - subnet: 172.30.0.0/16 diff --git a/docker/cln/Dockerfile b/docker/cln/Dockerfile new file mode 100644 index 0000000..40872df --- /dev/null +++ b/docker/cln/Dockerfile @@ -0,0 +1,278 @@ +# This Dockerfile is used by buildx to build ARM64, AMD64, and ARM32 Docker images from an AMD64 host. +# To speed up the build process, we are cross-compiling rather than relying on QEMU. +# There are four main stages: +# * downloader: Downloads specific binaries needed for c-lightning for each architecture. +# * builder: Cross-compiles for each architecture. +# * builder-python: Builds Python dependencies for cln-rest with QEMU. +# * final: Creates the runtime image. + +ARG BASE_DISTRO="debian:bullseye-slim" + +FROM --platform=$BUILDPLATFORM ${BASE_DISTRO} as base-downloader +RUN set -ex \ + && apt-get update \ + && apt-get install -qq --no-install-recommends ca-certificates dirmngr wget qemu-user-static binfmt-support + +FROM base-downloader as base-downloader-linux-amd64 +ENV TARBALL_ARCH_FINAL=x86_64-linux-gnu + +FROM base-downloader as base-downloader-linux-arm64 +ENV TARBALL_ARCH_FINAL=aarch64-linux-gnu + +FROM base-downloader as base-downloader-linux-arm +ENV TARBALL_ARCH_FINAL=arm-linux-gnueabihf + +FROM base-downloader-${TARGETOS}-${TARGETARCH} as downloader + +RUN set -ex \ + && apt-get update \ + && apt-get install -qq --no-install-recommends ca-certificates dirmngr wget + +WORKDIR /opt + + +ENV BITCOIN_VERSION=27.1 +ENV BITCOIN_TARBALL bitcoin-${BITCOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz +ENV BITCOIN_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/$BITCOIN_TARBALL +ENV BITCOIN_ASC_URL https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VERSION/SHA256SUMS + +RUN mkdir /opt/bitcoin && cd /opt/bitcoin \ + && wget -qO $BITCOIN_TARBALL "$BITCOIN_URL" \ + && wget -qO bitcoin "$BITCOIN_ASC_URL" \ + && grep $BITCOIN_TARBALL bitcoin | tee SHA256SUMS \ + && sha256sum -c SHA256SUMS \ + && BD=bitcoin-$BITCOIN_VERSION/bin \ + && tar -xzvf $BITCOIN_TARBALL $BD/ --strip-components=1 \ + && rm $BITCOIN_TARBALL + +ENV LITECOIN_VERSION 0.16.3 +ENV LITECOIN_URL https://download.litecoin.org/litecoin-${LITECOIN_VERSION}/linux/litecoin-${LITECOIN_VERSION}-${TARBALL_ARCH_FINAL}.tar.gz + +# install litecoin binaries +RUN mkdir /opt/litecoin && cd /opt/litecoin \ + && wget -qO litecoin.tar.gz "$LITECOIN_URL" \ + && tar -xzvf litecoin.tar.gz litecoin-$LITECOIN_VERSION/bin/litecoin-cli --strip-components=1 --exclude=*-qt \ + && rm litecoin.tar.gz + +FROM --platform=linux/amd64 ${BASE_DISTRO} as base-builder +RUN apt-get update -qq && \ + apt-get install -qq -y --no-install-recommends \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + curl \ + dirmngr \ + gettext \ + git \ + gnupg \ + jq \ + libpq-dev \ + libtool \ + libffi-dev \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + python3.9 \ + python3-dev \ + python3-mako \ + python3-pip \ + python3-venv \ + python3-setuptools \ + libev-dev \ + libevent-dev \ + qemu-user-static \ + wget \ + unzip \ + tclsh + +ENV PYTHON_VERSION=3 +RUN curl -sSL https://install.python-poetry.org | python3 - +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 +RUN pip3 install --upgrade pip setuptools wheel + +RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \ + wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip + +WORKDIR /opt/lightningd +RUN git init && \ + git remote add origin https://github.com/ElementsProject/lightning && \ + git fetch origin pull/7461/head:part4-corner-cases && \ + git checkout part4-corner-cases +COPY . /tmp/lightning +RUN git checkout part4-corner-cases + +# Do not build python plugins (clnrest & wss-proxy) here, python doesn't support cross compilation. +RUN sed -i '/^clnrest\|^wss-proxy/d' pyproject.toml && \ + /root/.local/bin/poetry export -o requirements.txt --without-hashes +RUN pip3 install -r requirements.txt && pip3 cache purge +WORKDIR / + +FROM base-builder as base-builder-linux-amd64 + +FROM base-builder as base-builder-linux-arm64 +ENV target_host=aarch64-linux-gnu \ + target_host_rust=aarch64-unknown-linux-gnu \ + target_host_qemu=qemu-aarch64-static + +RUN apt-get install -qq -y --no-install-recommends \ + libc6-arm64-cross \ + gcc-${target_host} \ + g++-${target_host} + +ENV AR=${target_host}-ar \ +AS=${target_host}-as \ +CC=${target_host}-gcc \ +CXX=${target_host}-g++ \ +LD=${target_host}-ld \ +STRIP=${target_host}-strip \ +QEMU_LD_PREFIX=/usr/${target_host} \ +HOST=${target_host} \ +TARGET=${target_host_rust} \ +RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \ +PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig" + +ENV \ +ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \ +SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX" + +FROM base-builder as base-builder-linux-arm + +ENV target_host=arm-linux-gnueabihf \ + target_host_rust=armv7-unknown-linux-gnueabihf \ + target_host_qemu=qemu-arm-static + +RUN apt-get install -qq -y --no-install-recommends \ + libc6-armhf-cross \ + gcc-${target_host} \ + g++-${target_host} + +ENV AR=${target_host}-ar \ +AS=${target_host}-as \ +CC=${target_host}-gcc \ +CXX=${target_host}-g++ \ +LD=${target_host}-ld \ +STRIP=${target_host}-strip \ +QEMU_LD_PREFIX=/usr/${target_host} \ +HOST=${target_host} \ +TARGET=${target_host_rust} \ +RUSTUP_INSTALL_OPTS="--target ${target_host_rust} --default-host ${target_host_rust}" \ +PKG_CONFIG_PATH="/usr/${target_host}/lib/pkgconfig" + +ENV \ +ZLIB_CONFIG="--prefix=${QEMU_LD_PREFIX}" \ +SQLITE_CONFIG="--host=${target_host} --prefix=$QEMU_LD_PREFIX" + +FROM base-builder-${TARGETOS}-${TARGETARCH} as builder + +ENV LIGHTNINGD_VERSION=master + +RUN mkdir zlib && tar xvf zlib.tar.gz -C zlib --strip-components=1 \ + && cd zlib \ + && ./configure ${ZLIB_CONFIG} \ + && make \ + && make install && cd .. && \ + rm zlib.tar.gz && \ + rm -rf zlib + +RUN unzip sqlite.zip \ + && cd sqlite-* \ + && ./configure --enable-static --disable-readline --disable-threadsafe --disable-load-extension ${SQLITE_CONFIG} \ + && make \ + && make install && cd .. && rm sqlite.zip && rm -rf sqlite-* + +ENV RUST_PROFILE=release +ENV PATH=$PATH:/root/.cargo/bin/ +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ${RUSTUP_INSTALL_OPTS} +RUN rustup toolchain install stable --component rustfmt --allow-downgrade + +COPY --from=downloader /usr/bin/${target_host_qemu} /usr/bin/${target_host_qemu} +WORKDIR /opt/lightningd + +# If cross-compiling, need to tell it to cargo. +RUN ( ! [ -n "${target_host}" ] ) || \ + (mkdir -p .cargo && echo "[target.${target_host_rust}]\nlinker = \"${target_host}-gcc\"" > .cargo/config) + +# Weird errors with cargo for cln-grpc on arm7 https://github.com/ElementsProject/lightning/issues/6596 +RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \ + (sed -i '/documentation = "https:\/\/docs.rs\/cln-grpc"/a include = ["**\/*.*"]' cln-grpc/Cargo.toml) + +# Ensure that the desired grpcio-tools & protobuf versions are installed +# https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381 +RUN /root/.local/bin/poetry lock --no-update && \ + /root/.local/bin/poetry install + +RUN ./configure --prefix=/tmp/lightning_install --enable-static && \ + make && \ + /root/.local/bin/poetry run make install + +# We need to build python plugins on the target's arch because python doesn't support cross build +FROM ${BASE_DISTRO} as builder-python +RUN apt-get update -qq && \ + apt-get install -qq -y --no-install-recommends \ + git \ + curl \ + libtool \ + pkg-config \ + autoconf \ + automake \ + build-essential \ + libffi-dev \ + libssl-dev \ + python3.9 \ + python3-dev \ + python3-pip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -sSL https://install.python-poetry.org | python3 - +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 + +ENV PYTHON_VERSION=3 +WORKDIR /opt/lightningd + +COPY plugins/clnrest/pyproject.toml plugins/clnrest/pyproject.toml +COPY plugins/wss-proxy/pyproject.toml plugins/wss-proxy/pyproject.toml + +RUN cd plugins/clnrest && \ + /root/.local/bin/poetry export -o requirements.txt --without-hashes && \ + pip3 install -r requirements.txt && \ + cd /opt/lightningd + +RUN cd plugins/wss-proxy && \ + /root/.local/bin/poetry export -o requirements.txt --without-hashes && \ + pip3 install -r requirements.txt && \ + cd /opt/lightningd && \ + pip3 cache purge + +FROM ${BASE_DISTRO} as final + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + tini \ + socat \ + inotify-tools \ + jq \ + python3.9 \ + python3-pip \ + libpq5 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV LIGHTNINGD_DATA=/root/.lightning +ENV LIGHTNINGD_RPC_PORT=9835 +ENV LIGHTNINGD_PORT=9735 +ENV LIGHTNINGD_NETWORK=bitcoin + +RUN mkdir $LIGHTNINGD_DATA && \ + touch $LIGHTNINGD_DATA/config +VOLUME [ "/root/.lightning" ] + +COPY --from=builder /tmp/lightning_install/ /usr/local/ +COPY --from=builder-python /usr/local/lib/python3.9/dist-packages/ /usr/local/lib/python3.9/dist-packages/ +COPY --from=downloader /opt/bitcoin/bin /usr/bin +COPY --from=downloader /opt/litecoin/bin /usr/bin +COPY entrypoint.sh entrypoint.sh + +EXPOSE 9735 9835 +ENTRYPOINT [ "/usr/bin/tini", "-g", "--", "./entrypoint.sh" ] \ No newline at end of file diff --git a/docker/cln/entrypoint.sh b/docker/cln/entrypoint.sh new file mode 100644 index 0000000..b0fb110 --- /dev/null +++ b/docker/cln/entrypoint.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +: "${EXPOSE_TCP:=false}" + +networkdatadir="${LIGHTNINGD_DATA}/${LIGHTNINGD_NETWORK}" + +set -m +lightningd --network="${LIGHTNINGD_NETWORK}" "$@" & + +echo "Core-Lightning starting" +while read -r i; do if [ "$i" = "lightning-rpc" ]; then break; fi; done \ + < <(inotifywait -e create,open --format '%f' --quiet "${networkdatadir}" --monitor) + +if [ "$EXPOSE_TCP" == "true" ]; then + echo "Core-Lightning started, RPC available on port $LIGHTNINGD_RPC_PORT" + + socat "TCP4-listen:$LIGHTNINGD_RPC_PORT,fork,reuseaddr" "UNIX-CONNECT:${networkdatadir}/lightning-rpc" & +fi + +# Now run any scripts which exist in the lightning-poststart.d directory +if [ -d "$LIGHTNINGD_DATA"/lightning-poststart.d ]; then + for f in "$LIGHTNINGD_DATA"/lightning-poststart.d/*; do + "$f" + done +fi + +fg %- \ No newline at end of file diff --git a/docker/cln/plugins/clnrest/pyproject.toml b/docker/cln/plugins/clnrest/pyproject.toml new file mode 100644 index 0000000..aeb2d14 --- /dev/null +++ b/docker/cln/plugins/clnrest/pyproject.toml @@ -0,0 +1,22 @@ +[tool.poetry] +name = "clnrest" +version = "24.05rc2" +description = "Transforms RPC calls into REST APIs" +authors = ["ShahanaFarooqui "] + +[tool.poetry.dependencies] +python = "^3.8" +json5 = "^0.9.14" +flask = "^2.3.3" +flask-restx = "^1.1.0" +gunicorn = "^21.2.0" +pyln-client = "^24.2" +flask-socketio = "^5.3.6" +gevent = "^23.9.0.post1" +gevent-websocket = "^0.10.1" +flask-cors = "^4.0.0" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/docker/cln/plugins/wss-proxy/pyproject.toml b/docker/cln/plugins/wss-proxy/pyproject.toml new file mode 100644 index 0000000..65e113a --- /dev/null +++ b/docker/cln/plugins/wss-proxy/pyproject.toml @@ -0,0 +1,14 @@ +[tool.poetry] +name = "wss-proxy" +version = "24.05rc2" +description = "Web secure socket proxy" +authors = ["ShahanaFarooqui "] + +[tool.poetry.dependencies] +python = "^3.8" +pyln-client = "^24.2" +websockets = "^12.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file