diff --git a/.circleci/config.yml b/.circleci/config.yml index 19019e4..8b1d050 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,31 @@ version: 2.1 +aliases: + - &run-always + branches: + only: /.*/ + tags: + only: /.*/ + + - &run-never + branches: + ignore: /.*/ + tags: + ignore: /.*/ + + - &run-on-release + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ + parameters: + cimg_base_version: + type: string + default: "stable" + docker_version: + type: string + default: "docker24" python_version: type: string default: "3.8" @@ -39,9 +64,9 @@ jobs: python3 -m unittest utils.tests.test_embedding python3 -m unittest utils.tests.test_os - docker_release: + container_build: docker: - - image: cimg/base:stable + - image: cimg/base:<< pipeline.parameters.cimg_base_version >> steps: - checkout - run: @@ -53,9 +78,16 @@ jobs: export DOCKER_TAG="${DOCKER_TAG//v}" echo "Docker tag will be: ${DOCKER_TAG}" + if [[ "${DOCKER_TAG}" == "" ]]; then + export DOCKER_TAG="dev" + fi + # Persists environment variable to future steps echo "export DOCKER_TAG=\"${DOCKER_TAG}\"" >> $BASH_ENV - - setup_remote_docker + echo "export DOCKER_TAG=\"${DOCKER_TAG}\"" > ~/tags.env + - setup_remote_docker: + version: << pipeline.parameters.docker_version >> + docker_layer_caching: true - run: name: Set Base Image Digest command: | @@ -70,6 +102,9 @@ jobs: echo "export BASE_DIGEST=\"${BASE_DIGEST}\"" >> $BASH_ENV - run: name: Build Docker Image + environment: + BUILDKIT_PROGRESS: plain + DOCKER_BUILLDKIT: 1 command: | echo "Building docker image bitcamo:${DOCKER_TAG}..." @@ -83,13 +118,37 @@ jobs: - run: name: Log Image Inspection command: docker inspect bitcamo:${DOCKER_TAG} + - run: + name: Save Container Image + command: | + mkdir ~/images + docker save -o ~/images/bitcamo.tar bitcamo:${DOCKER_TAG} + - persist_to_workspace: + root: /home/circleci + paths: + - images + - tags.env + + container_publish: + docker: + - image: cimg/base:<< pipeline.parameters.cimg_base_version >> + steps: + - attach_workspace: + at: ~/workspace + - setup_remote_docker: + version: << pipeline.parameters.docker_version >> + docker_layer_caching: true + - run: + name: Load saved image + command: docker load -i ~/workspace/images/bitcamo.tar - run: name: Authenticate with Github command: | - echo $GITHUB_TOKEN | docker login ghcr.io -u juburr --password-stdin + echo $GITHUB_TOKEN | docker login ghcr.io -u $GITHUB_USER --password-stdin - run: name: Tag and Push Docker Image command: | + $(cat ~/workspace/tags.env | xargs) docker tag bitcamo:${DOCKER_TAG} ghcr.io/juburr/bitcamo:${DOCKER_TAG} docker tag bitcamo:${DOCKER_TAG} ghcr.io/juburr/bitcamo:latest docker push ghcr.io/juburr/bitcamo:${DOCKER_TAG} @@ -105,12 +164,12 @@ workflows: only: /.*/ branches: only: /.*/ - - docker_release: - context: github_auth_npm_rw + - container_build: requires: - unit_tests - filters: - tags: - only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ - branches: - ignore: /.*/ + filters: *run-always + - container_publish: + context: github_bitcamo_rw + requires: + - container_build + filters: *run-on-release diff --git a/.dockerignore b/.dockerignore index f07c822..8b6a184 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,5 @@ +**/__pycache__ +.git .venv -notebooks \ No newline at end of file +notebooks +resources \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 5b914be..d812c1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM ubuntu:20.04 LABEL org.opencontainers.image.title="BitCamo" LABEL org.opencontainers.image.description="Creates adversarial Windows PE files for evading ML detection models" LABEL org.opencontainers.image.source="https://github.com/juburr/bitcamo" -LABEL org.opencontainers.image.authors="Justin Burr " +LABEL org.opencontainers.image.authors="Justin Burr" LABEL org.opencontainers.image.base.name="docker.io/library/ubuntu:20.04" RUN apt-get -y update && \ diff --git a/bitcamo.py b/bitcamo.py index 63aaf79..a464f5f 100755 --- a/bitcamo.py +++ b/bitcamo.py @@ -59,7 +59,7 @@ def post_attack(sample, attack_config): return sample def attack(samples, attack_config): - malconv = MalConv(attack_mode=True) + malconv = MalConv(attack_mode=True, verbose=attack_config.verbose) count = 0 for s in samples: count = count + 1 diff --git a/models/malconv.py b/models/malconv.py index d5df6fb..f3267b6 100644 --- a/models/malconv.py +++ b/models/malconv.py @@ -35,7 +35,7 @@ class MalConv: URL: https://arxiv.org/pdf/1804.04637.pdf """ - def __init__(self, attack_mode=False): + def __init__(self, attack_mode=False, verbose=False): self.attack_mode = attack_mode self.padding_char = 256 self.dict_size = 257 @@ -46,6 +46,7 @@ def __init__(self, attack_mode=False): self.model = self._load_full_model() self.embedder = None self.embedding_space_model = None + self.verbose = "auto" if verbose == True else 0 # Conserve memory by only loading the additional models when in attack mode if attack_mode: @@ -91,9 +92,13 @@ def _load_full_model(self): A pre-trained MalConv model """ model = load_model(self.model_path, compile = False) + lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( + initial_learning_rate=0.01, + decay_steps=10000, + decay_rate=1e-3) model.compile( loss='binary_crossentropy', - optimizer=SGD(learning_rate=0.01, momentum=0.9, nesterov=True, decay=1e-3), + optimizer=SGD(momentum=0.9, nesterov=True, learning_rate=lr_schedule), metrics=[metrics.binary_accuracy] ) return model @@ -120,9 +125,13 @@ def _load_embedding_space_model(self): outp = Dense(1, activation='sigmoid')(dense) emb_model = Model( inp, outp ) + lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( + initial_learning_rate=0.01, + decay_steps=10000, + decay_rate=1e-3) emb_model.compile( loss='binary_crossentropy', - optimizer=SGD(learning_rate=0.01, momentum=0.9, nesterov=True, decay=1e-3), + optimizer=SGD(momentum=0.9, nesterov=True, learning_rate=lr_schedule), metrics=[metrics.binary_accuracy] ) @@ -202,7 +211,7 @@ def predict_embedded(self, z): """ if self.attack_mode == False: raise Exception('function predict_embedded() is only available in attack mode') - return self.embedding_space_model.predict(z)[0][0] + return self.embedding_space_model.predict(z, verbose=self.verbose)[0][0] def predict(self, x): """ @@ -218,7 +227,7 @@ def predict(self, x): y_hat : numpy.float32 Model prediction between 0 (benign) and 1 (malicious). """ - return self.model.predict(self._pad(x).reshape(1, -1))[0][0] + return self.model.predict(self._pad(x).reshape(1, -1), verbose=self.verbose)[0][0] def evasion_achieved(self, y_hat, y_target): """ diff --git a/requirements.txt b/requirements.txt index 46f0ab3..aeaa516 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -absl-py==0.13.0 +absl-py==2.1.0 astunparse==1.6.3 -cachetools==4.2.2 -certifi==2021.5.30 -charset-normalizer==2.0.6 +cachetools==5.3.2 +certifi==2024.2.2 +charset-normalizer==3.3.2 clang==5.0 cleverhans==4.0.0 cloudpickle==2.0.0 @@ -11,49 +11,57 @@ cycler==0.10.0 decorator==5.1.0 dm-tree==0.1.6 easydict==1.9 -flatbuffers==1.12 +flatbuffers==23.5.26 +future==0.18.3 gast==0.4.0 -google-auth==1.35.0 +google-auth==2.27.0 google-auth-oauthlib==0.4.6 google-pasta==0.2.0 -grpcio==1.40.0 -h5py==3.1.0 -idna==3.2 -joblib==1.0.1 -keras==2.6.0 +grpcio==1.53.2 +h5py==3.10.0 +idna==3.6 +importlib-metadata==7.0.1 +joblib==1.2.0 +keras==2.11.0 Keras-Preprocessing==1.1.2 kiwisolver==1.3.2 -lief==0.11.5 -Markdown==3.3.4 +libclang==16.0.6 +lief==0.12.2 +Markdown==3.5.2 +MarkupSafe==2.1.5 matplotlib==3.4.3 mnist==0.2.2 nose==1.3.7 -numpy==1.19.5 -oauthlib==3.1.1 +numpy==1.22.0 +oauthlib==3.2.2 opt-einsum==3.3.0 +packaging==23.2 pandas==1.4.2 -Pillow==9.0.1 -protobuf==3.18.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 +pefile==2021.9.3 +pillow==10.2.0 +protobuf==3.18.3 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 pycodestyle==2.7.0 pyparsing==2.4.7 python-dateutil==2.8.2 pytz==2022.1 -requests==2.26.0 -requests-oauthlib==1.3.0 -rsa==4.7.2 -scipy==1.7.1 -six==1.15.0 -tensorboard==2.6.0 +requests==2.31.0 +requests-oauthlib==1.3.1 +rsa==4.9 +scipy==1.10.0 +six==1.16.0 +tensorboard==2.11.2 tensorboard-data-server==0.6.1 -tensorboard-plugin-wit==1.8.0 -tensorflow==2.6.4 -tensorflow-estimator==2.6.0 +tensorboard-plugin-wit==1.8.1 +tensorflow==2.11.1 +tensorflow-estimator==2.11.0 +tensorflow-io-gcs-filesystem==0.34.0 tensorflow-probability==0.14.0 -termcolor==1.1.0 +termcolor==2.4.0 tqdm==4.62.3 -typing-extensions==3.7.4.3 -urllib3==1.26.6 -Werkzeug==2.0.1 -wrapt==1.12.1 +typing_extensions==4.9.0 +urllib3==2.2.0 +Werkzeug==2.3.8 +wrapt==1.16.0 +zipp==3.17.0