Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'upstream/main' into coverity_fixes
Browse files Browse the repository at this point in the history
Garra1980 committed Oct 10, 2024
2 parents 77b4b81 + 9c85f18 commit b8ab4cf
Showing 118 changed files with 1,735 additions and 2,221 deletions.
33 changes: 33 additions & 0 deletions .github/workflows/bandit-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: Bandit Check
on:
workflow_dispatch:
schedule:
- cron: '5 23 * * 1'
push:
paths:
- .github/workflows/bandit-check.yml

permissions: read-all

jobs:
bandit-check:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Install Python
uses: actions/setup-python@v5

- name: Bandit check
run: |
pip install bandit
bandit -c bandit.yml -r ./scripts ./test --exit-zero -f custom --msg-template \
"{relpath}:{line:<4} {test_id}[bandit]:{severity}: {msg}"
bandit -c bandit.yml -r ./scripts ./test --exit-zero -f html -o bandit_report.html
- name: Upload report to artifacts
uses: actions/upload-artifact@v4
with:
name: Bandit Report
path: bandit_report.html
164 changes: 164 additions & 0 deletions .github/workflows/coverity.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
name: Coverity

on:
workflow_dispatch:
schedule:
- cron: "5 2 * * 1"
push:
paths:
- .github/workflows/coverity.yml

permissions: read-all

jobs:
coverity:
name: Coverity
runs-on: ubuntu-latest
env:
COV_TOKEN: jQxsG_hSGU57M7A7lqyBvg
LLVM_SHA_FILE: /home/runner/work/mlir-extensions/mlir-extensions/build_tools/llvm_version.txt

defaults:
run:
shell: bash -leo pipefail {0}

timeout-minutes: 450

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Cache coverity
id: cache-coverity
uses: actions/cache@v4
env:
COVERITY_CACHE_NUMBER: 1 # Increase to reset cache
with:
path: |
/home/runner/coverity
key: ${{ runner.os }}-${{ env.COVERITY_CACHE_NUMBER }}

- name: Setup conda
uses: conda-incubator/setup-miniconda@v3
with:
auto-update-conda: true
python-version: ${{ matrix.python }}
activate-environment: imex-devel

- name: Conda info
run: conda info

- name: Install Build tools
run: |
conda install cmake ninja conda-forge::lit conda-forge::doxygen
conda list
- name: Download coverity
if: steps.cache-coverity.outputs.cache-hit != 'true'
run: |
cd
curl -X POST https://scan.coverity.com/download/linux64 -d "token=$COV_TOKEN" --data-urlencode "project=MLIR Extensions" -o coverity.tgz
tar zxf coverity.tgz
mv -T cov-analysis-linux64-* coverity
- name: Add coverity to PATH
run: |
echo "$HOME/coverity/bin" >> $GITHUB_PATH
- name: Show coverity version
run: |
coverity --version
- name: Add coverity inputs to env
run: |
version="$(git rev-parse --short HEAD)"
email="alexei.fedotov@intel.com"
project_id=30755
echo "project_id=$project_id" | tee -a $GITHUB_ENV
echo "email=$email" | tee -a $GITHUB_ENV
echo "version=$version" | tee -a $GITHUB_ENV
- name: Setup Cache Vars
run: |
echo 'LLVM_SHA<<EOF' >> $GITHUB_ENV
cat $LLVM_SHA_FILE >> $GITHUB_ENV
echo 'EOF' >> $GITHUB_ENV
- name: Cache LLLVM-MLIR
id: cache-llvm-mlir
uses: actions/cache@v4
env:
LLVM_CACHE_NUMBER: 1 # Increase to reset cache
with:
path: |
/home/runner/work/llvm-mlir/_mlir_install/**
key: ${{ runner.os }}-build-llvm-${{ env.LLVM_CACHE_NUMBER }}-${{ env.LLVM_SHA }}

- name: Build LLVM-MLIR
if: steps.cache-llvm-mlir.outputs.cache-hit != 'true'
run: |
mkdir -p /home/runner/work/llvm-mlir
cd /home/runner/work/llvm-mlir
echo "INFO: Need to rebuild LLVM-MLIR. Previous installation for MLIR not found"
np=`nproc`
echo "INFO: nproc $np"
git clone https://github.com/llvm/llvm-project --branch main --single-branch
cd llvm-project
git checkout ${LLVM_SHA}
git apply /home/runner/work/mlir-extensions/mlir-extensions/build_tools/patches/*.patch
cmake -G Ninja -B _build -S llvm \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DLLVM_ENABLE_PROJECTS=mlir \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_USE_LINKER=gold \
-DLLVM_INSTALL_UTILS=ON \
-DLLVM_TARGETS_TO_BUILD=X86 \
-DLLVM_ENABLE_BINDINGS=OFF \
-DLLVM_ENABLE_ZSTD=OFF \
-DCMAKE_INSTALL_PREFIX=/home/runner/work/llvm-mlir/_mlir_install
cmake --build _build --target install
- name: Create coverity build
run: |
external_lit=`which lit`
echo ${external_lit}
cd /home/runner/work/mlir-extensions/mlir-extensions
cmake -S . -B _build -GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_EXTERNAL_LIT=${external_lit} \
-DMLIR_DIR=/home/runner/work/llvm-mlir/_mlir_install/lib/cmake/mlir \
-DLLVM_LIT_ARGS=-a
cov-build --dir $HOME/cov-int cmake --build _build --target check-imex
- name: Create coverity results tarball
run: |
cd $HOME
tar zcf cov-int.tgz cov-int
- name: Create coverity artifact
run: |
cd $HOME
curl -X POST \
-d version="$version" \
-d email="$email" \
-d token=$COV_TOKEN \
-d file_name="cov-int.tgz" \
https://scan.coverity.com/projects/$project_id/builds/init \
| tee response
upload_url="$(jq -r '.url' response)"
build_id="$(jq -r '.build_id' response)"
echo "upload_url=$upload_url" >> $GITHUB_ENV
echo "build_id=$build_id" | tee -a $GITHUB_ENV
- name: Upload coverity build
run: |
cd $HOME
curl -X PUT \
--header 'Content-Type: application/json' \
--upload-file cov-int.tgz \
$upload_url
curl -X PUT \
-d token=$COV_TOKEN \
https://scan.coverity.com/projects/$project_id/builds/$build_id/enqueue
6 changes: 3 additions & 3 deletions .github/workflows/pre-commit.yml
Original file line number Diff line number Diff line change
@@ -12,6 +12,6 @@ jobs:
pre-commit:
runs-on: gasp
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: intel-innersource/frameworks.actions.thirdparty.pre-commit-action@v3.0.0
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- uses: pre-commit/action@v3.0.1
186 changes: 186 additions & 0 deletions bandit.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@

### Bandit config file generated from:
# '/bandit-config-generator -o bandit.yaml'

### This config may optionally select a subset of tests to run or skip by
### filling out the 'tests' and 'skips' lists given below. If no tests are
### specified for inclusion then it is assumed all tests are desired. The skips
### set will remove specific tests from the include set. This can be controlled
### using the -t/-s CLI options. Note that the same test ID should not appear
### in both 'tests' and 'skips', this would be nonsensical and is detected by
### Bandit at runtime.

# Available tests:
# B101 : assert_used
# B102 : exec_used
# B103 : set_bad_file_permissions
# B104 : hardcoded_bind_all_interfaces
# B105 : hardcoded_password_string
# B106 : hardcoded_password_funcarg
# B107 : hardcoded_password_default
# B108 : hardcoded_tmp_directory
# B110 : try_except_pass
# B112 : try_except_continue
# B113 : request_without_timeout
# B201 : flask_debug_true
# B202 : tarfile_unsafe_members
# B301 : pickle
# B302 : marshal
# B303 : md5
# B304 : ciphers
# B305 : cipher_modes
# B306 : mktemp_q
# B307 : eval
# B308 : mark_safe
# B310 : urllib_urlopen
# B311 : random
# B312 : telnetlib
# B313 : xml_bad_cElementTree
# B314 : xml_bad_ElementTree
# B315 : xml_bad_expatreader
# B316 : xml_bad_expatbuilder
# B317 : xml_bad_sax
# B318 : xml_bad_minidom
# B319 : xml_bad_pulldom
# B320 : xml_bad_etree
# B321 : ftplib
# B323 : unverified_context
# B324 : hashlib_insecure_functions
# B401 : import_telnetlib
# B402 : import_ftplib
# B403 : import_pickle
# B404 : import_subprocess
# B405 : import_xml_etree
# B406 : import_xml_sax
# B407 : import_xml_expat
# B408 : import_xml_minidom
# B409 : import_xml_pulldom
# B410 : import_lxml
# B411 : import_xmlrpclib
# B412 : import_httpoxy
# B413 : import_pycrypto
# B415 : import_pyghmi
# B501 : request_with_no_cert_validation
# B502 : ssl_with_bad_version
# B503 : ssl_with_bad_defaults
# B504 : ssl_with_no_version
# B505 : weak_cryptographic_key
# B506 : yaml_load
# B507 : ssh_no_host_key_verification
# B508 : snmp_insecure_version
# B509 : snmp_weak_cryptography
# B601 : paramiko_calls
# B602 : subprocess_popen_with_shell_equals_true
# B603 : subprocess_without_shell_equals_true
# B604 : any_other_function_with_shell_equals_true
# B605 : start_process_with_a_shell
# B606 : start_process_with_no_shell
# B607 : start_process_with_partial_path
# B608 : hardcoded_sql_expressions
# B609 : linux_commands_wildcard_injection
# B610 : django_extra_used
# B611 : django_rawsql_used
# B612 : logging_config_insecure_listen
# B701 : jinja2_autoescape_false
# B702 : use_of_mako_templates
# B703 : django_mark_safe

# (optional) list included test IDs here, eg '[B101, B406]':
tests: []

# (optional) list skipped test IDs here, eg '[B101, B406]':
skips: []

### (optional) plugin settings - some test plugins require configuration data
### that may be given here, per-plugin. All bandit test plugins have a built in
### set of sensible defaults and these will be used if no configuration is
### provided. It is not necessary to provide settings for every (or any) plugin
### if the defaults are acceptable.
assert_used:
skips: ['./benchmarks/*'] # accept those assert in test scripts
hardcoded_tmp_directory:
tmp_dirs:
- /tmp
- /var/tmp
- /dev/shm
# subprocess_popen_with_shell_equals_true: #B602
# subprocess_without_shell_equals_true: #B603
# any_other_function_with_shell_equals_true: #B604
# start_process_with_a_shell: #B605
# start_process_with_no_shell: #B606
# start_process_with_partial_path: #B607
# linux_commands_wildcard_injection: #B609
# test ID B6* shares a configuration in the same family, namely shell_injection
shell_injection:
no_shell:
- os.execl
- os.execle
- os.execlp
- os.execlpe
- os.execv
- os.execve
- os.execvp
- os.execvpe
- os.spawnl
- os.spawnle
- os.spawnlp
- os.spawnlpe
- os.spawnv
- os.spawnve
- os.spawnvp
- os.spawnvpe
- os.startfile
shell:
- os.system
- os.popen
- os.popen2
- os.popen3
- os.popen4
- popen2.popen2
- popen2.popen3
- popen2.popen4
- popen2.Popen3
- popen2.Popen4
- commands.getoutput
- commands.getstatusoutput
- subprocess.getoutput
- subprocess.getstatusoutput
subprocess:
- subprocess.Popen
- subprocess.call
- subprocess.check_call
- subprocess.check_output
- subprocess.run
ssl_with_bad_defaults:
bad_protocol_versions:
- PROTOCOL_SSLv2
- SSLv2_METHOD
- SSLv23_METHOD
- PROTOCOL_SSLv3
- PROTOCOL_TLSv1
- SSLv3_METHOD
- TLSv1_METHOD
- PROTOCOL_TLSv1_1
- TLSv1_1_METHOD
ssl_with_bad_version:
bad_protocol_versions:
- PROTOCOL_SSLv2
- SSLv2_METHOD
- SSLv23_METHOD
- PROTOCOL_SSLv3
- PROTOCOL_TLSv1
- SSLv3_METHOD
- TLSv1_METHOD
- PROTOCOL_TLSv1_1
- TLSv1_1_METHOD
try_except_continue:
check_typed_exception: false
try_except_pass:
check_typed_exception: false
weak_cryptographic_key:
weak_key_size_dsa_high: 1024
weak_key_size_dsa_medium: 2048
weak_key_size_ec_high: 160
weak_key_size_ec_medium: 224
weak_key_size_rsa_high: 1024
weak_key_size_rsa_medium: 2048
2 changes: 1 addition & 1 deletion build_tools/llvm_version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
08a61eb01172054fc5f8c78ff527f01d9768569b
add6b2f35f2bcf1f59a2ab2d5b3dab124fe0895a
Loading

0 comments on commit b8ab4cf

Please sign in to comment.