Final cleanup and github action, windows native run enabled

.github/workflows/cla.yml

@@ -8,6 +8,7 @@ on:
 
 jobs:
   cla-check:
+    if: github.repository_owner == 'mlcommons'
     runs-on: ubuntu-latest
     steps:
       - name: "MLCommons CLA bot check"

.github/workflows/test-mlperf-inference-abtf-poc.yml

@@ -0,0 +1,97 @@
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference ABTF POC Test
+
+on:
+  pull_request_target:
+    branches: [ "main", "poc" ]
+    paths:
+      - '.github/workflows/test-mlperf-inference-abtf-poc.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build_ubuntu:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-20.04, ubuntu-24.04]
+        python-version: [ "3.8", "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+        exclude:
+        - os: ubuntu-24.04
+          python-version: "3.8"
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        pip install cm4mlops
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+        cm run script --quiet --tags=get,sys-utils-cm
+    - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker
+      run: |
+        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet --docker --docker_it=no  -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }}
+
+  build_macos:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, macos-13]
+        python-version: [ "3.9", "3.11", "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+        exclude:
+        - os: windows-latest
+          python-version: "3.9"
+        - os: windows-latest
+          python-version: "3.12"
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        pip install cm4mlops
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+    - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} natively
+      run: |
+        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off  -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }}
+
+  build_windows:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest]
+        python-version: [ "3.11" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        git config --system core.longpaths true
+        pip install cm4mlops
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+    - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} natively
+      run: |
+        cm run script --tags=run-abtf,inference,_poc-demo --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off --adr.raw-dataset-mlcommons-cognata.tags=_gdrive -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }}
+
+
+

docs/run/index.md

@@ -2,9 +2,20 @@
 
 ## Install Dependencies
 
-MLCommons CM Automation framework is used to run the POC reference. CM needs to be installed on the computer running the benchmark and CM requires Python 3.7+. Please follow the instructions to install CM as described in the link below:
+MLCommons CM Automation framework is used to run the POC reference. CM requires Python 3.7+, git, python3-pip and python3-venv. If these dependencies are present you can do
 
-* [CM Installation Page](https://docs.mlcommons.org/ck/install)
+Activate a venv for CM (Not mandatory but recommended)
+```
+python3 -m venv cm
+source cm/bin/activate
+```
+
+Install CM and pull the needed repositories
+```
+pip install cm4abtf
+```
+
+More installation details can be found at [CM Installation Page](https://docs.mlcommons.org/ck/install)
 
 Using an Ubuntu example, run `cm` in the terminal and if CM successfully installed, expect the following output:
 
@@ -15,30 +26,34 @@ Using an Ubuntu example, run `cm` in the terminal and if CM successfully install
 
 Note: the `(cm)` indicates that Python `venv` is active.
 
+Now, the `cm` cli commands will be used to run the POC reference.
+
 The computer running the POC reference needs Docker. Follow instructions described in the link below:
 
 * [Install Docker Engine](https://docs.docker.com/engine/install/)
 
-Note: if you're running Ubuntu, CM automatically installs Docker when running the benchmark.
-
-## Download Repositories
+Note: if you're running Ubuntu, CM automatically installs Docker when running the benchmark. We have tested the run on Ubuntu, RHEL, macOS and Windows
 
-Start by using CM to pull the necessary repositories.
-
-```bash
-cm pull repo gateoverflow@cm4mlops
-cm pull repo mlcommons@cm4abtf
-```
-
-Now, the `cm` cli commands will be used to run the POC reference.
 
 ## Start the Benchmark
 
 By running the script below, you are downloading the POC container and dataset, then launching the benchmark. All in one command!
 
 ```
-cm run script --tags=run-abtf,_poc-demo --quiet --docker --gh_token=<GH_TOKEN>  --docker_cache=no
+cm run script --tags=run-abtf,_poc-demo \
+  --quiet \
+  --docker \
+  --gh_token=<GH_TOKEN> \
+  --docker_cache=no
 ```
+!!! tip
+    * Use `--rerun` to force overwrite the previously generated results
+    * Use `--env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off` to use the prebuilt MLPerf Loadgen binary and not do a source compilation
+    * Use `--docker_os=[rhel|arch|ubuntu]` to change the docker OS
+    * Use `--docker_os_version=[8|9]` for `RHEL`, `[24.04,22.04,20.04]` for `ubuntu` and `[latest]` for `arch`  
+    * Use `--docker_base_image=[IMAGE_NAME]` to override the default base image for docker
+    * Skip `--docker` to do the run on the host machine without using a docker
+    * Github actions for this run can be seen [here](https://github.com/mlcommons/cm4abtf/actions/workflows/test-mlperf-inference-abtf-poc.yml)
 
 Depending on the computer used and internet connection, this can take a few minutes.
 
@@ -66,5 +81,6 @@ If you face any issues, please don't hesitate to reach out!
 * Join the [MLCommons Automotive Discord Server](https://discord.gg/jBxH9GvftZ)
 
 ## Contributors
-
-TBD
+* The POC reference model is trained and developed by Radoyeh Shojaei
+* MLPerf Loadgen integration for the POC reference is done by Grigori Fursin and Radoyeh Shojaei
+* CM workflow for the POC reference is done by Arjun Suresh and Grigori Fursin

script/app-abtf-inference/_cm.yaml

@@ -94,6 +94,8 @@ deps:
 
 docker:
   cm_repo: gateoverflow@cm4mlops
+  use_host_group_id: True
+  use_host_user_id: True
   real_run: false
   interactive: True
   cm_repos: 'cm pull repo mlcommons@cm4abtf --checkout=poc'

script/demo-ml-model-abtf-cognata-pytorch-loadgen/_cm.yaml

@@ -19,6 +19,7 @@ tags:
 default_env:
   CM_MLPERF_LOADGEN_MODE: accuracy
   CM_MLPERF_LOADGEN_SCENARIO: Offline
+  CM_MLPERF_LOADGEN_BUILD_FROM_SRC: 'on'
   CM_OUTPUT_FOLDER_NAME: test_results
   CM_MLPERF_RUN_STYLE: test
   CM_TEST_QUERY_COUNT: '10'
@@ -242,16 +243,6 @@ deps:
         - ray
 
 
-  ## Transformers
-  - tags: get,generic-python-lib,_transformers
-    names:
-    - ml-engine-transformers
-    enable_if_env:
-      CM_MODEL:
-      - bert-99
-      - bert-99.9
-      - gptj-99
-      - gptj-99.9
 
   ## Tensorflow
   - tags: get,generic-python-lib,_tensorflow
@@ -274,10 +265,22 @@ deps:
 
 
   # Install MLPerf loadgen
+  - tags: get,generic-python-lib,_package.mlcommons-loadgen
+    enable_if_env:
+      CM_MLPERF_LOADGEN_BUILD_FROM_SRC:
+        - "off"
+    names:
+    - loadgen
+    - mlperf-inference-loadgen
+
   - tags: get,loadgen
+    enable_if_any_env:
+      CM_MLPERF_LOADGEN_BUILD_FROM_SRC:
+        - "on"
     names:
     - loadgen
     - mlperf-inference-loadgen
+    - mlperf-inference-loadgen-from-src
 
 
 #
@@ -402,6 +405,8 @@ variations:
     deps:
       - tags: get,generic-python-lib,_opencv-python
       - tags: get,generic-python-lib,_numpy
+        version_max: "1.26.4"
+        version_max_usable: "1.26.4"
       - tags: get,generic-python-lib,_pycocotools
       - tags: get,generic-python-lib,_package.torchmetrics
       - tags: get,generic-python-lib,_package.faster-coco-eval

script/demo-ml-model-abtf-cognata-pytorch-loadgen/customize.py

@@ -201,7 +201,7 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio
         path_to_model = env.get('CM_MLPERF_CUSTOM_MODEL_PATH', env.get('CM_ML_MODEL_FILE_WITH_PATH', env.get('CM_ML_MODEL_CODE_WITH_PATH')))
         env['MODEL_FILE'] = path_to_model
 
-        cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " python/main.py --profile "+env['CM_MODEL']+"-"+env['CM_MLPERF_BACKEND'] + \
+        cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " " + os.path.join(run_dir, "python", "main.py") + " --profile "+env['CM_MODEL']+"-"+env['CM_MLPERF_BACKEND'] + \
           " --model=" + q + path_to_model + q + \
           " --dataset=" + env["CM_MLPERF_VISION_DATASET_OPTION"] + \
           " --dataset-path=" + q + cognata_dataset_path + q + \

script/run-abtf-inference-demo/_cm.yaml

@@ -43,6 +43,7 @@ input_mapping:
   save_console_log: CM_SAVE_CONSOLE_LOG
   execution_mode: CM_MLPERF_RUN_STYLE
   find_performance: CM_MLPERF_FIND_PERFORMANCE_MODE
+  gh_token: CM_GH_TOKEN
   gpu_name: CM_NVIDIA_GPU_NAME
   hw_name: CM_HW_NAME
   hw_notes_extra: CM_MLPERF_SUT_SW_NOTES_EXTRA