azure-pipelines.yml

# We use the "Secure Development Tools" which needs to be installed in any DevOps organization that use 
# this YAML file. The free public Azure Pipelines for OSS includes these tools by default already.

# TODO: add ESLint once it's added to the "Secure Development Tools". TSLint is depricated.

variables:
- name: ubuntu_image
  value: ubuntu-latest
- name: mac_image
  value: macOS-latest
- name: windows_image
  value: windows-latest

jobs:
- job: 'Build_ebm_native'
  strategy:
    matrix:
      Linux:
        image.name: '${{ variables.ubuntu_image }}'
      Mac:
        image.name: '${{ variables.mac_image }}'
      Windows:
        image.name: '${{ variables.windows_image }}'
    maxParallel: 3
  pool:
    vmImage: '$(image.name)'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: PythonScript@0
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      scriptSource: 'inline'
      script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
  - script: |
      # CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
      set PATH=%PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\
      set CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
      # the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
      cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
      .\build.bat
    condition: startsWith(variables['image.name'], 'windows')
    displayName: 'Building ebm_native code (Windows)'
  - script: |
      sudo apt -y install nvidia-cuda-toolkit
      nvcc --version
      /bin/sh ./build.sh -asm
    condition: startsWith(variables['image.name'], 'ubuntu')
    displayName: 'Building ebm_native code (Linux)'
  - script: |
      /bin/sh ./build.sh -asm
    condition: startsWith(variables['image.name'], 'macOS')
    displayName: 'Building ebm_native code (Mac)'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: staging
      artifactName: ebm_native-$(image.name)
    displayName: 'Publish ebm_native library'
  - task: PublishBuildArtifacts@1
    condition: or(startsWith(variables['image.name'], 'ubuntu'), startsWith(variables['image.name'], 'macOS'))
    continueOnError: true
    inputs:
      pathtoPublish: tmp/staging_asm_release_64
      artifactName: asm-$(image.name)
    displayName: 'Publish assembly'

- job: 'Test_ebm_native'
  dependsOn: 'Build_ebm_native'
  strategy:
    matrix:
      Linux:
        image.name: '${{ variables.ubuntu_image }}'
      Mac:
        image.name: '${{ variables.mac_image }}'
      Windows:
        image.name: '${{ variables.windows_image }}'
    maxParallel: 3
  pool:
    vmImage: '$(image.name)'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'specific'
    displayName: 'Download build artifacts'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/lib_ebm_native_*'
      targetFolder: 'staging'
      flattenFolders: true
    displayName: 'Move binary to staging directory'
  - task: PythonScript@0
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      scriptSource: 'inline'
      script: from urllib.request import urlretrieve; urlretrieve('https://developer.download.nvidia.com/compute/cuda/11.2.2/network_installers/cuda_11.2.2_win10_network.exe', 'cuda_11.2.2_win10_network.exe')
  - script: |
      /bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -existing_debug_64 -existing_release_64
    condition: startsWith(variables['image.name'], 'macOS')
    displayName: 'Testing ebm_native code (Mac)'
  - script: |
      /bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -debug_32 -release_32 -existing_debug_64 -existing_release_64 -no_valgrind
    condition: and(startsWith(variables['image.name'], 'ubuntu'), ne(variables['Build.Reason'], 'Schedule'))
    displayName: 'Testing ebm_native code (Linux) CI'
  - script: |
      /bin/sh ./shared/ebm_native/ebm_native_test/ebm_native_test.sh -debug_32 -release_32 -existing_debug_64 -existing_release_64
    condition: and(startsWith(variables['image.name'], 'ubuntu'), eq(variables['Build.Reason'], 'Schedule'))
    displayName: 'Testing ebm_native code (Linux) Schedule'
  - script: |
      # CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
      set PATH=%PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\
      set CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
      # the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
      cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
      .\shared\ebm_native\ebm_native_test\ebm_native_test.bat -pipeline
    condition: and(startsWith(variables['image.name'], 'windows'), ne(variables['Build.Reason'], 'Schedule'))
    displayName: 'Testing ebm_native code (Windows) CI'
  - script: |
      # CUDA installation https://docs.nvidia.com/cuda/pdf/CUDA_Installation_Guide_Windows.pdf
      set PATH=%PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\
      set CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2
      # the list of NVIDIA CUDA install options is at: https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html
      cuda_11.2.2_win10_network.exe -s nvcc_11.2 visual_studio_integration_11.2 cudart_11.2
      .\shared\ebm_native\ebm_native_test\ebm_native_test.bat -pipeline -analysis
    condition: and(startsWith(variables['image.name'], 'windows'), eq(variables['Build.Reason'], 'Schedule'))
    displayName: 'Testing ebm_native code (Windows) Schedule'
  - task: ComponentGovernanceComponentDetection@0
    displayName: 'Scan dependencies for incompatible licenses and security issues'
  - task: CredScan@2
    inputs:
      toolMajorVersion: 'V2'
    displayName: 'Scan codebase for leaked secrets'
    condition: startsWith(variables['image.name'], 'windows')
#  - task: BinSkim@3
#    displayName: 'Scan DLLs for compiler/linker security improvements'
#    condition: startsWith(variables['image.name'], 'windows')
#    inputs:
#      InputType: Basic
#      AnalyzeTarget: 'staging\*.dll'
#      AnalyzeSymPath: 'staging'
#      AnalyzeVerbose: true
#  - task: AntiMalware@3
#    displayName: 'AntiMalware scan'
  - task: SdtReport@1
    displayName: 'Generate security report'
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      TsvFile: false
      AllTools: false
      BinSkim: false
      CredScan: true
      MSRD: false
      RoslynAnalyzers: false
      TSLint: false
      ToolLogsNotFoundAction: 'Standard'
  - task: PublishSecurityAnalysisLogs@2
    displayName: 'Publish security report'
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      ArtifactName: 'CodeAnalysisLogs'
      ArtifactType: 'Container'
      AllTools: true
      ToolLogsNotFoundAction: 'Standard'
  - task: PostAnalysis@1
    displayName: 'Stop on security errors'
    condition: startsWith(variables['image.name'], 'windows')
    inputs:
      AllTools: false
      BinSkim: false
      CredScan: true
      RoslynAnalyzers: false
      TSLint: false
      ToolLogsNotFoundAction: 'Standard'

- job: 'Build_JS_Inline'
  pool:
    vmImage: '${{ variables.ubuntu_image }}'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '3.7'
      architecture: 'x64'
  - script: python build-js-bundles.py devops
    workingDirectory: 'python/scripts'
    displayName: 'Build JS bundle'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: 'python/interpret-core/interpret/lib'
      contents: 'interpret-inline.js'
      targetFolder: '$(Build.ArtifactStagingDirectory)'
    displayName: 'Move assets to staging'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: interpret-inline-bundle
    displayName: 'Publish interpret-inline.js library'

- job: 'Build_Package'
  dependsOn: ['Build_ebm_native', 'Build_JS_Inline']
  pool:
    vmImage: '${{ variables.ubuntu_image }}'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'specific'
    displayName: 'Download build artifacts'
  - task: UsePythonVersion@0
    condition: succeeded()
    inputs:
      versionSpec: '3.7'
      architecture: 'x64'
  - script: python -m pip install --upgrade pip setuptools wheel
    condition: succeeded()
    displayName: 'Install tools'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/lib_ebm_native_*_x64*'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move binary to Python layer'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/interpret-inline.js'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move interpret-inline.js to Python layer'
  - script: python setup.py bdist_wheel -d ../../staging
    condition: succeeded()
    workingDirectory: 'python/interpret-core'
    displayName: 'Build wheel (interpret-core)'
  - script: python setup.py bdist_wheel -d ../../staging
    condition: succeeded()
    workingDirectory: 'python/interpret'
    displayName: 'Build wheel (interpret)'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: staging
      contents: '*.whl'
      targetFolder: '$(Build.ArtifactStagingDirectory)/wheel'
    displayName: 'Move wheel for Build Artifact'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/wheel'
      artifactName: 'wheel'
    displayName: 'Publish wheel as Build Artifact'

- job: 'Test'
  dependsOn: 'Build_Package'
  # NOTE: Python 3.9 not supported yet in Azure DevOps.
  strategy:
    matrix:
      LinuxPython37:
        python.version: '3.7'
        image.name: '${{ variables.ubuntu_image }}'
      LinuxPython38:
        python.version: '3.8'
        image.name: '${{ variables.ubuntu_image }}'
      LinuxPython39:
        python.version: '3.9'
        image.name: '${{ variables.ubuntu_image }}'
      WindowsPython37:
        python.version: '3.7'
        image.name: '${{ variables.windows_image }}'
      WindowsPython38:
        python.version: '3.8'
        image.name: '${{ variables.windows_image }}'
      WindowsPython39:
        python.version: '3.9'
        image.name: '${{ variables.windows_image }}'
      MacPython37:
        python.version: '3.7'
        image.name: '${{ variables.mac_image }}'
      MacPython38:
        python.version: '3.8'
        image.name: '${{ variables.mac_image }}'
      MacPython39:
        python.version: '3.9'
        image.name: '${{ variables.mac_image }}'
    maxParallel: 9
  pool:
    vmImage: '$(image.name)'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'specific'
    displayName: 'Download build artifacts'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/lib_ebm_native_*'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move binary to Python layer'
  - task: CopyFiles@2
    condition: succeeded()
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      contents: '**/interpret-inline.js'
      targetFolder: 'python/interpret-core/interpret/lib'
      flattenFolders: true
    displayName: 'Move Interpret inline to Python layer'
  - task: UsePythonVersion@0
    condition: succeeded()
    inputs:
      versionSpec: '$(python.version)'
      architecture: 'x64'
  - script: python -m pip install --upgrade pip setuptools wheel
    condition: succeeded()
    displayName: 'Install tools'
  - script: python -m pip install numpy scipy
    condition: succeeded()
    workingDirectory: python/interpret-core
    displayName: 'Install numpy/scipy first for mis-specified pip packages.'
  - script: |
      mkdir -p ~/.matplotlib
      echo "backend: TkAgg" >> ~/.matplotlib/matplotlibrc
    condition: startsWith(variables['image.name'], 'macOS')
    displayName: 'Matplotlib patch for mac.'
  # - script: python -m pip install -r dev-requirements.txt
  - script: python -m pip install -e .[required,debug,notebook,plotly,lime,sensitivity,shap,ebm,linear,decisiontree,treeinterpreter,dash,skoperules,testing]
    condition: succeeded()
    workingDirectory: python/interpret-core
    displayName: 'Install requirements (Linux/Mac/Windows)'
  - script: |
      set PATH=%PATH%;%GeckoWebDriver%
      python -m pytest -vv -n auto --runslow --runselenium --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
    workingDirectory: python/interpret-core
    condition: and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master')))
    displayName: 'Run pytest (Windows scheduled / master)'
  - script: |
      python -m pytest -vv -n auto --runslow --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
    workingDirectory: python/interpret-core
    condition: not(and(startsWith(variables['image.name'], 'windows'), or(eq(variables['Build.Reason'], 'Schedule'), in(variables['Build.SourceBranch'], 'refs/heads/master'))))
    displayName: 'Run pytest (Rest)'
  - task: PublishTestResults@2
    condition: succeededOrFailed()
    inputs:
      testResultsFiles: '**/test-*.xml'
      testRunTitle: 'Publish test results for Python $(python.version) at $(image.name)'
    displayName: 'Publish test results'
  - task: PublishCodeCoverageResults@1
    inputs:
      codeCoverageTool: Cobertura
      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
      # reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'
    condition: startsWith(variables['image.name'], 'windows')
    displayName: 'Publish test coverage results'

- job: 'Publish_Package'
  # dependsOn: ['Test_ebm_native', 'Test']
  dependsOn: ['Test']
  pool:
    vmImage: '${{ variables.ubuntu_image }}'
  variables:
    runCodesignValidationInjection: false
  steps:
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'single'
      artifactName: 'wheel'
  - task: DownloadBuildArtifacts@0
    inputs:
      buildType: 'current'
      downloadType: 'single'
      artifactName: 'interpret-inline-bundle'
  - task: PublishPipelineArtifact@0
    inputs:
      sourceFolder: '$(System.ArtifactsDirectory)'
      artifactName: 'drop'

schedules:
- cron: "0 12 * * *"
  displayName: Daily midnight build
  branches:
    include:
    - develop
  always: true