Set python to 3.7 in docker container and release action (model-check…

…ing#2879)

An attempt to fix a release build issue due to cbmc-viewer depending on
voluptuous which now depends on python 3.7 for type annotations.

This adds a ppa in the dockerfile and github release action for ubuntu
18.04 and installs python 3.7.


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 and MIT licenses.

---------

Co-authored-by: Remi Delmas <[email protected]>
Co-authored-by: jaisnan <[email protected]>

.github/workflows/release.yml

@@ -68,17 +68,23 @@ jobs:
           df -h
 
       # This is required before checkout because the container does not
-      # have Git installed, so cannot run checkout action. The checkout
-      # action requires Git >=2.18, so use the Git maintainers' PPA.
+      # have Git installed, so cannot run checkout action.
+      # The checkout action requires Git >=2.18 and python 3.7, so use the Git maintainers' PPA.
+      # and the "deadsnakes" PPA, as the default version of python on ubuntu 22.04 is Python 3.10
       - name: Install system dependencies
         run: |
           apt-get update
           apt-get install -y software-properties-common apt-utils
           add-apt-repository ppa:git-core/ppa
+          add-apt-repository ppa:deadsnakes/ppa
           apt-get update
           apt-get install -y \
             build-essential bash-completion curl lsb-release sudo g++ gcc flex \
-            bison make patch git
+            bison make patch git python3.7 python3.7-dev python3.7-distutils
+          update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+          curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+          python3 get-pip.py --force-reinstall
+          rm get-pip.py
 
       - name: Checkout Kani
         uses: actions/checkout@v3

docs/src/install-guide.md

@@ -14,7 +14,7 @@ GitHub CI workflows, see [GitHub CI Action](./install-github-ci.md).
 
 The following must already be installed:
 
-* **Python version 3.6 or newer** and the package installer `pip`.
+* **Python version 3.7 or newer** and the package installer `pip`.
 * Rust 1.58 or newer installed via `rustup`.
 * `ctags` is required for Kani's `--visualize` option to work correctly. [Universal ctags](https://ctags.io/) is recommended.
 

scripts/ci/Dockerfile.bundle-test-ubuntu-18-04

@@ -10,9 +10,19 @@
 FROM ubuntu:18.04
 ENV DEBIAN_FRONTEND=noninteractive \
     DEBCONF_NONINTERACTIVE_SEEN=true
+
 RUN apt-get update && \
-    apt-get install -y python3 python3-pip curl ctags && \
-    curl -sSf https://sh.rustup.rs | sh -s -- -y
+    apt-get install --no-install-recommends -y build-essential software-properties-common && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt install --no-install-recommends -y python3.7 python3.7-dev python3.7-distutils curl ctags
+
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+
+RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3 get-pip.py --force-reinstall && \
+    rm get-pip.py
+
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 WORKDIR /tmp/kani

src/os_hacks.rs

@@ -5,7 +5,6 @@
 //! "flow" of code in setup.rs, this module contains all functions that implement os-specific
 //! workarounds.
 
-use std::ffi::OsString;
 use std::path::Path;
 use std::process::Command;
 
@@ -14,74 +13,6 @@ use os_info::Info;
 
 use crate::cmd::AutoRun;
 
-pub fn should_apply_ubuntu_18_04_python_hack(os: &os_info::Info) -> Result<bool> {
-    if os.os_type() != os_info::Type::Ubuntu {
-        return Ok(false);
-    }
-    // Check both versions: https://github.com/stanislav-tkach/os_info/issues/318
-    if *os.version() != os_info::Version::Semantic(18, 4, 0)
-        && *os.version() != os_info::Version::Custom("18.04".into())
-    {
-        return Ok(false);
-    }
-    // It's not enough to check that we're on Ubuntu 18.04 because the user may have
-    // manually updated to a newer version of Python instead of using what the OS ships.
-    // So check if it looks like the OS-shipped version as best we can.
-    let cmd = Command::new("python3").args(["-m", "pip", "--version"]).output()?;
-    let output = std::str::from_utf8(&cmd.stdout)?;
-    // The problem version looks like:
-    //    'pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)'
-    // So we'll test for version 9.
-    Ok(pip_major_version(output)? == 9)
-}
-
-/// Unit testable parsing function for extracting pip version numbers, from strings that look like:
-///    'pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)'
-fn pip_major_version(output: &str) -> Result<u32> {
-    // We don't want dependencies so parse with stdlib string functions as best we can.
-    let mut words = output.split_whitespace();
-    let _pip = words.next().context("No pip output")?;
-    let version = words.next().context("No pip version")?;
-
-    let mut versions = version.split('.');
-    let major = versions.next().context("No pip major version")?;
-
-    Ok(major.parse()?)
-}
-
-/// See [`crate::setup::setup_python_deps`]
-pub fn setup_python_deps_on_ubuntu_18_04(pyroot: &Path, pkg_versions: &[&str]) -> Result<()> {
-    println!("Applying a workaround for 18.04...");
-    // https://github.com/pypa/pip/issues/3826
-    // Ubuntu 18.04 has a patched-to-be-broken version of pip that just straight-up makes `--target` not work.
-    // Worse still, there is no apparent way to replicate the correct behavior cleanly.
-
-    // This is a really awful hack to simulate getting the same result. I can find no other solution.
-    // Example failed approach: `--system --target pyroot` fails to create a `pyroot/bin` with binaries.
-
-    // Step 1: use `--system --prefix pyroot`. This disables the broken behavior, and creates `bin` but...
-    Command::new("python3")
-        .args(["-m", "pip", "install", "--system", "--prefix"])
-        .arg(pyroot)
-        .args(pkg_versions)
-        .run()?;
-
-    // Step 2: move `pyroot/lib/python3.6/site-packages/*` up to `pyroot`
-    // This seems to successfully replicate the behavior of `--target`
-    // "mv" is not idempotent however so we need to do "cp -r" then delete
-    let mut cp_cmd = OsString::new();
-    cp_cmd.push("cp -r ");
-    cp_cmd.push(pyroot.as_os_str());
-    cp_cmd.push("/lib/python*/site-packages/* ");
-    cp_cmd.push(pyroot.as_os_str());
-    Command::new("bash").arg("-c").arg(cp_cmd).run()?;
-
-    // `lib` is the directory `--prefix` creates that `--target` does not.
-    std::fs::remove_dir_all(pyroot.join("lib"))?;
-
-    Ok(())
-}
-
 /// This is the final step of setup, where we look for OSes that require additional setup steps
 /// beyond the usual ones that we have done already.
 pub fn setup_os_hacks(kani_dir: &Path, os: &Info) -> Result<()> {
@@ -159,25 +90,3 @@ fn setup_nixos_patchelf(kani_dir: &Path) -> Result<()> {
 
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn check_pip_major_version() -> Result<()> {
-        // These read a lot better formatted on one line, so shorten them:
-        use pip_major_version as p;
-        // 18.04 example: (with extra newline to test whitespace handling)
-        assert_eq!(p("pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)\n")?, 9);
-        // a mac
-        assert_eq!(p("pip 21.1.1 from /usr/local/python3.9/site-packages/pip (python 3.9)")?, 21);
-        // 20.04
-        assert_eq!(p("pip 20.0.2 from /usr/lib/python3/dist-packages/pip (python 3.8)")?, 20);
-        // How mangled can we get and still "work"?
-        assert_eq!(p("pip 1")?, 1);
-        assert_eq!(p("p 1")?, 1);
-        assert_eq!(p("\n\n p 1 p")?, 1);
-        Ok(())
-    }
-}

src/setup.rs

@@ -83,7 +83,7 @@ pub fn setup(use_local_bundle: Option<OsString>) -> Result<()> {
 
     setup_rust_toolchain(&kani_dir)?;
 
-    setup_python_deps(&kani_dir, &os)?;
+    setup_python_deps(&kani_dir)?;
 
     os_hacks::setup_os_hacks(&kani_dir, &os)?;
 
@@ -152,18 +152,13 @@ fn setup_rust_toolchain(kani_dir: &Path) -> Result<String> {
 }
 
 /// Install into the pyroot the python dependencies we need
-fn setup_python_deps(kani_dir: &Path, os: &os_info::Info) -> Result<()> {
+fn setup_python_deps(kani_dir: &Path) -> Result<()> {
     println!("[4/5] Installing Kani python dependencies...");
     let pyroot = kani_dir.join("pyroot");
 
     // TODO: this is a repetition of versions from kani/kani-dependencies
     let pkg_versions = &["cbmc-viewer==3.8"];
 
-    if os_hacks::should_apply_ubuntu_18_04_python_hack(os)? {
-        os_hacks::setup_python_deps_on_ubuntu_18_04(&pyroot, pkg_versions)?;
-        return Ok(());
-    }
-
     Command::new("python3")
         .args(["-m", "pip", "install", "--target"])
         .arg(&pyroot)