From ad69f921c91059ac94b60c5319e225060ae9287b Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Fri, 1 Nov 2024 11:24:46 -0700 Subject: [PATCH] Test Windows Wheels and Fix symbol address resolution on Windows (#543) This adds CI for testing out windows wheels against various versions of python. This also applies a fix to use RVA for symbol address resolution on windows (as identified by @akhramov in #672) --- .github/workflows/build.yml | 30 +++++++++++++++++++++++++++++- ci/update_python_test_versions.py | 22 ++++++++++++++++++---- src/binary_parser.rs | 6 ++---- src/python_process_info.rs | 18 ++++++++++++++++-- tests/integration_test.py | 10 +++++++--- 5 files changed, 72 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85968bb4..5e8231cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -215,27 +215,55 @@ jobs: 3.13.0, ] # TODO: also test windows - os: [ubuntu-20.04, macos-13] + os: [ubuntu-20.04, macos-13, windows-latest] # some versions of python can't be tested on GHA with osx because of SIP: exclude: + - os: windows-latest + python-version: 3.6.15 + - os: windows-latest + python-version: 3.7.17 + - os: windows-latest + python-version: 3.8.18 + - os: windows-latest + python-version: 3.9.20 + - os: windows-latest + python-version: 3.10.15 - os: macos-13 python-version: 3.11.10 + - os: windows-latest + python-version: 3.11.10 - os: macos-13 python-version: 3.12.0 + - os: windows-latest + python-version: 3.12.0 - os: macos-13 python-version: 3.12.1 + - os: windows-latest + python-version: 3.12.1 - os: macos-13 python-version: 3.12.2 + - os: windows-latest + python-version: 3.12.2 - os: macos-13 python-version: 3.12.3 + - os: windows-latest + python-version: 3.12.3 - os: macos-13 python-version: 3.12.4 + - os: windows-latest + python-version: 3.12.4 - os: macos-13 python-version: 3.12.5 + - os: windows-latest + python-version: 3.12.5 - os: macos-13 python-version: 3.12.6 + - os: windows-latest + python-version: 3.12.6 - os: macos-13 python-version: 3.12.7 + - os: windows-latest + python-version: 3.12.7 steps: - uses: actions/checkout@v2 diff --git a/ci/update_python_test_versions.py b/ci/update_python_test_versions.py index d8564648..daf2394c 100644 --- a/ci/update_python_test_versions.py +++ b/ci/update_python_test_versions.py @@ -14,8 +14,14 @@ def parse_version(v): def get_github_python_versions(): versions_json = requests.get(_VERSIONS_URL).json() - raw_versions = [v["version"] for v in versions_json] + # windows platform support isn't great for older versions of python + # get a map of version: platform/arch so we can exclude here + platforms = {} + for v in versions_json: + platforms[v["version"]] = set((f["platform"], f["arch"]) for f in v["files"]) + + raw_versions = [v["version"] for v in versions_json] minor_versions = defaultdict(list) for version_str in raw_versions: @@ -46,11 +52,13 @@ def get_github_python_versions(): versions.extend(f"{major}.{minor}.{patch}" for patch in patches) - return versions + return versions, platforms def update_python_test_versions(): - versions = sorted(get_github_python_versions(), key=parse_version) + versions, platforms = get_github_python_versions() + versions = sorted(versions, key=parse_version) + build_yml_path = ( pathlib.Path(__file__).parent.parent / ".github" / "workflows" / "build.yml" ) @@ -82,9 +90,15 @@ def update_python_test_versions(): # since it currently fails in GHA on SIP errors exclusions = [] for v in versions: - if v.startswith("3.11.10") or v.startswith("3.12"): + # if we don't have a python version for osx/windows skip + if ("darwin", "x64") not in platforms[v] or v.startswith("3.12"): exclusions.append(" - os: macos-13\n") exclusions.append(f" python-version: {v}\n") + + if ("win32", "x64") not in platforms[v] or v.startswith("3.12"): + exclusions.append(" - os: windows-latest\n") + exclusions.append(f" python-version: {v}\n") + first_exclude_line = lines.index(" exclude:\n", first_line) last_exclude_line = lines.index("\n", first_exclude_line) lines = lines[: first_exclude_line + 1] + exclusions + lines[last_exclude_line:] diff --git a/src/binary_parser.rs b/src/binary_parser.rs index e9bd52a4..526401d6 100644 --- a/src/binary_parser.rs +++ b/src/binary_parser.rs @@ -201,10 +201,8 @@ pub fn parse_binary(filename: &Path, addr: u64, size: u64) -> Result { for export in pe.exports { if let Some(name) = export.name { - if let Some(export_offset) = export.offset { - if let Some(addr) = offset.checked_add(export_offset as u64) { - symbols.insert(name.to_string(), addr); - } + if let Some(addr) = offset.checked_add(export.rva as u64) { + symbols.insert(name.to_string(), addr); } } } diff --git a/src/python_process_info.rs b/src/python_process_info.rs index 65befeb1..301ec1cd 100644 --- a/src/python_process_info.rs +++ b/src/python_process_info.rs @@ -75,7 +75,14 @@ impl PythonProcessInfo { let map = maps.iter().find(|m| { if let Some(pathname) = m.filename() { if let Some(pathname) = pathname.to_str() { - return is_python_bin(pathname) && m.is_exec(); + #[cfg(not(windows))] + { + return is_python_bin(pathname) && m.is_exec(); + } + #[cfg(windows)] + { + return is_python_bin(pathname); + } } } false @@ -139,7 +146,14 @@ impl PythonProcessInfo { let libmap = maps.iter().find(|m| { if let Some(pathname) = m.filename() { if let Some(pathname) = pathname.to_str() { - return is_python_lib(pathname) && m.is_exec(); + #[cfg(not(windows))] + { + return is_python_lib(pathname) && m.is_exec(); + } + #[cfg(windows)] + { + return is_python_lib(pathname); + } } } false diff --git a/tests/integration_test.py b/tests/integration_test.py index 7408dd8d..023e098c 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -31,11 +31,15 @@ def _sample_process(self, script_name, options=None, include_profile_name=False) # record option, and setting different flags. To get the profile output # we're using the speedscope format (since we can read that in as json) with tempfile.NamedTemporaryFile() as profile_file: + filename = profile_file.name + if sys.platform.startswith("win"): + filename = "profile.json" + cmdline = [ PYSPY, "record", "-o", - profile_file.name, + filename, "--format", "speedscope", "-d", @@ -43,9 +47,9 @@ def _sample_process(self, script_name, options=None, include_profile_name=False) ] cmdline.extend(options or []) cmdline.extend(["--", sys.executable, script_name]) - env = dict(os.environ, RUST_LOG="debug") + env = dict(os.environ, RUST_LOG="info") subprocess.check_output(cmdline, env=env) - with open(profile_file.name) as f: + with open(filename) as f: profiles = json.load(f) frames = profiles["shared"]["frames"]