From 5656d4e3f616749fd53babb157b255a849afb283 Mon Sep 17 00:00:00 2001 From: "Jose M. Prieto" Date: Thu, 13 Jul 2023 20:35:24 +0200 Subject: [PATCH 1/4] Additional custom .pyenvrc You can now optionally add a custom .pyenvrc shell script in addition to the one delivered by the role. This custom .pyenvrc will be copied to the same path as .pyenvrc as .pyenvrc.custom. It will be sourced by .pyenvrc delivered by the role if exists. This will become one way you can customize the python build environment. This way you can set an environment not only as pyenv plugin python-build supports according to: https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#special-environment-variables But also other environment variables supported by the python source code compilation itself. This becomes very handy to really take full control over the python build customization. Signed-off-by: Jose M. Prieto --- tasks/install.yml | 8 ++++++++ templates/.pyenvrc.j2 | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tasks/install.yml b/tasks/install.yml index 7b22f12..2363074 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -29,6 +29,14 @@ owner: "{{ pyenv_owner }}" mode: "0644" +- name: Install custom .pyenvrc + ansible.builtin.copy: + src: "{{ pyenv_custom_pyenvrc_file }}" + dest: "{{ pyenvrc_path }}/.pyenvrc.custom" + owner: "{{ pyenv_owner }}" + mode: "0644" + when: pyenv_custom_pyenvrc_file is defined + - name: Use legacy variable pyenv_setting_path if available ansible.builtin.set_fact: pyenv_shellrc_file: "{{ pyenv_setting_path }}" diff --git a/templates/.pyenvrc.j2 b/templates/.pyenvrc.j2 index 06fa0d5..93a546f 100644 --- a/templates/.pyenvrc.j2 +++ b/templates/.pyenvrc.j2 @@ -2,5 +2,6 @@ # ------------------------------------- export PYENV_ROOT="{{ pyenv_path }}" export PATH="$PYENV_ROOT/bin:$PATH" +[ -f "{{ pyenvrc_path }}/.pyenvrc.custom" ] && source "{{ pyenvrc_path }}/.pyenvrc.custom" eval "$(pyenv init --path)" -eval "$(pyenv init - {{ pyenv_init_options}})" +eval "$(pyenv init - {{ pyenv_init_options }})" From a46ce79ecc92e2f15d54b393da18b8bf3034c355 Mon Sep 17 00:00:00 2001 From: "Jose M. Prieto" Date: Thu, 13 Jul 2023 20:52:27 +0200 Subject: [PATCH 2/4] Additional variables to customize python build task This commit adds the following role variables that can be defined by users to alter how pyenv plugin python-build compiles python source code when asked for install: - pyenv_tmpdir => TMPDIR - pyenv_python_build_build_path => PYTHON_BUILD_BUILD_PATH - pyenv_python_build_cache_path => PYTHON_BUILD_CACHE_PATH - pyenv_python_build_mirror_url => PYTHON_BUILD_MIRROR_URL - pyenv_python_build_mirror_url_skip_checksum => PYTHON_BUILD_MIRROR_URL_SKIP_CHECKSUM - pyenv_python_build_skip_mirror => PYTHON_BUILD_SKIP_MIRROR - pyenv_python_build_skip_homebrew => PYTHON_BUILD_SKIP_HOMEBREW - pyenv_python_build_root => PYTHON_BUILD_ROOT - pyenv_python_build_definitions => PYTHON_BUILD_DEFINITIONS - pyenv_python_configure_opts => PYTHON_CONFIGURE_OPTS - pyenv_python_cflags => PYTHON_CFLAGS - pyenv_python_make_opts => PYTHON_MAKE_OPTS - pyenv_python_make_install_opts => PYTHON_MAKE_INSTALL_OPTS - pyenv_configure_opts => CONFIGURE_OPTS - pyenv_cc => CC - pyenv_make => MAKE - pyenv_make_opts => MAKE_OPTS - pyenv_make_install_opts => MAKE_INSTALL_OPTS Each of those role variables corresponds to an environment variable as described by python-build plugin in: https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#special-environment-variables In addition the following role variable is also supported to support all options while building python for maximum performance: - pyenv_profile_task => PROFILE_TASK All those variables will be another way to customize the python build environment. Signed-off-by: Jose M. Prieto --- tasks/install.yml | 2 -- templates/.pyenvrc.j2 | 57 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/tasks/install.yml b/tasks/install.yml index 2363074..8649d1a 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -80,8 +80,6 @@ - name: "Install Python interpreters {{ pyenv_python_versions }}" ansible.builtin.shell: ". {{ pyenvrc_path }}/.pyenvrc && pyenv install {{ item }}" - environment: - PYTHON_CONFIGURE_OPTS: "{{ pyenv_python_configure_opts }}" args: creates: "{{ pyenv_path }}/versions/{{ item }}/bin/python" with_items: "{{ pyenv_python_versions }}" diff --git a/templates/.pyenvrc.j2 b/templates/.pyenvrc.j2 index 93a546f..2a5ad3c 100644 --- a/templates/.pyenvrc.j2 +++ b/templates/.pyenvrc.j2 @@ -2,6 +2,63 @@ # ------------------------------------- export PYENV_ROOT="{{ pyenv_path }}" export PATH="$PYENV_ROOT/bin:$PATH" +{% if pyenv_tmpdir is defined %} +export TMPDIR="{{ pyenv_tmpdir }}" +{% endif %} +{% if pyenv_python_build_build_path is defined %} +export PYTHON_BUILD_BUILD_PATH="{{ pyenv_python_build_build_path }}" +{% endif %} +{% if pyenv_python_build_cache_path is defined %} +export PYTHON_BUILD_CACHE_PATH="{{ pyenv_python_build_cache_path }}" +{% endif %} +{% if pyenv_python_build_mirror_url is defined %} +export PYTHON_BUILD_MIRROR_URL="{{ pyenv_python_build_mirror_url }}" +{% endif %} +{% if pyenv_python_build_mirror_url_skip_checksum is defined %} +export PYTHON_BUILD_MIRROR_URL_SKIP_CHECKSUM="{{ pyenv_python_build_mirror_url_skip_checksum }}" +{% endif %} +{% if pyenv_python_build_skip_mirror is defined %} +export PYTHON_BUILD_SKIP_MIRROR="{{ pyenv_python_build_skip_mirror }}" +{% endif %} +{% if pyenv_python_build_skip_homebrew is defined %} +export PYTHON_BUILD_SKIP_HOMEBREW="{{ pyenv_python_build_skip_homebrew }}" +{% endif %} +{% if pyenv_python_build_root is defined %} +export PYTHON_BUILD_ROOT="{{ pyenv_python_build_root }}" +{% endif %} +{% if pyenv_python_build_definitions is defined %} +export PYTHON_BUILD_DEFINITIONS="{{ pyenv_python_build_definitions }}" +{% endif %} +{% if pyenv_python_configure_opts is defined %} +export PYTHON_CONFIGURE_OPTS="{{ pyenv_python_configure_opts }}" +{% endif %} +{% if pyenv_python_cflags is defined %} +export PYTHON_CFLAGS="{{ pyenv_python_cflags }}" +{% endif %} +{% if pyenv_python_make_opts is defined %} +export PYTHON_MAKE_OPTS="{{ pyenv_python_make_opts }}" +{% endif %} +{% if pyenv_python_make_install_opts is defined %} +export PYTHON_MAKE_INSTALL_OPTS="{{ pyenv_python_make_install_opts }}" +{% endif %} +{% if pyenv_configure_opts is defined %} +export CONFIGURE_OPTS="{{ pyenv_configure_opts }}" +{% endif %} +{% if pyenv_cc is defined %} +export CC="{{ pyenv_cc }}" +{% endif %} +{% if pyenv_make is defined %} +export MAKE="{{ pyenv_make }}" +{% endif %} +{% if pyenv_make_opts is defined %} +export MAKE_OPTS="{{ pyenv_make_opts }}" +{% endif %} +{% if pyenv_make_install_opts is defined %} +export MAKE_INSTALL_OPTS="{{ pyenv_make_install_opts }}" +{% endif %} +{% if pyenv_profile_task is defined %} +export PROFILE_TASK="{{ pyenv_profile_task }}" +{% endif %} [ -f "{{ pyenvrc_path }}/.pyenvrc.custom" ] && source "{{ pyenvrc_path }}/.pyenvrc.custom" eval "$(pyenv init --path)" eval "$(pyenv init - {{ pyenv_init_options }})" From 082be43a0282e64d8f65f77a07e6d1c505a2c2c3 Mon Sep 17 00:00:00 2001 From: "Jose M. Prieto" Date: Thu, 13 Jul 2023 21:09:36 +0200 Subject: [PATCH 3/4] Additional options in call pyenv install The new role variable pyenv_install_extra_opts can be defined to optinally pass extra options to call to command pyenv install. Signed-off-by: Jose M. Prieto --- defaults/main.yml | 1 + tasks/install.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 4092282..78fac5d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -25,6 +25,7 @@ pyenv_init_options: "{% if pyenv_env != 'user' %}--no-rehash{% endif %}" pyenv_update: false # additional options for the build process, e.g "--enable-shared" +pyenv_install_extra_opts: "" pyenv_python_configure_opts: "" pyenv_uninstall_python_w_wrong_configure_opts: false diff --git a/tasks/install.yml b/tasks/install.yml index 8649d1a..83d2241 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -79,7 +79,7 @@ when: pyenv_uninstall_python_w_wrong_configure_opts - name: "Install Python interpreters {{ pyenv_python_versions }}" - ansible.builtin.shell: ". {{ pyenvrc_path }}/.pyenvrc && pyenv install {{ item }}" + ansible.builtin.shell: ". {{ pyenvrc_path }}/.pyenvrc && pyenv install {{ pyenv_install_extra_opts }} {{ item }}" args: creates: "{{ pyenv_path }}/versions/{{ item }}/bin/python" with_items: "{{ pyenv_python_versions }}" From 0f18d5c470f8df545cdb3afc179e5411f1c6241a Mon Sep 17 00:00:00 2001 From: "Jose M. Prieto" Date: Fri, 14 Jul 2023 09:57:45 +0200 Subject: [PATCH 4/4] Update README.md with new role variables New role variables, default values of them and a brief explanation has been included with this commit. Signed-off-by: Jose M. Prieto --- README.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 275badd..606080a 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,28 @@ Here is the list of all variables and their default values: - `pyenv_update_git_install: true` (get latest pyenv from git) - `pyenv_enable_autocompletion: false` - `pyenv_enable_virtualenvs: true` -- `pyenv_shellrc_file "{% if pyenv_env == 'user' %}~/.bashrc{% else %}/etc/profile.d/pyenv.sh{% endif %}"` +- `pyenv_shellrc_file: "{% if pyenv_env == 'user' %}~/.bashrc{% else %}/etc/profile.d/pyenv.sh{% endif %}"` +- `pyenv_tmpdir: (must be explicitly defined)` - env variable `TMPDIR` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_build_path: (must be explicitly defined)` - env variable `PYTHON_BUILD_BUILD_PATH` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_cache_path: (must be explicitly defined)` - env variable `PYTHON_BUILD_CACHE_PATH` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_mirror_url: (must be explicitly defined)` - env variable `PYTHON_BUILD_MIRROR_URL` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_mirror_url_skip_checksum: (must be explicitly defined)` - env variable `PYTHON_BUILD_MIRROR_URL_SKIP_CHECKSUM` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_skip_mirror: (must be explicitly defined)` - env variable `PYTHON_BUILD_SKIP_MIRROR` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_skip_homebrew: (must be explicitly defined)` - env variable `PYTHON_BUILD_SKIP_HOMEBREW` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_root: (must be explicitly defined)` - env variable `PYTHON_BUILD_ROOT` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_build_definitions: (must be explicitly defined)` - env variable `PYTHON_BUILD_DEFINITIONS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_configure_opts: (must be explicitly defined)` - env variable `PYTHON_CONFIGURE_OPTS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_cflags: (must be explicitly defined)` - env variable `PYTHON_CFLAGS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_make_opts: (must be explicitly defined)` - env variable `PYTHON_MAKE_OPTS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_python_make_install_opts: (must be explicitly defined)` - env variable `PYTHON_MAKE_INSTALL_OPTS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_configure_opts: (must be explicitly defined)` - env variable `CONFIGURE_OPTS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_cc: (must be explicitly defined)` - env variable `CC` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_make: (must be explicitly defined)` - env variable `MAKE` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_make_opts: (must be explicitly defined)` - env variable `MAKE_OPTS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_make_install_opts: (must be explicitly defined)` - env variable `MAKE_INSTALL_OPTS` used by [python-build][python-build] as described in [Special Environment Variables][special-env-vars]. +- `pyenv_profile_task: (must be explicitly defined)` - env variable `PROFILE_TASK` to customize the task used for profile guided optimization as described in [building Python for maximum performance][max-performance]. See also [here](https://docs.python.org/3/using/configure.html#cmdoption-enable-optimizations). +- `pyenv_custom_pyenvrc_file: (must be explicitly defined)` - path to a custom `.pyenvrc` shell file that will be sourced from `{{ pyenvrc_path }}/.pyenvrc`. It allows you to freely customize the environment to be used during `pyenv` execution. If defined, this file will be copied as `{{ pyenvrc_path }}/.pyenvrc.custom`. +- `pyenv_install_extra_opts: ("" -no extra options added-)` - check output of `pyenv install --help` for available additional options. ## Dependencies @@ -65,6 +86,10 @@ None. py_version: 3.11.4 - venv_name: latest_v310 py_version: 3.10.12 + pyenv_make_opts: "-j4" + pyenv_python_configure_opts: "--enable-optimizations --with-lto --with-ensurepip=upgrade" + pyenv_python_cflags: "-march=native -mtune=native" + pyenv_profile_task: "-m test.regrtest --pgo -j0" ``` ## License @@ -80,3 +105,6 @@ _Ansible role Pyenv_ is free and open source software. [mit]: https://opensource.org/licenses/MIT [pyenv]: https://github.com/yyuu/pyenv [staticdev]: https://github.com/staticdev +[python-build]: https://github.com/pyenv/pyenv/tree/master/plugins/python-build "python-build plugin" +[special-env-vars]: https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#special-environment-variables "Special environment variables" +[max-performance]: https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#building-for-maximum-performance "Building for maximum performance"