From 2effe7c929f053b1771c305cc01e8934a680deff Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Mon, 18 Nov 2024 13:44:05 +0100 Subject: [PATCH] Restore VirtualPythonEnv being a generator --- extensions/deployers/virtual_python_env.py | 128 ----------------- extensions/generators/EnvScriptBuilder.py | 37 +++++ .../generators/GitHubActionsBuildEnv.py | 22 +-- extensions/generators/GitHubActionsRunEnv.py | 24 +--- extensions/generators/VirtualPythonEnv.py | 129 ++++++++++++++++++ 5 files changed, 179 insertions(+), 161 deletions(-) delete mode 100644 extensions/deployers/virtual_python_env.py create mode 100644 extensions/generators/EnvScriptBuilder.py create mode 100644 extensions/generators/VirtualPythonEnv.py diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py deleted file mode 100644 index 78d72f5..0000000 --- a/extensions/deployers/virtual_python_env.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import sys -from io import StringIO -from shutil import which -from pathlib import Path - -from conan import ConanFile -from conan.errors import ConanException -from conan.tools.files import copy, save, load -from conan.tools.scm import Version -from conan.tools.env import VirtualRunEnv - - -def populate_pip_requirements(key, pip_requirements, conan_data, actual_os): - if conan_data is not None and key in conan_data: - for system in (system for system in conan_data[key] if system in ("any", actual_os)): - for name, req in conan_data[key][system].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): - pip_requirements[name] = req - - -def populate_full_pip_requirements(conanfile, key, pip_requirements, actual_os): - populate_pip_requirements(key, pip_requirements, conanfile.conan_data, actual_os) - - for name, dep in reversed(conanfile.dependencies.host.items()): - populate_pip_requirements(key, pip_requirements, dep.conan_data, actual_os) - - -def install_pip_requirements(file_suffix, file_content, output_folder, conanfile, venv_vars, py_interp_venv): - if len(file_content) > 0: - pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt') - save(conanfile, pip_file_path, "\n".join(file_content)) - with venv_vars.apply(): - conanfile.run(f"{py_interp_venv} -m pip install -r {pip_file_path}", env="conanrun") - - -def deploy(graph, output_folder, **kwargs): - if graph.root.conanfile.name is None: - conanfile: ConanFile = graph.nodes[1].conanfile - else: - conanfile: ConanFile = graph.root.conanfile - - if output_folder is None: - output_folder = "venv" - else: - output_folder = str(Path(output_folder, "venv")) - - bin_venv_path = "Scripts" if conanfile.settings.os == "Windows" else "bin" - - # Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter - try: - cpython = conanfile.dependencies["cpython"] - py_interp = cpython.conf_info.get("user.cpython:python").replace("\\", "/") - except KeyError: - py_interp = sys.executable - - vr = VirtualRunEnv(conanfile) - env = vr.environment() - sys_vars = env.vars(conanfile, scope="run") - - conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'") - with sys_vars.apply(): - conanfile.run(f"""{py_interp} -m venv --copies {output_folder}""", env="conanrun", scope="run") - - # Make sure there executable is named the same on all three OSes this allows it to be called with `python` - # simplifying GH Actions steps - if conanfile.settings.os != "Windows": - py_interp_venv = Path(output_folder, bin_venv_path, "python") - if not py_interp_venv.exists(): - py_interp_venv.hardlink_to( - Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) - else: - py_interp_venv = Path(output_folder, bin_venv_path, - Path(sys.executable).stem + Path(sys.executable).suffix) - - buffer = StringIO() - outer = '"' if conanfile.settings.os == "Windows" else "'" - inner = "'" if conanfile.settings.os == "Windows" else '"' - with sys_vars.apply(): - conanfile.run( - f"""{py_interp_venv} -c {outer}import sysconfig; print(sysconfig.get_path({inner}purelib{inner})){outer}""", - env="conanrun", - stdout=buffer) - pythonpath = buffer.getvalue().splitlines()[-1] - - env.define_path("VIRTUAL_ENV", output_folder) - env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("PYTHONPATH", pythonpath) - env.unset("PYTHONHOME") - venv_vars = env.vars(graph.root.conanfile, scope="run") - venv_vars.save_script("virtual_python_env") - - # Install some base_packages - with venv_vars.apply(): - conanfile.run(f"""{py_interp_venv} -m pip install wheel setuptools""", env="conanrun") - - if conanfile.settings.os != "Windows": - content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(graph.root.conanfile, - os.path.join( - output_folder, - bin_venv_path, - "activate")) - save(graph.root.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) - - pip_requirements = {} - populate_full_pip_requirements(conanfile, "pip_requirements", pip_requirements, str(conanfile.settings.os)) - - requirements_hashed_txt = [] - requirements_url_txt = [] - for name, req in pip_requirements.items(): - if "url" in req: - requirements_url_txt.append(req['url']) - else: - requirement_txt = [f"{name}=={req['version']}"] - - if "hashes" in req: - for hash_str in req['hashes']: - requirement_txt.append(f"--hash={hash_str}") - - requirements_hashed_txt.append(" ".join(requirement_txt)) - - install_pip_requirements("hashed", requirements_hashed_txt, output_folder, conanfile, venv_vars, py_interp_venv) - install_pip_requirements("url", requirements_url_txt, output_folder, conanfile, venv_vars, py_interp_venv) - - if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool) and conanfile.conan_data is not None and "pip_requirements_dev" in conanfile.conan_data: - install_pip_requirements("dev", conanfile.conan_data["pip_requirements_dev"], output_folder, conanfile, venv_vars, py_interp_venv) diff --git a/extensions/generators/EnvScriptBuilder.py b/extensions/generators/EnvScriptBuilder.py new file mode 100644 index 0000000..1a225ee --- /dev/null +++ b/extensions/generators/EnvScriptBuilder.py @@ -0,0 +1,37 @@ +from conan.tools.files import save + +class EnvScriptBuilder: + def __init__(self): + self._variables = {} + + def set_variable(self, name: str, value: str): + self._variables[name] = value + + def set_environment(self, env): + for name, value in env.items(): + self.set_variable(name, value) + + def save(self, path, conanfile, append_to=None) -> None: + file_path = path + + content = "" + for name, value in self._variables.items(): + set_variable = f'{name}={value}' + + if append_to is not None: + set_variable = f"echo {set_variable} >> {append_to}" + else: + set_variable = f"export {set_variable}" + + content += f"{set_variable}\n" + + if conanfile.settings.get_safe("os") == "Windows": + if conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): + file_path += ".ps1" + else: + file_path += ".bat" + else: + file_path += ".sh" + + conanfile.output.info(f"Saving environment script to {file_path}") + save(conanfile, file_path, content) diff --git a/extensions/generators/GitHubActionsBuildEnv.py b/extensions/generators/GitHubActionsBuildEnv.py index e8b05f8..6beb39a 100644 --- a/extensions/generators/GitHubActionsBuildEnv.py +++ b/extensions/generators/GitHubActionsBuildEnv.py @@ -1,32 +1,22 @@ from pathlib import Path -from jinja2 import Template - from conan import ConanFile from conan.tools.env import VirtualBuildEnv -from conan.tools.files import save + +from EnvScriptBuilder import EnvScriptBuilder class GitHubActionsBuildEnv: def __init__(self, conanfile: ConanFile): self.conanfile: ConanFile = conanfile - self.settings = self.conanfile.settings def generate(self): - template = Template( - """{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") build_env = VirtualBuildEnv(self.conanfile) env = build_env.environment() envvars = env.vars(self.conanfile, scope="build") env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - content = template.render(envvars=envvars, env_prefix=env_prefix) - filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_buildenv")) - if self.conanfile.settings.get_safe("os") == "Windows": - if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): - filepath += ".ps1" - else: - filepath += ".bat" - else: - filepath += ".sh" - save(self.conanfile, filepath, content) + + script_builder = EnvScriptBuilder() + script_builder.set_environment(envvars) + script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV") diff --git a/extensions/generators/GitHubActionsRunEnv.py b/extensions/generators/GitHubActionsRunEnv.py index ff31436..8fa9c28 100644 --- a/extensions/generators/GitHubActionsRunEnv.py +++ b/extensions/generators/GitHubActionsRunEnv.py @@ -1,32 +1,22 @@ from pathlib import Path -from jinja2 import Template - from conan import ConanFile from conan.tools.env import VirtualRunEnv -from conan.tools.files import save + +from EnvScriptBuilder import EnvScriptBuilder class GitHubActionsRunEnv: def __init__(self, conanfile: ConanFile): self.conanfile: ConanFile = conanfile - self.settings = self.conanfile.settings def generate(self): - template = Template( - """{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") - build_env = VirtualRunEnv(self.conanfile) - env = build_env.environment() + run_env = VirtualRunEnv(self.conanfile) + env = run_env.environment() envvars = env.vars(self.conanfile, scope="run") env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - content = template.render(envvars=envvars, env_prefix=env_prefix) filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_runenv")) - if self.conanfile.settings.get_safe("os") == "Windows": - if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): - filepath += ".ps1" - else: - filepath += ".bat" - else: - filepath += ".sh" - save(self.conanfile, filepath, content) + script_builder = EnvScriptBuilder() + script_builder.set_environment(envvars) + script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV") diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py new file mode 100644 index 0000000..60cd661 --- /dev/null +++ b/extensions/generators/VirtualPythonEnv.py @@ -0,0 +1,129 @@ +import os +import sys +from io import StringIO +from shutil import which +from pathlib import Path + +from conan import ConanFile +from conan.errors import ConanException +from conan.tools.files import copy, save, load +from conan.tools.scm import Version +from conan.tools.env import VirtualRunEnv +import subprocess + + +class VirtualPythonEnv: + def __init__(self, conanfile: ConanFile): + self.conanfile: ConanFile = conanfile + + def generate(self) -> None: + output_folder = "venv" + bin_venv_path = "Scripts" if self.conanfile.settings.os == "Windows" else "bin" + + # Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter + try: + cpython = self.conanfile.dependencies["cpython"] + py_interp = cpython.conf_info.get("user.cpython:python").replace("\\", "/") + except KeyError: + py_interp = sys.executable + + run_env = VirtualRunEnv(self.conanfile) + env = run_env.environment() + env_vars = env.vars(self.conanfile, scope="run") + + self.conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'") + with env_vars.apply(): + subprocess.run([py_interp, "-m", "venv", "--copies", output_folder]) + + # Make sure there executable is named the same on all three OSes this allows it to be called with `python` + # simplifying GH Actions steps + if self.conanfile.settings.os != "Windows": + py_interp_venv = Path(output_folder, bin_venv_path, "python") + if not py_interp_venv.exists(): + py_interp_venv.hardlink_to( + Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) + else: + py_interp_venv = Path(output_folder, bin_venv_path, + Path(sys.executable).stem + Path(sys.executable).suffix) + + with env_vars.apply(): + buffer = subprocess.run([py_interp_venv, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], capture_output=True, encoding="utf-8").stdout + pythonpath = buffer.splitlines()[-1] + + env.define_path("VIRTUAL_ENV", output_folder) + env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("PYTHONPATH", pythonpath) + env.unset("PYTHONHOME") + + filepath = str(Path(self.conanfile.generators_folder).joinpath("supercoucou_runenv")) + env_vars.save_script(filepath) + + # Install some base_packages + with env_vars.apply(): + subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"]) + + if self.conanfile.settings.os != "Windows": + content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(self.conanfile, + os.path.join( + output_folder, + bin_venv_path, + "activate")) + save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) + + pip_requirements = {} + self._populate_pip_requirements(self.conanfile, "pip_requirements", pip_requirements, str(self.conanfile.settings.os)) + + requirements_hashed_txt = [] + requirements_url_txt = [] + for name, req in pip_requirements.items(): + if "url" in req: + requirements_url_txt.append(req['url']) + else: + requirement_txt = [f"{name}=={req['version']}"] + + if "hashes" in req: + for hash_str in req['hashes']: + requirement_txt.append(f"--hash={hash_str}") + + requirements_hashed_txt.append(" ".join(requirement_txt)) + + self._install_pip_requirements("hashed", requirements_hashed_txt, output_folder, env_vars, py_interp_venv) + self._install_pip_requirements("url", requirements_url_txt, output_folder, env_vars, py_interp_venv) + + if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default = False, check_type = bool): + pip_requirements_dev = [] + self._populate_pip_requirements_dev(self.conanfile, pip_requirements_dev) + print(pip_requirements_dev) + self._install_pip_requirements("dev", pip_requirements_dev, output_folder, env_vars, py_interp_venv) + + + def _populate_pip_requirements_dev(self, conanfile, pip_requirements_dev, add_dependencies = True): + if hasattr(conanfile, "conan_data") and "pip_requirements_dev" in conanfile.conan_data: + print(conanfile.conan_data["pip_requirements_dev"]) + pip_requirements_dev += conanfile.conan_data["pip_requirements_dev"] + + if add_dependencies: + for name, dep in reversed(self.conanfile.dependencies.host.items()): + self._populate_pip_requirements_dev(dep, pip_requirements_dev, add_dependencies = False) + + + def _populate_pip_requirements(self, conanfile, key, pip_requirements, actual_os, add_dependencies = True): + if hasattr(conanfile, "conan_data") and key in conanfile.conan_data: + for system in (system for system in conanfile.conan_data[key] if system in ("any", actual_os)): + for name, req in conanfile.conan_data[key][system].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): + pip_requirements[name] = req + + if add_dependencies: + for name, dep in reversed(self.conanfile.dependencies.host.items()): + self._populate_pip_requirements(dep, key, pip_requirements, actual_os, add_dependencies = False) + + + def _install_pip_requirements(self, file_suffix, file_content, output_folder, env_vars, py_interp_venv): + if len(file_content) > 0: + pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt') + save(self.conanfile, pip_file_path, "\n".join(file_content)) + with env_vars.apply(): + subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", pip_file_path])