mirror
/
conan-config
mirror of https://github.com/Ultimaker/conan-config.git
1
0
Fork 1
Code Issues Projects Releases Wiki Activity
Browse Source

Restore VirtualPythonEnv being a generator

pull/12/head
Erwan MATHIEU 1 year ago
parent
0fb89b8a61
commit
2effe7c929
5 changed files with 179 additions and 161 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 128
      extensions/deployers/virtual_python_env.py
  2. 37
      extensions/generators/EnvScriptBuilder.py
  3. 22
      extensions/generators/GitHubActionsBuildEnv.py
  4. 24
      extensions/generators/GitHubActionsRunEnv.py
  5. 129
      extensions/generators/VirtualPythonEnv.py

128
extensions/deployers/virtual_python_env.py
Unescape View File

@ -1,128 +0,0 @@
import os
import sys
from io import StringIO
from shutil import which
from pathlib import Path
from conan import ConanFile
from conan.errors import ConanException
from conan.tools.files import copy, save, load
from conan.tools.scm import Version
from conan.tools.env import VirtualRunEnv
def populate_pip_requirements(key, pip_requirements, conan_data, actual_os):
if conan_data is not None and key in conan_data:
for system in (system for system in conan_data[key] if system in ("any", actual_os)):
for name, req in conan_data[key][system].items():
if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]):
pip_requirements[name] = req
def populate_full_pip_requirements(conanfile, key, pip_requirements, actual_os):
populate_pip_requirements(key, pip_requirements, conanfile.conan_data, actual_os)
for name, dep in reversed(conanfile.dependencies.host.items()):
populate_pip_requirements(key, pip_requirements, dep.conan_data, actual_os)
def install_pip_requirements(file_suffix, file_content, output_folder, conanfile, venv_vars, py_interp_venv):
if len(file_content) > 0:
pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt')
save(conanfile, pip_file_path, "\n".join(file_content))
with venv_vars.apply():
conanfile.run(f"{py_interp_venv} -m pip install -r {pip_file_path}", env="conanrun")
def deploy(graph, output_folder, **kwargs):
if graph.root.conanfile.name is None:
conanfile: ConanFile = graph.nodes[1].conanfile
else:
conanfile: ConanFile = graph.root.conanfile
if output_folder is None:
output_folder = "venv"
else:
output_folder = str(Path(output_folder, "venv"))
bin_venv_path = "Scripts" if conanfile.settings.os == "Windows" else "bin"
# Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter
try:
cpython = conanfile.dependencies["cpython"]
py_interp = cpython.conf_info.get("user.cpython:python").replace("\\", "/")
except KeyError:
py_interp = sys.executable
vr = VirtualRunEnv(conanfile)
env = vr.environment()
sys_vars = env.vars(conanfile, scope="run")
conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'")
with sys_vars.apply():
conanfile.run(f"""{py_interp} -m venv --copies {output_folder}""", env="conanrun", scope="run")
# Make sure there executable is named the same on all three OSes this allows it to be called with `python`
# simplifying GH Actions steps
if conanfile.settings.os != "Windows":
py_interp_venv = Path(output_folder, bin_venv_path, "python")
if not py_interp_venv.exists():
py_interp_venv.hardlink_to(
Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix))
else:
py_interp_venv = Path(output_folder, bin_venv_path,
Path(sys.executable).stem + Path(sys.executable).suffix)
buffer = StringIO()
outer = '"' if conanfile.settings.os == "Windows" else "'"
inner = "'" if conanfile.settings.os == "Windows" else '"'
with sys_vars.apply():
conanfile.run(
f"""{py_interp_venv} -c {outer}import sysconfig; print(sysconfig.get_path({inner}purelib{inner})){outer}""",
env="conanrun",
stdout=buffer)
pythonpath = buffer.getvalue().splitlines()[-1]
env.define_path("VIRTUAL_ENV", output_folder)
env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path))
env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path))
env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path))
env.prepend_path("PYTHONPATH", pythonpath)
env.unset("PYTHONHOME")
venv_vars = env.vars(graph.root.conanfile, scope="run")
venv_vars.save_script("virtual_python_env")
# Install some base_packages
with venv_vars.apply():
conanfile.run(f"""{py_interp_venv} -m pip install wheel setuptools""", env="conanrun")
if conanfile.settings.os != "Windows":
content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(graph.root.conanfile,
os.path.join(
output_folder,
bin_venv_path,
"activate"))
save(graph.root.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content)
pip_requirements = {}
populate_full_pip_requirements(conanfile, "pip_requirements", pip_requirements, str(conanfile.settings.os))
requirements_hashed_txt = []
requirements_url_txt = []
for name, req in pip_requirements.items():
if "url" in req:
requirements_url_txt.append(req['url'])
else:
requirement_txt = [f"{name}=={req['version']}"]
if "hashes" in req:
for hash_str in req['hashes']:
requirement_txt.append(f"--hash={hash_str}")
requirements_hashed_txt.append(" ".join(requirement_txt))
install_pip_requirements("hashed", requirements_hashed_txt, output_folder, conanfile, venv_vars, py_interp_venv)
install_pip_requirements("url", requirements_url_txt, output_folder, conanfile, venv_vars, py_interp_venv)
if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool) and conanfile.conan_data is not None and "pip_requirements_dev" in conanfile.conan_data:
install_pip_requirements("dev", conanfile.conan_data["pip_requirements_dev"], output_folder, conanfile, venv_vars, py_interp_venv)

37
extensions/generators/EnvScriptBuilder.py
Unescape View File

@ -0,0 +1,37 @@
from conan.tools.files import save
class EnvScriptBuilder:
def __init__(self):
self._variables = {}
def set_variable(self, name: str, value: str):
self._variables[name] = value
def set_environment(self, env):
for name, value in env.items():
self.set_variable(name, value)
def save(self, path, conanfile, append_to=None) -> None:
file_path = path
content = ""
for name, value in self._variables.items():
set_variable = f'{name}={value}'
if append_to is not None:
set_variable = f"echo {set_variable} >> {append_to}"
else:
set_variable = f"export {set_variable}"
content += f"{set_variable}\n"
if conanfile.settings.get_safe("os") == "Windows":
if conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool):
file_path += ".ps1"
else:
file_path += ".bat"
else:
file_path += ".sh"
conanfile.output.info(f"Saving environment script to {file_path}")
save(conanfile, file_path, content)

22
extensions/generators/GitHubActionsBuildEnv.py
Unescape View File

@ -1,32 +1,22 @@
from pathlib import Path
from jinja2 import Template
from conan import ConanFile
from conan.tools.env import VirtualBuildEnv
from conan.tools.files import save
from EnvScriptBuilder import EnvScriptBuilder
class GitHubActionsBuildEnv:
def __init__(self, conanfile: ConanFile):
self.conanfile: ConanFile = conanfile
self.settings = self.conanfile.settings
def generate(self):
template = Template(
"""{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""")
build_env = VirtualBuildEnv(self.conanfile)
env = build_env.environment()
envvars = env.vars(self.conanfile, scope="build")
env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else ""
content = template.render(envvars=envvars, env_prefix=env_prefix)
filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_buildenv"))
if self.conanfile.settings.get_safe("os") == "Windows":
if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool):
filepath += ".ps1"
else:
filepath += ".bat"
else:
filepath += ".sh"
save(self.conanfile, filepath, content)
script_builder = EnvScriptBuilder()
script_builder.set_environment(envvars)
script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV")

24
extensions/generators/GitHubActionsRunEnv.py
Unescape View File

@ -1,32 +1,22 @@
from pathlib import Path
from jinja2 import Template
from conan import ConanFile
from conan.tools.env import VirtualRunEnv
from conan.tools.files import save
from EnvScriptBuilder import EnvScriptBuilder
class GitHubActionsRunEnv:
def __init__(self, conanfile: ConanFile):
self.conanfile: ConanFile = conanfile
self.settings = self.conanfile.settings
def generate(self):
template = Template(
"""{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""")
build_env = VirtualRunEnv(self.conanfile)
env = build_env.environment()
run_env = VirtualRunEnv(self.conanfile)
env = run_env.environment()
envvars = env.vars(self.conanfile, scope="run")
env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else ""
content = template.render(envvars=envvars, env_prefix=env_prefix)
filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_runenv"))
if self.conanfile.settings.get_safe("os") == "Windows":
if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool):
filepath += ".ps1"
else:
filepath += ".bat"
else:
filepath += ".sh"
save(self.conanfile, filepath, content)
script_builder = EnvScriptBuilder()
script_builder.set_environment(envvars)
script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV")

129
extensions/generators/VirtualPythonEnv.py
Unescape View File

@ -0,0 +1,129 @@
import os
import sys
from io import StringIO
from shutil import which
from pathlib import Path
from conan import ConanFile
from conan.errors import ConanException
from conan.tools.files import copy, save, load
from conan.tools.scm import Version
from conan.tools.env import VirtualRunEnv
import subprocess
class VirtualPythonEnv:
def __init__(self, conanfile: ConanFile):
self.conanfile: ConanFile = conanfile
def generate(self) -> None:
output_folder = "venv"
bin_venv_path = "Scripts" if self.conanfile.settings.os == "Windows" else "bin"
# Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter
try:
cpython = self.conanfile.dependencies["cpython"]
py_interp = cpython.conf_info.get("user.cpython:python").replace("\\", "/")
except KeyError:
py_interp = sys.executable
run_env = VirtualRunEnv(self.conanfile)
env = run_env.environment()
env_vars = env.vars(self.conanfile, scope="run")
self.conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'")
with env_vars.apply():
subprocess.run([py_interp, "-m", "venv", "--copies", output_folder])
# Make sure there executable is named the same on all three OSes this allows it to be called with `python`
# simplifying GH Actions steps
if self.conanfile.settings.os != "Windows":
py_interp_venv = Path(output_folder, bin_venv_path, "python")
if not py_interp_venv.exists():
py_interp_venv.hardlink_to(
Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix))
else:
py_interp_venv = Path(output_folder, bin_venv_path,
Path(sys.executable).stem + Path(sys.executable).suffix)
with env_vars.apply():
buffer = subprocess.run([py_interp_venv, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], capture_output=True, encoding="utf-8").stdout
pythonpath = buffer.splitlines()[-1]
env.define_path("VIRTUAL_ENV", output_folder)
env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path))
env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path))
env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path))
env.prepend_path("PYTHONPATH", pythonpath)
env.unset("PYTHONHOME")
filepath = str(Path(self.conanfile.generators_folder).joinpath("supercoucou_runenv"))
env_vars.save_script(filepath)
# Install some base_packages
with env_vars.apply():
subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"])
if self.conanfile.settings.os != "Windows":
content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(self.conanfile,
os.path.join(
output_folder,
bin_venv_path,
"activate"))
save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content)
pip_requirements = {}
self._populate_pip_requirements(self.conanfile, "pip_requirements", pip_requirements, str(self.conanfile.settings.os))
requirements_hashed_txt = []
requirements_url_txt = []
for name, req in pip_requirements.items():
if "url" in req:
requirements_url_txt.append(req['url'])
else:
requirement_txt = [f"{name}=={req['version']}"]
if "hashes" in req:
for hash_str in req['hashes']:
requirement_txt.append(f"--hash={hash_str}")
requirements_hashed_txt.append(" ".join(requirement_txt))
self._install_pip_requirements("hashed", requirements_hashed_txt, output_folder, env_vars, py_interp_venv)
self._install_pip_requirements("url", requirements_url_txt, output_folder, env_vars, py_interp_venv)
if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default = False, check_type = bool):
pip_requirements_dev = []
self._populate_pip_requirements_dev(self.conanfile, pip_requirements_dev)
print(pip_requirements_dev)
self._install_pip_requirements("dev", pip_requirements_dev, output_folder, env_vars, py_interp_venv)
def _populate_pip_requirements_dev(self, conanfile, pip_requirements_dev, add_dependencies = True):
if hasattr(conanfile, "conan_data") and "pip_requirements_dev" in conanfile.conan_data:
print(conanfile.conan_data["pip_requirements_dev"])
pip_requirements_dev += conanfile.conan_data["pip_requirements_dev"]
if add_dependencies:
for name, dep in reversed(self.conanfile.dependencies.host.items()):
self._populate_pip_requirements_dev(dep, pip_requirements_dev, add_dependencies = False)
def _populate_pip_requirements(self, conanfile, key, pip_requirements, actual_os, add_dependencies = True):
if hasattr(conanfile, "conan_data") and key in conanfile.conan_data:
for system in (system for system in conanfile.conan_data[key] if system in ("any", actual_os)):
for name, req in conanfile.conan_data[key][system].items():
if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]):
pip_requirements[name] = req
if add_dependencies:
for name, dep in reversed(self.conanfile.dependencies.host.items()):
self._populate_pip_requirements(dep, key, pip_requirements, actual_os, add_dependencies = False)
def _install_pip_requirements(self, file_suffix, file_content, output_folder, env_vars, py_interp_venv):
if len(file_content) > 0:
pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt')
save(self.conanfile, pip_file_path, "\n".join(file_content))
with env_vars.apply():
subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", pip_file_path])
Write Preview
Loading…
Cancel
Save