From df43c4f06347541c235c156afbe0a24910b38b1c Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Tue, 28 Nov 2023 07:06:37 +0100 Subject: [PATCH 01/64] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index bcde285..0d879af 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ # Ultimaker Conan configuration files -> Conan is preparing a major update to 2.0, to ensure future proofing our recipes make sure you set the `CONAN_V2_MODE=1` -> environment variable. -> https://docs.conan.io/en/latest/reference/conan_v2_mode.html +For UltiMaker developers From a2e26606994a5de5ec864365f8f9055df4c91bc3 Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Tue, 28 Nov 2023 07:09:09 +0100 Subject: [PATCH 02/64] Update README.md --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d879af..05c7f81 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ # Ultimaker Conan configuration files -For UltiMaker developers +**Conan configuration for UltiMaker developers** + +use as: +``` +conan config install https://github.com/Ultimaker/conan-config.git -a "-b dev" +``` From d0899e11b717101147ee50b9c9fb8057751b928c Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Tue, 28 Nov 2023 07:10:05 +0100 Subject: [PATCH 03/64] Update remotes.txt --- remotes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/remotes.txt b/remotes.txt index e027f9c..524f6db 100644 --- a/remotes.txt +++ b/remotes.txt @@ -1,3 +1,4 @@ cura https://ultimaker.jfrog.io/artifactory/api/conan/cura-internal True cura-conan-dev https://cura.jfrog.io/artifactory/api/conan/cura-conan-dev True +cura-private-conan-dev https://cura.jfrog.io/artifactory/api/conan/cura-private-conan-dev True conan-center https://center.conan.io True From 9b90adf27b7e80ab9a946e69ddef9c38c5b2755e Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Wed, 29 Nov 2023 06:42:31 +0100 Subject: [PATCH 04/64] Renamed remotes Contributes to CURA-10831 --- remotes.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/remotes.txt b/remotes.txt index 524f6db..69e3f5a 100644 --- a/remotes.txt +++ b/remotes.txt @@ -1,4 +1,2 @@ -cura https://ultimaker.jfrog.io/artifactory/api/conan/cura-internal True -cura-conan-dev https://cura.jfrog.io/artifactory/api/conan/cura-conan-dev True -cura-private-conan-dev https://cura.jfrog.io/artifactory/api/conan/cura-private-conan-dev True -conan-center https://center.conan.io True +cura https://cura.jfrog.io/artifactory/api/conan/cura-conan-dev True +cura-private https://cura.jfrog.io/artifactory/api/conan/cura-private-conan-dev True From 3dec0e6542592689afceee14b246ae3c5595323f Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Wed, 29 Nov 2023 17:03:48 +0100 Subject: [PATCH 05/64] Set `CONAN_DISABLE_STRICT_MODE` for cpython Dirty fix for Windows Contributes to CURA-10831 --- profiles/cura.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index a68b097..d6b3863 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -19,5 +19,5 @@ dulcificum:compiler.cppstd=20 {% endif %} [options] [env] - +cpython*:CONAN_DISABLE_STRICT_MODE=1 [conf] From 3226488623c642b40ca7ce3f62d3f33de046d11e Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Sat, 23 Dec 2023 18:30:28 +0100 Subject: [PATCH 06/64] Use cpp std 20 for cura_settings --- profiles/cura.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index d6b3863..073d99d 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -12,6 +12,7 @@ curaengine_grpc_definitions:compiler.cppstd=20 scripta:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum:compiler.cppstd=20 +cura_settings:compiler.cppstd=20 {% if compiler == 'gcc' %}compiler.libcxx=libstdc++11 {% elif compiler == 'apple-clang' %}compiler.libcxx=libc++ From 491dc7c006aaa26d2ab83eaa9e3dd9b30feec3f7 Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Sun, 11 Feb 2024 17:38:33 +0100 Subject: [PATCH 07/64] Getting the configuration ready for Conan v2 Contributes to CURA-11622 --- conan.conf | 24 --- extensions/deployers/virtual_python_env.py | 105 +++++++++++ .../generators}/GitHubActionsBuildEnv.py | 0 .../generators}/GitHubActionsRunEnv.py | 0 .../generators}/PyCharmRunEnv.py | 0 generators/VirtualPythonEnv.py | 174 ------------------ global.conf | 6 +- profiles/cura.jinja | 23 +-- profiles/cura_build.jinja | 11 -- remotes.json | 19 ++ remotes.txt | 2 - 11 files changed, 131 insertions(+), 233 deletions(-) delete mode 100644 conan.conf create mode 100644 extensions/deployers/virtual_python_env.py rename {generators => extensions/generators}/GitHubActionsBuildEnv.py (100%) rename {generators => extensions/generators}/GitHubActionsRunEnv.py (100%) rename {generators => extensions/generators}/PyCharmRunEnv.py (100%) delete mode 100644 generators/VirtualPythonEnv.py create mode 100644 remotes.json delete mode 100644 remotes.txt diff --git a/conan.conf b/conan.conf deleted file mode 100644 index 0ab8d76..0000000 --- a/conan.conf +++ /dev/null @@ -1,24 +0,0 @@ -[log] -run_to_output = True # environment CONAN_LOG_RUN_TO_OUTPUT -run_to_file = False # environment CONAN_LOG_RUN_TO_FILE -level = warn # environment CONAN_LOGGING_LEVEL -print_run_commands = True # environment CONAN_PRINT_RUN_COMMANDS - -[general] -default_profile = default -compression_level = 9 # environment CONAN_COMPRESSION_LEVEL -sysrequires_sudo = True # environment CONAN_SYSREQUIRES_SUDO -request_timeout = 60 # environment CONAN_REQUEST_TIMEOUT (seconds) -default_package_id_mode = full_package_mode # environment CONAN_DEFAULT_PACKAGE_ID_MODE -revisions_enabled = True -scm_to_conandata = True -use_always_short_paths = True -config_install_interval = 12h - -[storage] -path = ./data - -[proxies] - -[hooks] -attribute_checker diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py new file mode 100644 index 0000000..ecc6d52 --- /dev/null +++ b/extensions/deployers/virtual_python_env.py @@ -0,0 +1,105 @@ +import os +import sys +from io import StringIO +from shutil import which +from pathlib import Path + +from conan import ConanFile +from conan.errors import ConanException +from conan.tools.files import copy, save, load +from conan.tools.scm import Version +from conan.tools.env import VirtualRunEnv + + +def deploy(graph, output_folder, **kwargs): + conanfile: ConanFile = graph.root.conanfile + + bin_venv_path = "Scripts" if conanfile.settings.os == "Windows" else "bin" + + # Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter + try: + cpython = conanfile.dependencies["cpython"] + py_interp = cpython.conf_info.get("user.cpython:python").replace("\\", "/") + except KeyError: + py_interp = sys.executable + + run_env = VirtualRunEnv(conanfile) + env = run_env.environment() + sys_vars = env.vars(conanfile, scope="run") + + with sys_vars.apply(): + conanfile.run(f"""{py_interp} -m venv --copies {output_folder}""", env="conanrun", scope="run") + + # Make sure there executable is named the same on all three OSes this allows it to be called with `python` + # simplifying GH Actions steps + if conanfile.settings.os != "Windows": + py_interp_venv = Path(output_folder, bin_venv_path, "python") + if not py_interp_venv.exists(): + py_interp_venv.hardlink_to( + Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) + else: + py_interp_venv = Path(output_folder, bin_venv_path, + Path(sys.executable).stem + Path(sys.executable).suffix) + + buffer = StringIO() + outer = '"' if conanfile.settings.os == "Windows" else "'" + inner = "'" if conanfile.settings.os == "Windows" else '"' + with sys_vars.apply(): + conanfile.run( + f"""{py_interp_venv} -c {outer}import sysconfig; print(sysconfig.get_path({inner}purelib{inner})){outer}""", + env="conanrun", + stdout=buffer) + pythonpath = buffer.getvalue().splitlines()[-1] + + env.define_path("VIRTUAL_ENV", output_folder) + env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("PYTHONPATH", pythonpath) + env.unset("PYTHONHOME") + venv_vars = env.vars(conanfile, scope="run") + venv_vars.save_script("virtual_python_env") + + # Install some base_packages + with venv_vars.apply(): + conanfile.run(f"""{py_interp_venv} -m pip install wheel setuptools""", env="conanrun") + + if conanfile.settings.os != "Windows": + content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(conanfile, + os.path.join( + output_folder, + bin_venv_path, + "activate")) + save(conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) + + pip_requirements = {} + if conanfile.conan_data is not None and "pip_requirements" in conanfile.conan_data: + for oses in ("any", str(conanfile.settings.os)): + for name, req in conanfile.conan_data["pip_requirements"][oses].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): + pip_requirements[name] = req + + for name, dep in reversed(conanfile.dependencies.host.items()): + if dep.conan_data is None: + continue + if "pip_requirements" in dep.conan_data: + for oses in ("any", str(conanfile.settings.os)): + for name, req in conanfile.conan_data["pip_requirements"][oses].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version( + req["version"]): + pip_requirements[name] = req + + requirements_txt = "" + for name, req in pip_requirements.items(): + if "url" in req: + requirements_txt += f"{req['url']}" + else: + requirements_txt += f"{name}=={req['version']}" + for hash_str in req['hashes']: + requirements_txt += f" --hash={hash_str}" + requirements_txt += "\n" + + save(conanfile, os.path.join(output_folder, 'conan', 'requirements.txt'), requirements_txt) + with venv_vars.apply(): + conanfile.run(f"{py_interp_venv} -m pip install -r {os.path.join(output_folder, 'conan', 'requirements.txt')}", + env="conanrun") diff --git a/generators/GitHubActionsBuildEnv.py b/extensions/generators/GitHubActionsBuildEnv.py similarity index 100% rename from generators/GitHubActionsBuildEnv.py rename to extensions/generators/GitHubActionsBuildEnv.py diff --git a/generators/GitHubActionsRunEnv.py b/extensions/generators/GitHubActionsRunEnv.py similarity index 100% rename from generators/GitHubActionsRunEnv.py rename to extensions/generators/GitHubActionsRunEnv.py diff --git a/generators/PyCharmRunEnv.py b/extensions/generators/PyCharmRunEnv.py similarity index 100% rename from generators/PyCharmRunEnv.py rename to extensions/generators/PyCharmRunEnv.py diff --git a/generators/VirtualPythonEnv.py b/generators/VirtualPythonEnv.py deleted file mode 100644 index acd7e50..0000000 --- a/generators/VirtualPythonEnv.py +++ /dev/null @@ -1,174 +0,0 @@ -import sys - -import os -from io import StringIO - -from pathlib import Path - -from jinja2 import Template - -from conan import ConanFile -from conan.tools.env import VirtualRunEnv -from conans.model import Generator -from conans.errors import ConanException - - -class VirtualPythonEnv(Generator): - - @property - def _script_ext(self): - if self.conanfile.settings.get_safe("os") == "Windows": - return ".ps1" - return ".sh" - - @property - def _venv_path(self): - if self.settings.os == "Windows": - return "Scripts" - return "bin" - - @property - def filename(self): - pass - - @property - def content(self): - conanfile: ConanFile = self.conanfile - python_interpreter = Path(self.conanfile.deps_user_info["cpython"].python) - - # When on Windows execute as Windows Path - if conanfile.settings.os == "Windows": - python_interpreter = Path(*[f'"{p}"' if " " in p else p for p in python_interpreter.parts]) - - # Create the virtual environment - print(f"Creating virtual environment in {conanfile.install_folder}") - print(f"Creating virtual environment in {conanfile.build_folder}") - if conanfile.in_local_cache: - venv_folder = conanfile.install_folder - else: - venv_folder = conanfile.build_folder if conanfile.build_folder else Path(os.getcwd(), "venv") - - conanfile.output.info(f"Creating virtual environment in {venv_folder}") - conanfile.run(f"""{python_interpreter} -m venv {venv_folder}""", run_environment = True, env = "conanrun") - - # Make sure there executable is named the same on all three OSes this allows it to be called with `python` - # simplifying GH Actions steps - if conanfile.settings.os != "Windows": - python_venv_interpreter = Path(venv_folder, self._venv_path, "python") - if not python_venv_interpreter.exists(): - python_venv_interpreter.hardlink_to( - Path(venv_folder, self._venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) - else: - python_venv_interpreter = Path(venv_folder, self._venv_path, Path(sys.executable).stem + Path(sys.executable).suffix) - - if not python_venv_interpreter.exists(): - raise ConanException(f"Virtual environment Python interpreter not found at: {python_venv_interpreter}") - if conanfile.settings.os == "Windows": - python_venv_interpreter = Path(*[f'"{p}"' if " " in p else p for p in python_venv_interpreter.parts]) - - buffer = StringIO() - outer = '"' if conanfile.settings.os == "Windows" else "'" - inner = "'" if conanfile.settings.os == "Windows" else '"' - conanfile.run(f"{python_venv_interpreter} -c {outer}import sysconfig; print(sysconfig.get_path({inner}purelib{inner})){outer}", - env = "conanrun", - output = buffer) - pythonpath = buffer.getvalue().splitlines()[-1] - - run_env = VirtualRunEnv(conanfile) - env = run_env.environment() - - if hasattr(self.conanfile, f"_{self.conanfile.name}_run_env"): - project_run_env = getattr(self.conanfile, f"_{self.conanfile.name}_run_env") - if project_run_env: - env.compose_env(project_run_env) # TODO: Add logic for dependencies - - env.define_path("VIRTUAL_ENV", venv_folder) - env.prepend_path("PATH", os.path.join(venv_folder, self._venv_path)) - env.prepend_path("LD_LIBRARY_PATH", os.path.join(venv_folder, self._venv_path)) - env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(venv_folder, self._venv_path)) - env.prepend_path("PYTHONPATH", pythonpath) - env.unset("PYTHONHOME") - - # Install some base_packages - conanfile.run(f"""{python_venv_interpreter} -m pip install wheel setuptools""", run_environment = True, env = "conanrun") - - # Install pip_requirements from dependencies - for dep_name in reversed(conanfile.deps_user_info): - dep_user_info = conanfile.deps_user_info[dep_name] - if len(dep_user_info.vars) == 0: - continue - pip_req_paths = [conanfile.deps_cpp_info[dep_name].res_paths[i] for i, req_path in - enumerate(conanfile.deps_cpp_info[dep_name].resdirs) if req_path.endswith("pip_requirements")] - if len(pip_req_paths) != 1: - continue - pip_req_base_path = Path(pip_req_paths[0]) - if hasattr(dep_user_info, "pip_requirements"): - req_txt = pip_req_base_path.joinpath(dep_user_info.pip_requirements) - if req_txt.exists(): - conanfile.run(f"{python_venv_interpreter} -m pip install -r {req_txt} --upgrade", run_environment = True, - env = "conanrun") - conanfile.output.success(f"Dependency {dep_name} specifies pip_requirements in user_info installed!") - else: - conanfile.output.warn(f"Dependency {dep_name} specifies pip_requirements in user_info but {req_txt} can't be found!") - - if hasattr(dep_user_info, "pip_requirements_git"): - req_txt = pip_req_base_path.joinpath(dep_user_info.pip_requirements_git) - if req_txt.exists(): - conanfile.run(f"{python_venv_interpreter} -m pip install -r {req_txt} --upgrade", run_environment = True, - env = "conanrun") - conanfile.output.success(f"Dependency {dep_name} specifies pip_requirements_git in user_info installed!") - else: - conanfile.output.warn( - f"Dependency {dep_name} specifies pip_requirements_git in user_info but {req_txt} can't be found!") - - if hasattr(dep_user_info, "pip_requirements_build"): - req_txt = pip_req_base_path.joinpath(dep_user_info.pip_requirements_build) - if req_txt.exists(): - conanfile.run(f"{python_venv_interpreter} -m pip install -r {req_txt} --upgrade", run_environment = True, - env = "conanrun") - conanfile.output.success(f"Dependency {dep_name} specifies pip_requirements_build in user_info installed!") - else: - conanfile.output.warn( - f"Dependency {dep_name} specifies pip_requirements_build in user_info but {req_txt} can't be found!") - - if not conanfile.in_local_cache and hasattr(conanfile, "requirements_txts"): - # Install the Python requirements of the current conanfile requirements*.txt - pip_req_base_path = Path(conanfile.source_folder) - - for req_path in sorted(conanfile.requirements_txts, reverse = True): - req_txt = pip_req_base_path.joinpath(req_path) - if req_txt.exists(): - conanfile.run(f"{python_venv_interpreter} -m pip install -r {req_txt} --upgrade", run_environment = True, - env = "conanrun") - conanfile.output.success(f"Requirements file {req_txt} installed!") - else: - conanfile.output.warn(f"Requirements file {req_txt} can't be found!") - - # Add all dlls/dylibs/so found in site-packages to the PATH, DYLD_LIBRARY_PATH and LD_LIBRARY_PATH - dll_paths = list({ dll.parent for dll in Path(pythonpath).glob("**/*.dll") }) - for dll_path in dll_paths: - env.append_path("PATH", str(dll_path)) - - dylib_paths = list({ dylib.parent for dylib in Path(pythonpath).glob("**/*.dylib") }) - for dylib_path in dylib_paths: - env.append_path("DYLD_LIBRARY_PATH", str(dylib_path)) - - so_paths = list({ so.parent for so in Path(pythonpath).glob("**/*.dylib") }) - for so_path in so_paths: - env.append_path("LD_LIBRARY_PATH", str(so_path)) - - full_envvars = env.vars(conanfile, scope = "conanrun") - - # Generate the Python Virtual Environment Script - full_envvars.save_sh(Path(venv_folder, self._venv_path, "activate")) - full_envvars.save_bat(Path(venv_folder, self._venv_path, "activate.bat")) - full_envvars.save_ps1(Path(venv_folder, self._venv_path, "Activate.ps1")) - - # Generate the GitHub Action activation script - env_prefix = "Env:" if conanfile.settings.os == "Windows" else "" - activate_github_actions_buildenv = Template(r"""{% for var, value in envvars.items() %}echo "{{ var }}={{ value }}" >> ${{ env_prefix }}GITHUB_ENV -{% endfor %}""").render(envvars = full_envvars, env_prefix = env_prefix) - - return { - str(Path(venv_folder, self._venv_path, f"activate_github_actions_env{self._script_ext}")): activate_github_actions_buildenv - } diff --git a/global.conf b/global.conf index 33886ac..5cea560 100644 --- a/global.conf +++ b/global.conf @@ -1,11 +1,7 @@ core:default_profile = cura.jinja core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja -tools.env.virtualenv:auto_use = True tools.gnu:define_libcxx11_abi = True tools.build:skip_test = False -# FIXME: Needs to be commented out for OpenSSL to work but if we wan't to create ps1 scripts it needs to be set to True -# Otherwise .bat files are created. Maybe we should define this on a recipe basis: -# -#{% if platform.system() == 'Windows' %}tools.env.virtualenv:powershell=True{% endif %} +{% if platform.system() == 'Windows' %}tools.env.virtualenv:powershell=True{% endif %} \ No newline at end of file diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 073d99d..0d7faa9 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -1,24 +1,13 @@ include(default) -[build_requires] - [settings] compiler.cppstd=17 -curaengine:compiler.cppstd=20 curaengine*:compiler.cppstd=20 -curaengine_plugin_infill_generate:compiler.cppstd=20 -curaengine_plugin_gradual_flow:compiler.cppstd=20 -curaengine_grpc_definitions:compiler.cppstd=20 -scripta:compiler.cppstd=20 +curaengine_plugin_infill_generate*:compiler.cppstd=20 +curaengine_plugin_gradual_flow*:compiler.cppstd=20 +curaengine_grpc_definitions*:compiler.cppstd=20 +scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 -dulcificum:compiler.cppstd=20 -cura_settings:compiler.cppstd=20 +dulcificum*:compiler.cppstd=20 +cura_settings*:compiler.cppstd=20 -{% if compiler == 'gcc' %}compiler.libcxx=libstdc++11 -{% elif compiler == 'apple-clang' %}compiler.libcxx=libc++ -{% elif compiler == 'Visual Studio' %}compiler.toolset=v143 -{% endif %} -[options] -[env] -cpython*:CONAN_DISABLE_STRICT_MODE=1 -[conf] diff --git a/profiles/cura_build.jinja b/profiles/cura_build.jinja index c747fd0..de80517 100644 --- a/profiles/cura_build.jinja +++ b/profiles/cura_build.jinja @@ -1,15 +1,4 @@ include(default) -[build_requires] - [settings] compiler.cppstd=17 -{% if compiler == 'gcc' %}compiler.libcxx=libstdc++11 -{% elif compiler == 'apple-clang' %}compiler.libcxx=libc++ -{% elif compiler == 'Visual Studio' %}compiler.toolset=v143 -{% endif %} -[options] - -[env] - -[conf] diff --git a/remotes.json b/remotes.json new file mode 100644 index 0000000..8cd6759 --- /dev/null +++ b/remotes.json @@ -0,0 +1,19 @@ +{ + "remotes": [ + { + "name": "conancenter", + "url": "https://center.conan.io", + "verify_ssl": true + }, + { + "name": "cura", + "url": "https://cura.jfrog.io/artifactory/api/conan/cura-conan-dev", + "verify_ssl": true + }, + { + "name": "cura-private", + "url": "https://cura.jfrog.io/artifactory/api/conan/cura-private-conan-dev", + "verify_ssl": true + } + ] +} \ No newline at end of file diff --git a/remotes.txt b/remotes.txt deleted file mode 100644 index 69e3f5a..0000000 --- a/remotes.txt +++ /dev/null @@ -1,2 +0,0 @@ -cura https://cura.jfrog.io/artifactory/api/conan/cura-conan-dev True -cura-private https://cura.jfrog.io/artifactory/api/conan/cura-private-conan-dev True From db1ff62786f1207d89ed47a18166252b755458b3 Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Mon, 12 Feb 2024 08:04:26 +0100 Subject: [PATCH 08/64] Resolve for prerelease Conan v2 discourages the usage of alias. We should use prereleases in the future. https://docs.conan.io/2/devops/versioning/resolve_prereleases.html Contributes to CURA-11622 --- global.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/global.conf b/global.conf index 5cea560..4b9863a 100644 --- a/global.conf +++ b/global.conf @@ -3,5 +3,6 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True tools.build:skip_test = False +core.version_ranges:resolve_prereleases=True {% if platform.system() == 'Windows' %}tools.env.virtualenv:powershell=True{% endif %} \ No newline at end of file From 5ea103fa68924c5ef669242258be46ccb6f040af Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Mon, 12 Feb 2024 08:36:18 +0100 Subject: [PATCH 09/64] Fix old generators to work with Conan v2 Contributes to CURA-11622 --- .../generators/GitHubActionsBuildEnv.py | 31 ++++++++++--------- extensions/generators/GitHubActionsRunEnv.py | 29 ++++++++--------- extensions/generators/PyCharmRunEnv.py | 29 ++++++++--------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/extensions/generators/GitHubActionsBuildEnv.py b/extensions/generators/GitHubActionsBuildEnv.py index d7b7305..bd194c7 100644 --- a/extensions/generators/GitHubActionsBuildEnv.py +++ b/extensions/generators/GitHubActionsBuildEnv.py @@ -2,29 +2,30 @@ from pathlib import Path from jinja2 import Template +from conan import ConanFile from conan.tools.env import VirtualBuildEnv -from conans.model import Generator -class GitHubActionsBuildEnv(Generator): +class GitHubActionsBuildEnv: + def __init__(self, conanfile: ConanFile): + self.conanfile: ConanFile = conanfile + self.settings = self.conanfile.settings + + def generate(self): + template = Template( + """{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") + build_env = VirtualBuildEnv(self.conanfile) + env = build_env.environment() + envvars = env.vars(self.conanfile, scope="build") + env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" + content = template.render(envvars=envvars, env_prefix=env_prefix) - @property - def filename(self): filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_buildenv")) if self.conanfile.settings.get_safe("os") == "Windows": - if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type = bool): + if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): filepath += ".ps1" else: filepath += ".bat" else: filepath += ".sh" - return filepath - - @property - def content(self): - template = Template("""{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") - build_env = VirtualBuildEnv(self.conanfile) - env = build_env.environment() - envvars = env.vars(self.conanfile, scope = "build") - env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - return template.render(envvars = envvars, env_prefix = env_prefix) + save(self.conanfile, filepath, content) diff --git a/extensions/generators/GitHubActionsRunEnv.py b/extensions/generators/GitHubActionsRunEnv.py index b79a130..cd50c16 100644 --- a/extensions/generators/GitHubActionsRunEnv.py +++ b/extensions/generators/GitHubActionsRunEnv.py @@ -2,29 +2,30 @@ from pathlib import Path from jinja2 import Template +from conan import ConanFile from conan.tools.env import VirtualRunEnv -from conans.model import Generator -class GitHubActionsRunEnv(Generator): +class GitHubActionsRunEnv: + def __init__(self, conanfile: ConanFile): + self.conanfile: ConanFile = conanfile + self.settings = self.conanfile.settings - @property - def filename(self): + def generate(self): + template = Template( + """{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") + build_env = VirtualRunEnv(self.conanfile) + env = build_env.environment() + envvars = env.vars(self.conanfile, scope="run") + env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" + content = template.render(envvars=envvars, env_prefix=env_prefix) filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_runenv")) if self.conanfile.settings.get_safe("os") == "Windows": - if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type = bool): + if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): filepath += ".ps1" else: filepath += ".bat" else: filepath += ".sh" - return filepath - @property - def content(self): - template = Template("""{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") - build_env = VirtualRunEnv(self.conanfile) - env = build_env.environment() - envvars = env.vars(self.conanfile, scope = "run") - env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - return template.render(envvars = envvars, env_prefix = env_prefix) + save(self.conanfile, filepath, content) diff --git a/extensions/generators/PyCharmRunEnv.py b/extensions/generators/PyCharmRunEnv.py index 14f5570..c80f398 100644 --- a/extensions/generators/PyCharmRunEnv.py +++ b/extensions/generators/PyCharmRunEnv.py @@ -1,12 +1,17 @@ from pathlib import Path from typing import Dict -from conans import tools -from conans.model import Generator +from conan import ConanFile +from conan.tools.scm import Version from jinja2 import Template from conan.tools.env.virtualrunenv import VirtualRunEnv -class PyCharmRunEnv(Generator): + +class PyCharmRunEnv: + def __init__(self, conanfile: ConanFile): + self.conanfile: ConanFile = conanfile + self.settings = self.conanfile.settings + @property def _base_dir(self): return Path("$PROJECT_DIR$", "venv") @@ -14,7 +19,8 @@ class PyCharmRunEnv(Generator): @property def _py_interp(self): if self.settings.os == "Windows": - py_interp = Path(*[f'"{p}"' if " " in p else p for p in self._base_dir.joinpath("Scripts", "python.exe").parts]) + py_interp = Path( + *[f'"{p}"' if " " in p else p for p in self._base_dir.joinpath("Scripts", "python.exe").parts]) return py_interp return self._base_dir.joinpath("bin", "python") @@ -22,15 +28,11 @@ class PyCharmRunEnv(Generator): def _site_packages(self): if self.settings.os == "Windows": return self._base_dir.joinpath("Lib", "site-packages") - py_version = tools.Version(self.conanfile.deps_cpp_info["cpython"].version) + py_version = Version(self.dependencies["cpython"].ref.version) return self._base_dir.joinpath("lib", f"python{py_version.major}.{py_version.minor}", "site-packages") - @property - def filename(self): - pass - - @property - def content(self) -> Dict[str, str]: + def generate(self) -> Dict[str, str]: + # Mapping of file names -> files # Mapping of file names -> files run_configurations: Dict[str, str] = {} @@ -57,6 +59,5 @@ class PyCharmRunEnv(Generator): with open(target["jinja_path"], "r") as f: template = Template(f.read()) run_configuration = template.render(target) - run_configurations[str(Path(self.conanfile.source_folder).joinpath(".run", f"{target['name']}.run.xml"))] = run_configuration - - return run_configurations + run_configurations[str(Path(self.conanfile.source_folder).joinpath(".run", + f"{target['name']}.run.xml"))] = run_configuration From ede2a5fcee47bbbcdc84a2d45d015b57c05e0aab Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Mon, 12 Feb 2024 08:39:48 +0100 Subject: [PATCH 10/64] Explicitly us a new conan-v2 remote Contributes to CURA-11622 --- remotes.json | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/remotes.json b/remotes.json index 8cd6759..9386449 100644 --- a/remotes.json +++ b/remotes.json @@ -6,13 +6,8 @@ "verify_ssl": true }, { - "name": "cura", - "url": "https://cura.jfrog.io/artifactory/api/conan/cura-conan-dev", - "verify_ssl": true - }, - { - "name": "cura-private", - "url": "https://cura.jfrog.io/artifactory/api/conan/cura-private-conan-dev", + "name": "cura-conan-v2", + "url": "https://cura.jfrog.io/artifactory/api/conan/cura-conan-v2-local-dev", "verify_ssl": true } ] From d692941e8ec3b51ca7c814846e9bd852aa9f3e6b Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Mon, 12 Feb 2024 14:56:58 +0100 Subject: [PATCH 11/64] Implicitly use venv as output folder Contributes to CURA-11622 --- extensions/deployers/virtual_python_env.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index ecc6d52..90f7469 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -13,6 +13,8 @@ from conan.tools.env import VirtualRunEnv def deploy(graph, output_folder, **kwargs): conanfile: ConanFile = graph.root.conanfile + if output_folder is None: + output_folder = "venv" bin_venv_path = "Scripts" if conanfile.settings.os == "Windows" else "bin" From 6838b940c0f13fb257eb0014e8c8423eb5f68855 Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Mon, 12 Feb 2024 18:02:58 +0100 Subject: [PATCH 12/64] fix running from cli Contributes to CURA-11622 --- extensions/deployers/virtual_python_env.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index 90f7469..95927f1 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -12,7 +12,10 @@ from conan.tools.env import VirtualRunEnv def deploy(graph, output_folder, **kwargs): - conanfile: ConanFile = graph.root.conanfile + if graph.root.conanfile.name is None: + conanfile: ConanFile = graph.nodes[1].conanfile + else: + conanfile: ConanFile = graph.root.conanfile if output_folder is None: output_folder = "venv" @@ -25,10 +28,11 @@ def deploy(graph, output_folder, **kwargs): except KeyError: py_interp = sys.executable - run_env = VirtualRunEnv(conanfile) - env = run_env.environment() + vr = VirtualRunEnv(conanfile) + env = vr.environment() sys_vars = env.vars(conanfile, scope="run") + conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'") with sys_vars.apply(): conanfile.run(f"""{py_interp} -m venv --copies {output_folder}""", env="conanrun", scope="run") @@ -59,7 +63,7 @@ def deploy(graph, output_folder, **kwargs): env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) env.prepend_path("PYTHONPATH", pythonpath) env.unset("PYTHONHOME") - venv_vars = env.vars(conanfile, scope="run") + venv_vars = env.vars(graph.root.conanfile, scope="run") venv_vars.save_script("virtual_python_env") # Install some base_packages @@ -67,12 +71,12 @@ def deploy(graph, output_folder, **kwargs): conanfile.run(f"""{py_interp_venv} -m pip install wheel setuptools""", env="conanrun") if conanfile.settings.os != "Windows": - content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(conanfile, + content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(graph.root.conanfile, os.path.join( output_folder, bin_venv_path, "activate")) - save(conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) + save(graph.root.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) pip_requirements = {} if conanfile.conan_data is not None and "pip_requirements" in conanfile.conan_data: @@ -81,7 +85,7 @@ def deploy(graph, output_folder, **kwargs): if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): pip_requirements[name] = req - for name, dep in reversed(conanfile.dependencies.host.items()): + for name, dep in reversed(graph.root.conanfile.dependencies.host.items()): if dep.conan_data is None: continue if "pip_requirements" in dep.conan_data: @@ -101,7 +105,7 @@ def deploy(graph, output_folder, **kwargs): requirements_txt += f" --hash={hash_str}" requirements_txt += "\n" - save(conanfile, os.path.join(output_folder, 'conan', 'requirements.txt'), requirements_txt) + save(graph.root.conanfile, os.path.join(output_folder, 'conan', 'requirements.txt'), requirements_txt) with venv_vars.apply(): conanfile.run(f"{py_interp_venv} -m pip install -r {os.path.join(output_folder, 'conan', 'requirements.txt')}", env="conanrun") From 2a88edcd4b7eb692abf8be871355ed163bca0dec Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Tue, 3 Sep 2024 13:32:47 +0200 Subject: [PATCH 13/64] Change to new conan v2 artifacotry repository adress. part of CURA-11622 --- remotes.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/remotes.json b/remotes.json index 9386449..cdfb47b 100644 --- a/remotes.json +++ b/remotes.json @@ -6,9 +6,9 @@ "verify_ssl": true }, { - "name": "cura-conan-v2", - "url": "https://cura.jfrog.io/artifactory/api/conan/cura-conan-v2-local-dev", + "name": "cura-conan2", + "url": "https://cura.jfrog.io/artifactory/api/conan/cura-conan2-dev", "verify_ssl": true } ] -} \ No newline at end of file +} From b767fc92fa77d2c1a428a64c0a8d0418e5b75262 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 4 Sep 2024 11:35:59 +0200 Subject: [PATCH 14/64] Add cura-private, disabled by default CURA-11622 --- remotes.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/remotes.json b/remotes.json index cdfb47b..5e2e0bd 100644 --- a/remotes.json +++ b/remotes.json @@ -9,6 +9,12 @@ "name": "cura-conan2", "url": "https://cura.jfrog.io/artifactory/api/conan/cura-conan2-dev", "verify_ssl": true + }, + { + "name": "cura-private-conan2", + "url": "https://cura.jfrog.io/artifactory/api/conan/cura-private-conan2-dev", + "verify_ssl": true, + "disabled": true } ] } From d26e99dfa140cd6e6ff18867e9982106678c694e Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 11 Sep 2024 10:17:52 +0200 Subject: [PATCH 15/64] Don't force-use PowerShell on Windows for conan --- global.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/global.conf b/global.conf index 4b9863a..4b23116 100644 --- a/global.conf +++ b/global.conf @@ -4,5 +4,3 @@ tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True tools.build:skip_test = False core.version_ranges:resolve_prereleases=True - -{% if platform.system() == 'Windows' %}tools.env.virtualenv:powershell=True{% endif %} \ No newline at end of file From 9de30b205474be20ddd080360c3c3ebf26a158f8 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 13 Sep 2024 13:20:14 +0200 Subject: [PATCH 16/64] Fix missing import --- extensions/generators/GitHubActionsBuildEnv.py | 1 + extensions/generators/GitHubActionsRunEnv.py | 1 + 2 files changed, 2 insertions(+) diff --git a/extensions/generators/GitHubActionsBuildEnv.py b/extensions/generators/GitHubActionsBuildEnv.py index bd194c7..e8b05f8 100644 --- a/extensions/generators/GitHubActionsBuildEnv.py +++ b/extensions/generators/GitHubActionsBuildEnv.py @@ -4,6 +4,7 @@ from jinja2 import Template from conan import ConanFile from conan.tools.env import VirtualBuildEnv +from conan.tools.files import save class GitHubActionsBuildEnv: diff --git a/extensions/generators/GitHubActionsRunEnv.py b/extensions/generators/GitHubActionsRunEnv.py index cd50c16..ff31436 100644 --- a/extensions/generators/GitHubActionsRunEnv.py +++ b/extensions/generators/GitHubActionsRunEnv.py @@ -4,6 +4,7 @@ from jinja2 import Template from conan import ConanFile from conan.tools.env import VirtualRunEnv +from conan.tools.files import save class GitHubActionsRunEnv: From 8302c35138261eeb2a3ccc4d1475aaaf51863409 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 19 Sep 2024 12:08:08 +0200 Subject: [PATCH 17/64] Set dependencies option in profiles --- global.conf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/global.conf b/global.conf index 4b23116..1da4fd8 100644 --- a/global.conf +++ b/global.conf @@ -2,5 +2,14 @@ core:default_profile = cura.jinja core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True -tools.build:skip_test = False core.version_ranges:resolve_prereleases=True + +[options] +asio-grpc/*:local_allocator="recycling_allocator" +grpc/*:csharp_plugin=False +grpc/*:node_plugin=False +grpc/*:objective_c_plugin=False +grpc/*:php_plugin=False +grpc/*:ruby_plugin=False +grpc/*:python_plugin=False +boost/*:header_only=True From 887618cee805a4e9824d84f31ac50e13a102149d Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 19 Sep 2024 12:12:49 +0200 Subject: [PATCH 18/64] Set dependencies option in proper files --- global.conf | 10 ---------- profiles/cura.jinja | 9 +++++++++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/global.conf b/global.conf index 1da4fd8..04d10c8 100644 --- a/global.conf +++ b/global.conf @@ -3,13 +3,3 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True - -[options] -asio-grpc/*:local_allocator="recycling_allocator" -grpc/*:csharp_plugin=False -grpc/*:node_plugin=False -grpc/*:objective_c_plugin=False -grpc/*:php_plugin=False -grpc/*:ruby_plugin=False -grpc/*:python_plugin=False -boost/*:header_only=True diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 0d7faa9..ef250df 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -11,3 +11,12 @@ umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 +[options] +asio-grpc/*:local_allocator="recycling_allocator" +grpc/*:csharp_plugin=False +grpc/*:node_plugin=False +grpc/*:objective_c_plugin=False +grpc/*:php_plugin=False +grpc/*:ruby_plugin=False +grpc/*:python_plugin=False +boost/*:header_only=True From a0253e614ade8b8fb3a28470d445bea09ad8728b Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 19 Sep 2024 12:15:31 +0200 Subject: [PATCH 19/64] Fix option value --- profiles/cura.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index ef250df..217a6cb 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -12,7 +12,7 @@ dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 [options] -asio-grpc/*:local_allocator="recycling_allocator" +asio-grpc/*:local_allocator=recycling_allocator grpc/*:csharp_plugin=False grpc/*:node_plugin=False grpc/*:objective_c_plugin=False From c959ae480defe561f33eb5866c5d6ced8f052b4b Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 20 Sep 2024 15:50:27 +0200 Subject: [PATCH 20/64] Fix crash when all oses not present in requirements --- extensions/deployers/virtual_python_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index 95927f1..542a9cd 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -80,8 +80,8 @@ def deploy(graph, output_folder, **kwargs): pip_requirements = {} if conanfile.conan_data is not None and "pip_requirements" in conanfile.conan_data: - for oses in ("any", str(conanfile.settings.os)): - for name, req in conanfile.conan_data["pip_requirements"][oses].items(): + for system in (system for system in conanfile.conan_data["pip_requirements"] if system in ("any", str(conanfile.settings.os))): + for name, req in conanfile.conan_data["pip_requirements"][system].items(): if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): pip_requirements[name] = req From e03d18be7d7869c67e4764b655887d7afea8fbfa Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 20 Sep 2024 17:53:56 +0200 Subject: [PATCH 21/64] Also install dev packages when necessary --- extensions/deployers/virtual_python_env.py | 43 +++++++++++++--------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index 542a9cd..6c59cd8 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -11,6 +11,21 @@ from conan.tools.scm import Version from conan.tools.env import VirtualRunEnv +def populate_pip_requirements(key, pip_requirements, conan_data, actual_os): + if conan_data is not None and key in conan_data: + for system in (system for system in conan_data[key] if system in ("any", actual_os)): + for name, req in conan_data[key][system].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): + pip_requirements[name] = req + + +def populate_full_pip_requirements(conanfile, key, pip_requirements, actual_os): + populate_pip_requirements(key, pip_requirements, conanfile.conan_data, actual_os) + + for name, dep in reversed(conanfile.dependencies.host.items()): + populate_pip_requirements(key, pip_requirements, dep.conan_data, actual_os) + + def deploy(graph, output_folder, **kwargs): if graph.root.conanfile.name is None: conanfile: ConanFile = graph.nodes[1].conanfile @@ -78,31 +93,25 @@ def deploy(graph, output_folder, **kwargs): "activate")) save(graph.root.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) + actual_os = str(conanfile.settings.os) pip_requirements = {} - if conanfile.conan_data is not None and "pip_requirements" in conanfile.conan_data: - for system in (system for system in conanfile.conan_data["pip_requirements"] if system in ("any", str(conanfile.settings.os))): - for name, req in conanfile.conan_data["pip_requirements"][system].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): - pip_requirements[name] = req - for name, dep in reversed(graph.root.conanfile.dependencies.host.items()): - if dep.conan_data is None: - continue - if "pip_requirements" in dep.conan_data: - for oses in ("any", str(conanfile.settings.os)): - for name, req in conanfile.conan_data["pip_requirements"][oses].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version( - req["version"]): - pip_requirements[name] = req + populate_full_pip_requirements(conanfile, "pip_requirements", pip_requirements, actual_os) + + if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool): + populate_full_pip_requirements(conanfile, "pip_requirements_dev", pip_requirements, actual_os) requirements_txt = "" for name, req in pip_requirements.items(): if "url" in req: requirements_txt += f"{req['url']}" - else: + elif 'version': requirements_txt += f"{name}=={req['version']}" - for hash_str in req['hashes']: - requirements_txt += f" --hash={hash_str}" + + if 'hashes' in req: + for hash_str in req['hashes']: + requirements_txt += f" --hash={hash_str}" + requirements_txt += "\n" save(graph.root.conanfile, os.path.join(output_folder, 'conan', 'requirements.txt'), requirements_txt) From 6c5758713caa16ddcf7553ad6fa54d126a3f8b7b Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Mon, 23 Sep 2024 09:15:55 +0200 Subject: [PATCH 22/64] set build folder vars to take OS and Compiler into account See https://chat.google.com/room/AAAAiTO1PDw/Ex_B-0g2X0o/Ex_B-0g2X0o?cls=10 --- global.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/global.conf b/global.conf index 04d10c8..e4c7f9e 100644 --- a/global.conf +++ b/global.conf @@ -3,3 +3,4 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True +tools.cmake.cmake_layout:build_folder_vars="['settings.os', 'settings.compiler']" From d24b8f96776dd81b2fbb67d314b3448f72e448a1 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Mon, 23 Sep 2024 09:45:47 +0200 Subject: [PATCH 23/64] Revert "Also install dev packages when necessary" This reverts commit e03d18be7d7869c67e4764b655887d7afea8fbfa. --- extensions/deployers/virtual_python_env.py | 43 +++++++++------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index 6c59cd8..542a9cd 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -11,21 +11,6 @@ from conan.tools.scm import Version from conan.tools.env import VirtualRunEnv -def populate_pip_requirements(key, pip_requirements, conan_data, actual_os): - if conan_data is not None and key in conan_data: - for system in (system for system in conan_data[key] if system in ("any", actual_os)): - for name, req in conan_data[key][system].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): - pip_requirements[name] = req - - -def populate_full_pip_requirements(conanfile, key, pip_requirements, actual_os): - populate_pip_requirements(key, pip_requirements, conanfile.conan_data, actual_os) - - for name, dep in reversed(conanfile.dependencies.host.items()): - populate_pip_requirements(key, pip_requirements, dep.conan_data, actual_os) - - def deploy(graph, output_folder, **kwargs): if graph.root.conanfile.name is None: conanfile: ConanFile = graph.nodes[1].conanfile @@ -93,25 +78,31 @@ def deploy(graph, output_folder, **kwargs): "activate")) save(graph.root.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) - actual_os = str(conanfile.settings.os) pip_requirements = {} + if conanfile.conan_data is not None and "pip_requirements" in conanfile.conan_data: + for system in (system for system in conanfile.conan_data["pip_requirements"] if system in ("any", str(conanfile.settings.os))): + for name, req in conanfile.conan_data["pip_requirements"][system].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): + pip_requirements[name] = req - populate_full_pip_requirements(conanfile, "pip_requirements", pip_requirements, actual_os) - - if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool): - populate_full_pip_requirements(conanfile, "pip_requirements_dev", pip_requirements, actual_os) + for name, dep in reversed(graph.root.conanfile.dependencies.host.items()): + if dep.conan_data is None: + continue + if "pip_requirements" in dep.conan_data: + for oses in ("any", str(conanfile.settings.os)): + for name, req in conanfile.conan_data["pip_requirements"][oses].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version( + req["version"]): + pip_requirements[name] = req requirements_txt = "" for name, req in pip_requirements.items(): if "url" in req: requirements_txt += f"{req['url']}" - elif 'version': + else: requirements_txt += f"{name}=={req['version']}" - - if 'hashes' in req: - for hash_str in req['hashes']: - requirements_txt += f" --hash={hash_str}" - + for hash_str in req['hashes']: + requirements_txt += f" --hash={hash_str}" requirements_txt += "\n" save(graph.root.conanfile, os.path.join(output_folder, 'conan', 'requirements.txt'), requirements_txt) From c76eb1e2e54f090c4d11fc37674f93bce8e955bb Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Mon, 23 Sep 2024 10:23:49 +0200 Subject: [PATCH 24/64] Fix setting syntax --- global.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.conf b/global.conf index e4c7f9e..f122ea8 100644 --- a/global.conf +++ b/global.conf @@ -3,4 +3,4 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True -tools.cmake.cmake_layout:build_folder_vars="['settings.os', 'settings.compiler']" +tools.cmake.cmake_layout:build_folder_vars=['settings.os', 'settings.compiler'] From 4432247c9eddd4736e9d1d179729133e5979e12b Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Mon, 23 Sep 2024 11:09:52 +0200 Subject: [PATCH 25/64] Allow installing dev pip packages --- extensions/deployers/virtual_python_env.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index 542a9cd..cdb649b 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -105,7 +105,16 @@ def deploy(graph, output_folder, **kwargs): requirements_txt += f" --hash={hash_str}" requirements_txt += "\n" - save(graph.root.conanfile, os.path.join(output_folder, 'conan', 'requirements.txt'), requirements_txt) + pip_requirements_path = os.path.join(output_folder, 'conan', 'requirements.txt') + save(graph.root.conanfile, pip_requirements_path, requirements_txt) with venv_vars.apply(): - conanfile.run(f"{py_interp_venv} -m pip install -r {os.path.join(output_folder, 'conan', 'requirements.txt')}", + conanfile.run(f"{py_interp_venv} -m pip install -r {pip_requirements_path}", env="conanrun") + + if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool): + if "pip_requirements_dev" in conanfile.conan_data: + pip_requirements_dev = conanfile.conan_data["pip_requirements_dev"] + pip_requirements_dev_path = os.path.join(output_folder, 'conan', 'requirements_dev.txt') + save(graph.root.conanfile, pip_requirements_dev_path, '\n'.join(pip_requirements_dev)) + with venv_vars.apply(): + conanfile.run(f"{py_interp_venv} -m pip install -r {pip_requirements_dev_path}", env="conanrun") From 636b4bc0c699d3987f0fc76d999279f8767f759c Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 24 Sep 2024 10:42:46 +0200 Subject: [PATCH 26/64] Use shared openssl for Linux globally --- profiles/cura.jinja | 3 +++ 1 file changed, 3 insertions(+) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 217a6cb..d683638 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -20,3 +20,6 @@ grpc/*:php_plugin=False grpc/*:ruby_plugin=False grpc/*:python_plugin=False boost/*:header_only=True +{% if platform.system() == 'Linux' %} +openssl/*:shared=True +{% endif %} From f93b952354eeef3c29a01d21b626b0b10655472f Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 24 Sep 2024 10:48:29 +0200 Subject: [PATCH 27/64] Setup packages use globally --- profiles/cura.jinja | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index d683638..34a175e 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -23,3 +23,12 @@ boost/*:header_only=True {% if platform.system() == 'Linux' %} openssl/*:shared=True {% endif %} +pyarcus/*:shared=True +pysavitar/*:shared=True +pynest2d/*:shared=True +cpython/*:shared=True +{% if platform.system() == 'Windows' %} +dulcificum/*:shared=False +{% else %} +dulcificum/*:shared=True +{% endif %} From 52dad5e342aea9c3290e92443d568f9afd0cc5f2 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 24 Sep 2024 11:13:15 +0200 Subject: [PATCH 28/64] Add more global options --- profiles/cura.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 34a175e..f7b3af2 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -32,3 +32,4 @@ dulcificum/*:shared=False {% else %} dulcificum/*:shared=True {% endif %} +clipper/*:shared=True From 1cbc3ddf45e7d5a07edf4d436154327535e0e7b1 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 24 Sep 2024 16:06:29 +0200 Subject: [PATCH 29/64] Add ability to install pip requirements by url --- extensions/deployers/virtual_python_env.py | 73 ++++++++++++---------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index cdb649b..9678e50 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -11,6 +11,29 @@ from conan.tools.scm import Version from conan.tools.env import VirtualRunEnv +def populate_pip_requirements(key, pip_requirements, conan_data, actual_os): + if conan_data is not None and key in conan_data: + for system in (system for system in conan_data[key] if system in ("any", actual_os)): + for name, req in conan_data[key][system].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): + pip_requirements[name] = req + + +def populate_full_pip_requirements(conanfile, key, pip_requirements, actual_os): + populate_pip_requirements(key, pip_requirements, conanfile.conan_data, actual_os) + + for name, dep in reversed(conanfile.dependencies.host.items()): + populate_pip_requirements(key, pip_requirements, dep.conan_data, actual_os) + + +def install_pip_requirements(file_suffix, file_content, output_folder, conanfile, venv_vars, py_interp_venv): + if len(file_content) > 0: + pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt') + save(conanfile, pip_file_path, "\n".join(file_content)) + with venv_vars.apply(): + conanfile.run(f"{py_interp_venv} -m pip install -r {pip_file_path}", env="conanrun") + + def deploy(graph, output_folder, **kwargs): if graph.root.conanfile.name is None: conanfile: ConanFile = graph.nodes[1].conanfile @@ -79,42 +102,24 @@ def deploy(graph, output_folder, **kwargs): save(graph.root.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) pip_requirements = {} - if conanfile.conan_data is not None and "pip_requirements" in conanfile.conan_data: - for system in (system for system in conanfile.conan_data["pip_requirements"] if system in ("any", str(conanfile.settings.os))): - for name, req in conanfile.conan_data["pip_requirements"][system].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): - pip_requirements[name] = req + populate_full_pip_requirements(conanfile, "pip_requirements", pip_requirements, str(conanfile.settings.os)) - for name, dep in reversed(graph.root.conanfile.dependencies.host.items()): - if dep.conan_data is None: - continue - if "pip_requirements" in dep.conan_data: - for oses in ("any", str(conanfile.settings.os)): - for name, req in conanfile.conan_data["pip_requirements"][oses].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version( - req["version"]): - pip_requirements[name] = req - - requirements_txt = "" + requirements_hashed_txt = [] + requirements_url_txt = [] for name, req in pip_requirements.items(): if "url" in req: - requirements_txt += f"{req['url']}" + requirements_url_txt.append(req['url']) else: - requirements_txt += f"{name}=={req['version']}" - for hash_str in req['hashes']: - requirements_txt += f" --hash={hash_str}" - requirements_txt += "\n" + requirement_txt = [f"{name}=={req['version']}"] - pip_requirements_path = os.path.join(output_folder, 'conan', 'requirements.txt') - save(graph.root.conanfile, pip_requirements_path, requirements_txt) - with venv_vars.apply(): - conanfile.run(f"{py_interp_venv} -m pip install -r {pip_requirements_path}", - env="conanrun") - - if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool): - if "pip_requirements_dev" in conanfile.conan_data: - pip_requirements_dev = conanfile.conan_data["pip_requirements_dev"] - pip_requirements_dev_path = os.path.join(output_folder, 'conan', 'requirements_dev.txt') - save(graph.root.conanfile, pip_requirements_dev_path, '\n'.join(pip_requirements_dev)) - with venv_vars.apply(): - conanfile.run(f"{py_interp_venv} -m pip install -r {pip_requirements_dev_path}", env="conanrun") + if "hashes" in req: + for hash_str in req['hashes']: + requirement_txt.append(f"--hash={hash_str}") + + requirements_hashed_txt.append(" ".join(requirement_txt)) + + install_pip_requirements("hashed", requirements_hashed_txt, output_folder, conanfile, venv_vars, py_interp_venv) + install_pip_requirements("url", requirements_url_txt, output_folder, conanfile, venv_vars, py_interp_venv) + + if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool) and conanfile.conan_data is not None and "pip_requirements_dev" in conanfile.conan_data: + install_pip_requirements("dev", conanfile.conan_data["pip_requirements_dev"], output_folder, conanfile, venv_vars, py_interp_venv) From e71297c0a3c056e02903b619a11c917c7786e325 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 25 Sep 2024 16:52:05 +0200 Subject: [PATCH 30/64] Finish implementation of new PyCharm generator --- extensions/deployers/virtual_python_env.py | 3 +++ extensions/generators/PyCharmRunEnv.py | 22 +++++++++------------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py index 9678e50..78d72f5 100644 --- a/extensions/deployers/virtual_python_env.py +++ b/extensions/deployers/virtual_python_env.py @@ -39,8 +39,11 @@ def deploy(graph, output_folder, **kwargs): conanfile: ConanFile = graph.nodes[1].conanfile else: conanfile: ConanFile = graph.root.conanfile + if output_folder is None: output_folder = "venv" + else: + output_folder = str(Path(output_folder, "venv")) bin_venv_path = "Scripts" if conanfile.settings.os == "Windows" else "bin" diff --git a/extensions/generators/PyCharmRunEnv.py b/extensions/generators/PyCharmRunEnv.py index c80f398..3e32a71 100644 --- a/extensions/generators/PyCharmRunEnv.py +++ b/extensions/generators/PyCharmRunEnv.py @@ -1,9 +1,10 @@ from pathlib import Path from typing import Dict +from jinja2 import Template from conan import ConanFile from conan.tools.scm import Version -from jinja2 import Template +from conan.tools.files import save from conan.tools.env.virtualrunenv import VirtualRunEnv @@ -28,17 +29,13 @@ class PyCharmRunEnv: def _site_packages(self): if self.settings.os == "Windows": return self._base_dir.joinpath("Lib", "site-packages") - py_version = Version(self.dependencies["cpython"].ref.version) + py_version = Version(self.conanfile.dependencies["cpython"].ref.version) return self._base_dir.joinpath("lib", f"python{py_version.major}.{py_version.minor}", "site-packages") - def generate(self) -> Dict[str, str]: - # Mapping of file names -> files - # Mapping of file names -> files - run_configurations: Dict[str, str] = {} - - if not hasattr(self.conanfile, "_pycharm_targets"): + def generate(self) -> None: + if self.conanfile.conan_data is None or "pycharm_targets" not in self.conanfile.conan_data: # There are no _pycharm_targets in the conanfile for the package using this generator. - return run_configurations + return # Collect environment variables for use in the template env = VirtualRunEnv(self.conanfile).environment() @@ -50,14 +47,13 @@ class PyCharmRunEnv: env.compose_env(project_run_env) # TODO: Add logic for dependencies # Create Pycharm run configuration from template for each target - for target in self.conanfile._pycharm_targets: + for target in self.conanfile.conan_data["pycharm_targets"]: target["env_vars"] = env.vars(self.conanfile, scope="run") target["sdk_path"] = str(self._py_interp) if "parameters" not in target: target["parameters"] = "" - with open(target["jinja_path"], "r") as f: + with open(Path(self.conanfile.source_folder, target["jinja_path"]), "r") as f: template = Template(f.read()) run_configuration = template.render(target) - run_configurations[str(Path(self.conanfile.source_folder).joinpath(".run", - f"{target['name']}.run.xml"))] = run_configuration + save(self.conanfile, Path(".run", f"{target['name']}.run.xml"), run_configuration) From 0fb89b8a61e71f7a426a67065d2cd258fb084c0b Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 26 Sep 2024 16:00:28 +0200 Subject: [PATCH 31/64] Disable Python options We absolutely don't use tk and ncurses inside Python, so just disable them so that we don't have to ship them --- profiles/cura.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index f7b3af2..623c4ad 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -27,6 +27,8 @@ pyarcus/*:shared=True pysavitar/*:shared=True pynest2d/*:shared=True cpython/*:shared=True +cpython/*:with_tkinter=False +cpython/*:with_curses=False {% if platform.system() == 'Windows' %} dulcificum/*:shared=False {% else %} From 2effe7c929f053b1771c305cc01e8934a680deff Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Mon, 18 Nov 2024 13:44:05 +0100 Subject: [PATCH 32/64] Restore VirtualPythonEnv being a generator --- extensions/deployers/virtual_python_env.py | 128 ----------------- extensions/generators/EnvScriptBuilder.py | 37 +++++ .../generators/GitHubActionsBuildEnv.py | 22 +-- extensions/generators/GitHubActionsRunEnv.py | 24 +--- extensions/generators/VirtualPythonEnv.py | 129 ++++++++++++++++++ 5 files changed, 179 insertions(+), 161 deletions(-) delete mode 100644 extensions/deployers/virtual_python_env.py create mode 100644 extensions/generators/EnvScriptBuilder.py create mode 100644 extensions/generators/VirtualPythonEnv.py diff --git a/extensions/deployers/virtual_python_env.py b/extensions/deployers/virtual_python_env.py deleted file mode 100644 index 78d72f5..0000000 --- a/extensions/deployers/virtual_python_env.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import sys -from io import StringIO -from shutil import which -from pathlib import Path - -from conan import ConanFile -from conan.errors import ConanException -from conan.tools.files import copy, save, load -from conan.tools.scm import Version -from conan.tools.env import VirtualRunEnv - - -def populate_pip_requirements(key, pip_requirements, conan_data, actual_os): - if conan_data is not None and key in conan_data: - for system in (system for system in conan_data[key] if system in ("any", actual_os)): - for name, req in conan_data[key][system].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): - pip_requirements[name] = req - - -def populate_full_pip_requirements(conanfile, key, pip_requirements, actual_os): - populate_pip_requirements(key, pip_requirements, conanfile.conan_data, actual_os) - - for name, dep in reversed(conanfile.dependencies.host.items()): - populate_pip_requirements(key, pip_requirements, dep.conan_data, actual_os) - - -def install_pip_requirements(file_suffix, file_content, output_folder, conanfile, venv_vars, py_interp_venv): - if len(file_content) > 0: - pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt') - save(conanfile, pip_file_path, "\n".join(file_content)) - with venv_vars.apply(): - conanfile.run(f"{py_interp_venv} -m pip install -r {pip_file_path}", env="conanrun") - - -def deploy(graph, output_folder, **kwargs): - if graph.root.conanfile.name is None: - conanfile: ConanFile = graph.nodes[1].conanfile - else: - conanfile: ConanFile = graph.root.conanfile - - if output_folder is None: - output_folder = "venv" - else: - output_folder = str(Path(output_folder, "venv")) - - bin_venv_path = "Scripts" if conanfile.settings.os == "Windows" else "bin" - - # Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter - try: - cpython = conanfile.dependencies["cpython"] - py_interp = cpython.conf_info.get("user.cpython:python").replace("\\", "/") - except KeyError: - py_interp = sys.executable - - vr = VirtualRunEnv(conanfile) - env = vr.environment() - sys_vars = env.vars(conanfile, scope="run") - - conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'") - with sys_vars.apply(): - conanfile.run(f"""{py_interp} -m venv --copies {output_folder}""", env="conanrun", scope="run") - - # Make sure there executable is named the same on all three OSes this allows it to be called with `python` - # simplifying GH Actions steps - if conanfile.settings.os != "Windows": - py_interp_venv = Path(output_folder, bin_venv_path, "python") - if not py_interp_venv.exists(): - py_interp_venv.hardlink_to( - Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) - else: - py_interp_venv = Path(output_folder, bin_venv_path, - Path(sys.executable).stem + Path(sys.executable).suffix) - - buffer = StringIO() - outer = '"' if conanfile.settings.os == "Windows" else "'" - inner = "'" if conanfile.settings.os == "Windows" else '"' - with sys_vars.apply(): - conanfile.run( - f"""{py_interp_venv} -c {outer}import sysconfig; print(sysconfig.get_path({inner}purelib{inner})){outer}""", - env="conanrun", - stdout=buffer) - pythonpath = buffer.getvalue().splitlines()[-1] - - env.define_path("VIRTUAL_ENV", output_folder) - env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("PYTHONPATH", pythonpath) - env.unset("PYTHONHOME") - venv_vars = env.vars(graph.root.conanfile, scope="run") - venv_vars.save_script("virtual_python_env") - - # Install some base_packages - with venv_vars.apply(): - conanfile.run(f"""{py_interp_venv} -m pip install wheel setuptools""", env="conanrun") - - if conanfile.settings.os != "Windows": - content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(graph.root.conanfile, - os.path.join( - output_folder, - bin_venv_path, - "activate")) - save(graph.root.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) - - pip_requirements = {} - populate_full_pip_requirements(conanfile, "pip_requirements", pip_requirements, str(conanfile.settings.os)) - - requirements_hashed_txt = [] - requirements_url_txt = [] - for name, req in pip_requirements.items(): - if "url" in req: - requirements_url_txt.append(req['url']) - else: - requirement_txt = [f"{name}=={req['version']}"] - - if "hashes" in req: - for hash_str in req['hashes']: - requirement_txt.append(f"--hash={hash_str}") - - requirements_hashed_txt.append(" ".join(requirement_txt)) - - install_pip_requirements("hashed", requirements_hashed_txt, output_folder, conanfile, venv_vars, py_interp_venv) - install_pip_requirements("url", requirements_url_txt, output_folder, conanfile, venv_vars, py_interp_venv) - - if conanfile.conf.get("user.deployer.virtual_python_env:dev_tools", default = False, check_type = bool) and conanfile.conan_data is not None and "pip_requirements_dev" in conanfile.conan_data: - install_pip_requirements("dev", conanfile.conan_data["pip_requirements_dev"], output_folder, conanfile, venv_vars, py_interp_venv) diff --git a/extensions/generators/EnvScriptBuilder.py b/extensions/generators/EnvScriptBuilder.py new file mode 100644 index 0000000..1a225ee --- /dev/null +++ b/extensions/generators/EnvScriptBuilder.py @@ -0,0 +1,37 @@ +from conan.tools.files import save + +class EnvScriptBuilder: + def __init__(self): + self._variables = {} + + def set_variable(self, name: str, value: str): + self._variables[name] = value + + def set_environment(self, env): + for name, value in env.items(): + self.set_variable(name, value) + + def save(self, path, conanfile, append_to=None) -> None: + file_path = path + + content = "" + for name, value in self._variables.items(): + set_variable = f'{name}={value}' + + if append_to is not None: + set_variable = f"echo {set_variable} >> {append_to}" + else: + set_variable = f"export {set_variable}" + + content += f"{set_variable}\n" + + if conanfile.settings.get_safe("os") == "Windows": + if conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): + file_path += ".ps1" + else: + file_path += ".bat" + else: + file_path += ".sh" + + conanfile.output.info(f"Saving environment script to {file_path}") + save(conanfile, file_path, content) diff --git a/extensions/generators/GitHubActionsBuildEnv.py b/extensions/generators/GitHubActionsBuildEnv.py index e8b05f8..6beb39a 100644 --- a/extensions/generators/GitHubActionsBuildEnv.py +++ b/extensions/generators/GitHubActionsBuildEnv.py @@ -1,32 +1,22 @@ from pathlib import Path -from jinja2 import Template - from conan import ConanFile from conan.tools.env import VirtualBuildEnv -from conan.tools.files import save + +from EnvScriptBuilder import EnvScriptBuilder class GitHubActionsBuildEnv: def __init__(self, conanfile: ConanFile): self.conanfile: ConanFile = conanfile - self.settings = self.conanfile.settings def generate(self): - template = Template( - """{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") build_env = VirtualBuildEnv(self.conanfile) env = build_env.environment() envvars = env.vars(self.conanfile, scope="build") env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - content = template.render(envvars=envvars, env_prefix=env_prefix) - filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_buildenv")) - if self.conanfile.settings.get_safe("os") == "Windows": - if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): - filepath += ".ps1" - else: - filepath += ".bat" - else: - filepath += ".sh" - save(self.conanfile, filepath, content) + + script_builder = EnvScriptBuilder() + script_builder.set_environment(envvars) + script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV") diff --git a/extensions/generators/GitHubActionsRunEnv.py b/extensions/generators/GitHubActionsRunEnv.py index ff31436..8fa9c28 100644 --- a/extensions/generators/GitHubActionsRunEnv.py +++ b/extensions/generators/GitHubActionsRunEnv.py @@ -1,32 +1,22 @@ from pathlib import Path -from jinja2 import Template - from conan import ConanFile from conan.tools.env import VirtualRunEnv -from conan.tools.files import save + +from EnvScriptBuilder import EnvScriptBuilder class GitHubActionsRunEnv: def __init__(self, conanfile: ConanFile): self.conanfile: ConanFile = conanfile - self.settings = self.conanfile.settings def generate(self): - template = Template( - """{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""") - build_env = VirtualRunEnv(self.conanfile) - env = build_env.environment() + run_env = VirtualRunEnv(self.conanfile) + env = run_env.environment() envvars = env.vars(self.conanfile, scope="run") env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - content = template.render(envvars=envvars, env_prefix=env_prefix) filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_runenv")) - if self.conanfile.settings.get_safe("os") == "Windows": - if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): - filepath += ".ps1" - else: - filepath += ".bat" - else: - filepath += ".sh" - save(self.conanfile, filepath, content) + script_builder = EnvScriptBuilder() + script_builder.set_environment(envvars) + script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV") diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py new file mode 100644 index 0000000..60cd661 --- /dev/null +++ b/extensions/generators/VirtualPythonEnv.py @@ -0,0 +1,129 @@ +import os +import sys +from io import StringIO +from shutil import which +from pathlib import Path + +from conan import ConanFile +from conan.errors import ConanException +from conan.tools.files import copy, save, load +from conan.tools.scm import Version +from conan.tools.env import VirtualRunEnv +import subprocess + + +class VirtualPythonEnv: + def __init__(self, conanfile: ConanFile): + self.conanfile: ConanFile = conanfile + + def generate(self) -> None: + output_folder = "venv" + bin_venv_path = "Scripts" if self.conanfile.settings.os == "Windows" else "bin" + + # Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter + try: + cpython = self.conanfile.dependencies["cpython"] + py_interp = cpython.conf_info.get("user.cpython:python").replace("\\", "/") + except KeyError: + py_interp = sys.executable + + run_env = VirtualRunEnv(self.conanfile) + env = run_env.environment() + env_vars = env.vars(self.conanfile, scope="run") + + self.conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'") + with env_vars.apply(): + subprocess.run([py_interp, "-m", "venv", "--copies", output_folder]) + + # Make sure there executable is named the same on all three OSes this allows it to be called with `python` + # simplifying GH Actions steps + if self.conanfile.settings.os != "Windows": + py_interp_venv = Path(output_folder, bin_venv_path, "python") + if not py_interp_venv.exists(): + py_interp_venv.hardlink_to( + Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) + else: + py_interp_venv = Path(output_folder, bin_venv_path, + Path(sys.executable).stem + Path(sys.executable).suffix) + + with env_vars.apply(): + buffer = subprocess.run([py_interp_venv, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], capture_output=True, encoding="utf-8").stdout + pythonpath = buffer.splitlines()[-1] + + env.define_path("VIRTUAL_ENV", output_folder) + env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) + env.prepend_path("PYTHONPATH", pythonpath) + env.unset("PYTHONHOME") + + filepath = str(Path(self.conanfile.generators_folder).joinpath("supercoucou_runenv")) + env_vars.save_script(filepath) + + # Install some base_packages + with env_vars.apply(): + subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"]) + + if self.conanfile.settings.os != "Windows": + content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(self.conanfile, + os.path.join( + output_folder, + bin_venv_path, + "activate")) + save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) + + pip_requirements = {} + self._populate_pip_requirements(self.conanfile, "pip_requirements", pip_requirements, str(self.conanfile.settings.os)) + + requirements_hashed_txt = [] + requirements_url_txt = [] + for name, req in pip_requirements.items(): + if "url" in req: + requirements_url_txt.append(req['url']) + else: + requirement_txt = [f"{name}=={req['version']}"] + + if "hashes" in req: + for hash_str in req['hashes']: + requirement_txt.append(f"--hash={hash_str}") + + requirements_hashed_txt.append(" ".join(requirement_txt)) + + self._install_pip_requirements("hashed", requirements_hashed_txt, output_folder, env_vars, py_interp_venv) + self._install_pip_requirements("url", requirements_url_txt, output_folder, env_vars, py_interp_venv) + + if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default = False, check_type = bool): + pip_requirements_dev = [] + self._populate_pip_requirements_dev(self.conanfile, pip_requirements_dev) + print(pip_requirements_dev) + self._install_pip_requirements("dev", pip_requirements_dev, output_folder, env_vars, py_interp_venv) + + + def _populate_pip_requirements_dev(self, conanfile, pip_requirements_dev, add_dependencies = True): + if hasattr(conanfile, "conan_data") and "pip_requirements_dev" in conanfile.conan_data: + print(conanfile.conan_data["pip_requirements_dev"]) + pip_requirements_dev += conanfile.conan_data["pip_requirements_dev"] + + if add_dependencies: + for name, dep in reversed(self.conanfile.dependencies.host.items()): + self._populate_pip_requirements_dev(dep, pip_requirements_dev, add_dependencies = False) + + + def _populate_pip_requirements(self, conanfile, key, pip_requirements, actual_os, add_dependencies = True): + if hasattr(conanfile, "conan_data") and key in conanfile.conan_data: + for system in (system for system in conanfile.conan_data[key] if system in ("any", actual_os)): + for name, req in conanfile.conan_data[key][system].items(): + if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): + pip_requirements[name] = req + + if add_dependencies: + for name, dep in reversed(self.conanfile.dependencies.host.items()): + self._populate_pip_requirements(dep, key, pip_requirements, actual_os, add_dependencies = False) + + + def _install_pip_requirements(self, file_suffix, file_content, output_folder, env_vars, py_interp_venv): + if len(file_content) > 0: + pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt') + save(self.conanfile, pip_file_path, "\n".join(file_content)) + with env_vars.apply(): + subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", pip_file_path]) From d600c11d574d433f5de2392d8142dc8131b20e4b Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Mon, 18 Nov 2024 13:56:50 +0100 Subject: [PATCH 33/64] Fix generated script name --- extensions/generators/VirtualPythonEnv.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index 60cd661..617c24f 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -56,9 +56,7 @@ class VirtualPythonEnv: env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) env.prepend_path("PYTHONPATH", pythonpath) env.unset("PYTHONHOME") - - filepath = str(Path(self.conanfile.generators_folder).joinpath("supercoucou_runenv")) - env_vars.save_script(filepath) + env_vars.save_script("virtual_python_env") # Install some base_packages with env_vars.apply(): @@ -95,13 +93,11 @@ class VirtualPythonEnv: if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default = False, check_type = bool): pip_requirements_dev = [] self._populate_pip_requirements_dev(self.conanfile, pip_requirements_dev) - print(pip_requirements_dev) self._install_pip_requirements("dev", pip_requirements_dev, output_folder, env_vars, py_interp_venv) def _populate_pip_requirements_dev(self, conanfile, pip_requirements_dev, add_dependencies = True): if hasattr(conanfile, "conan_data") and "pip_requirements_dev" in conanfile.conan_data: - print(conanfile.conan_data["pip_requirements_dev"]) pip_requirements_dev += conanfile.conan_data["pip_requirements_dev"] if add_dependencies: From 083646ed6131fc726d66e0a5ec004084a1735257 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 19 Nov 2024 14:56:29 +0100 Subject: [PATCH 34/64] Add setting to specify the venv folder --- extensions/generators/VirtualPythonEnv.py | 44 +++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index 617c24f..bc59d2e 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -17,6 +17,10 @@ class VirtualPythonEnv: self.conanfile: ConanFile = conanfile def generate(self) -> None: + ''' + Creates a Python venv using the CPython installed by conan, then create a script so that this venv can be easily used + in Conan commands, and finally install the pip dependencies declared in the conanfile data + ''' output_folder = "venv" bin_venv_path = "Scripts" if self.conanfile.settings.os == "Windows" else "bin" @@ -31,6 +35,9 @@ class VirtualPythonEnv: env = run_env.environment() env_vars = env.vars(self.conanfile, scope="run") + base_folder = self.conanfile.conf.get("user.generator.virtual_python_env:base_folder", default = "", check_type = str) + output_folder = os.path.join(base_folder if len(base_folder) > 0 else os.getcwd(), output_folder) + self.conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'") with env_vars.apply(): subprocess.run([py_interp, "-m", "venv", "--copies", output_folder]) @@ -46,6 +53,7 @@ class VirtualPythonEnv: py_interp_venv = Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix) + # Generate a script that mimics the venv activate script but is callable easily in Conan commands with env_vars.apply(): buffer = subprocess.run([py_interp_venv, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], capture_output=True, encoding="utf-8").stdout pythonpath = buffer.splitlines()[-1] @@ -62,13 +70,13 @@ class VirtualPythonEnv: with env_vars.apply(): subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"]) - if self.conanfile.settings.os != "Windows": - content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(self.conanfile, - os.path.join( - output_folder, - bin_venv_path, - "activate")) - save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) + # if self.conanfile.settings.os != "Windows": + # content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(self.conanfile, + # os.path.join( + # output_folder, + # bin_venv_path, + # "activate")) + # save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) pip_requirements = {} self._populate_pip_requirements(self.conanfile, "pip_requirements", pip_requirements, str(self.conanfile.settings.os)) @@ -91,18 +99,26 @@ class VirtualPythonEnv: self._install_pip_requirements("url", requirements_url_txt, output_folder, env_vars, py_interp_venv) if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default = False, check_type = bool): - pip_requirements_dev = [] - self._populate_pip_requirements_dev(self.conanfile, pip_requirements_dev) - self._install_pip_requirements("dev", pip_requirements_dev, output_folder, env_vars, py_interp_venv) + self._populate_and_install_pip_requirements_list("dev", output_folder, env_vars, py_interp_venv) + + if self.conanfile.conf.get("user.generator.virtual_python_env:installer_tools", default = False, check_type = bool): + self._populate_and_install_pip_requirements_list("installer", output_folder, env_vars, py_interp_venv) + + + def _populate_and_install_pip_requirements_list(self, suffix, output_folder, env_vars, py_interp_venv): + pip_requirements_list = [] + self._populate_pip_requirements_list(self.conanfile, pip_requirements_list, suffix) + self._install_pip_requirements(suffix, pip_requirements_list, output_folder, env_vars, py_interp_venv) - def _populate_pip_requirements_dev(self, conanfile, pip_requirements_dev, add_dependencies = True): - if hasattr(conanfile, "conan_data") and "pip_requirements_dev" in conanfile.conan_data: - pip_requirements_dev += conanfile.conan_data["pip_requirements_dev"] + def _populate_pip_requirements_list(self, conanfile, pip_requirements_list, suffix, add_dependencies = True): + attribute_name = f"pip_requirements_{suffix}" + if hasattr(conanfile, "conan_data") and attribute_name in conanfile.conan_data: + pip_requirements_list += conanfile.conan_data[attribute_name] if add_dependencies: for name, dep in reversed(self.conanfile.dependencies.host.items()): - self._populate_pip_requirements_dev(dep, pip_requirements_dev, add_dependencies = False) + self._populate_pip_requirements_list(dep, pip_requirements_list, suffix, add_dependencies = False) def _populate_pip_requirements(self, conanfile, key, pip_requirements, actual_os, add_dependencies = True): From 67a32082522ee4e95fb5963551ca39d614f9da51 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 20 Nov 2024 13:28:04 +0100 Subject: [PATCH 35/64] Always generate all requirements files --- extensions/generators/VirtualPythonEnv.py | 139 +++++++++++++--------- 1 file changed, 81 insertions(+), 58 deletions(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index bc59d2e..5ed7640 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -21,7 +21,7 @@ class VirtualPythonEnv: Creates a Python venv using the CPython installed by conan, then create a script so that this venv can be easily used in Conan commands, and finally install the pip dependencies declared in the conanfile data ''' - output_folder = "venv" + venv_name = f"{self.conanfile.name}_venv" bin_venv_path = "Scripts" if self.conanfile.settings.os == "Windows" else "bin" # Check if CPython is added as a dependency use the Conan recipe if available; if not use system interpreter @@ -35,22 +35,21 @@ class VirtualPythonEnv: env = run_env.environment() env_vars = env.vars(self.conanfile, scope="run") - base_folder = self.conanfile.conf.get("user.generator.virtual_python_env:base_folder", default = "", check_type = str) - output_folder = os.path.join(base_folder if len(base_folder) > 0 else os.getcwd(), output_folder) + venv_folder = os.path.abspath(venv_name) - self.conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{output_folder}'") + self.conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{venv_folder}'") with env_vars.apply(): - subprocess.run([py_interp, "-m", "venv", "--copies", output_folder]) + subprocess.run([py_interp, "-m", "venv", "--copies", venv_folder]) # Make sure there executable is named the same on all three OSes this allows it to be called with `python` # simplifying GH Actions steps if self.conanfile.settings.os != "Windows": - py_interp_venv = Path(output_folder, bin_venv_path, "python") + py_interp_venv = Path(venv_folder, bin_venv_path, "python") if not py_interp_venv.exists(): py_interp_venv.hardlink_to( - Path(output_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) + Path(venv_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)) else: - py_interp_venv = Path(output_folder, bin_venv_path, + py_interp_venv = Path(venv_folder, bin_venv_path, Path(sys.executable).stem + Path(sys.executable).suffix) # Generate a script that mimics the venv activate script but is callable easily in Conan commands @@ -58,15 +57,15 @@ class VirtualPythonEnv: buffer = subprocess.run([py_interp_venv, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], capture_output=True, encoding="utf-8").stdout pythonpath = buffer.splitlines()[-1] - env.define_path("VIRTUAL_ENV", output_folder) - env.prepend_path("PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("LD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) - env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(output_folder, bin_venv_path)) + env.define_path("VIRTUAL_ENV", venv_folder) + env.prepend_path("PATH", os.path.join(venv_folder, bin_venv_path)) + env.prepend_path("LD_LIBRARY_PATH", os.path.join(venv_folder, bin_venv_path)) + env.prepend_path("DYLD_LIBRARY_PATH", os.path.join(venv_folder, bin_venv_path)) env.prepend_path("PYTHONPATH", pythonpath) env.unset("PYTHONHOME") env_vars.save_script("virtual_python_env") - # Install some base_packages + # Install some base packages with env_vars.apply(): subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"]) @@ -78,64 +77,88 @@ class VirtualPythonEnv: # "activate")) # save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) - pip_requirements = {} - self._populate_pip_requirements(self.conanfile, "pip_requirements", pip_requirements, str(self.conanfile.settings.os)) + requirements_base = self._make_pip_requirements_files() + requirements_dev = self._make_pip_requirements_files("dev") + requirements_installer = self._make_pip_requirements_files("installer") - requirements_hashed_txt = [] - requirements_url_txt = [] - for name, req in pip_requirements.items(): - if "url" in req: - requirements_url_txt.append(req['url']) - else: - requirement_txt = [f"{name}=={req['version']}"] + self._install_pip_requirements(requirements_base, env_vars, py_interp_venv) + + if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default=False, check_type=bool): + self._install_pip_requirements(requirements_dev, env_vars, py_interp_venv) + + if self.conanfile.conf.get("user.generator.virtual_python_env:installer_tools", default=False, + check_type=bool): + self._install_pip_requirements(requirements_installer, env_vars, py_interp_venv) + + def _install_pip_requirements(self, files_paths, env_vars, py_interp_venv): + with env_vars.apply(): + for file_path in files_paths: + self.conanfile.output.info(f"Installing pip requirements from {file_path}") + subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", file_path]) - if "hashes" in req: - for hash_str in req['hashes']: - requirement_txt.append(f"--hash={hash_str}") - requirements_hashed_txt.append(" ".join(requirement_txt)) + def _make_pip_requirements_files(self, suffix = None): + actual_os = str(self.conanfile.settings.os) - self._install_pip_requirements("hashed", requirements_hashed_txt, output_folder, env_vars, py_interp_venv) - self._install_pip_requirements("url", requirements_url_txt, output_folder, env_vars, py_interp_venv) + pip_requirements = VirtualPythonEnv._populate_pip_requirements(self.conanfile, suffix, actual_os) - if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default = False, check_type = bool): - self._populate_and_install_pip_requirements_list("dev", output_folder, env_vars, py_interp_venv) + for _, dependency in reversed(self.conanfile.dependencies.host.items()): + pip_requirements |= VirtualPythonEnv._populate_pip_requirements(dependency, suffix, actual_os) - if self.conanfile.conf.get("user.generator.virtual_python_env:installer_tools", default = False, check_type = bool): - self._populate_and_install_pip_requirements_list("installer", output_folder, env_vars, py_interp_venv) + # We need to make separate files because pip accepts either files containing hashes for all or none of the packages + requirements_basic_txt = [] + requirements_hashes_txt = [] + for package_name, package_desc in pip_requirements.items(): + package_requirement = package_name + (f"=={package_desc['version']}" if "version" in package_desc else "") - def _populate_and_install_pip_requirements_list(self, suffix, output_folder, env_vars, py_interp_venv): - pip_requirements_list = [] - self._populate_pip_requirements_list(self.conanfile, pip_requirements_list, suffix) - self._install_pip_requirements(suffix, pip_requirements_list, output_folder, env_vars, py_interp_venv) + if "hashes" in package_desc: + package_requirement_with_hashes = [package_requirement] + for hash_str in package_desc['hashes']: + package_requirement_with_hashes.append(f"--hash={hash_str}") + requirements_hashes_txt.append(" ".join(package_requirement_with_hashes)) + if "url" in package_desc: + requirements_basic_txt.append(package_desc['url']) + else: + requirements_basic_txt.append(package_requirement) + + generated_files = [] + self._make_pip_requirements_file(requirements_basic_txt, "basic", suffix, generated_files) + self._make_pip_requirements_file(requirements_hashes_txt, "hashes", suffix, generated_files) + + return generated_files - def _populate_pip_requirements_list(self, conanfile, pip_requirements_list, suffix, add_dependencies = True): - attribute_name = f"pip_requirements_{suffix}" - if hasattr(conanfile, "conan_data") and attribute_name in conanfile.conan_data: - pip_requirements_list += conanfile.conan_data[attribute_name] - if add_dependencies: - for name, dep in reversed(self.conanfile.dependencies.host.items()): - self._populate_pip_requirements_list(dep, pip_requirements_list, suffix, add_dependencies = False) + def _make_pip_requirements_file(self, requirements_txt, requirements_type, suffix, generated_files): + if len(requirements_txt) > 0: + file_suffixes = [file_suffix for file_suffix in [suffix, requirements_type] if file_suffix is not None] + file_basename = "_".join(["pip", "requirements"] + file_suffixes) + file_path = os.path.abspath(f"{file_basename}.txt") + self.conanfile.output.info(f"Generating pip requirements file at '{file_path}'") + save(self.conanfile, file_path, "\n".join(requirements_txt)) + generated_files.append(file_path) + + + @staticmethod + def _populate_pip_requirements(conanfile, suffix, actual_os): + pip_requirements = {} + data_key = "pip_requirements" + (f"_{suffix}" if suffix is not None else "") + if hasattr(conanfile, "conan_data") and data_key in conanfile.conan_data: + pip_requirements_data = conanfile.conan_data[data_key] + for system in (system for system in pip_requirements_data if system in ("any_os", actual_os)): + for package_name, package_desc in pip_requirements_data[system].items(): - def _populate_pip_requirements(self, conanfile, key, pip_requirements, actual_os, add_dependencies = True): - if hasattr(conanfile, "conan_data") and key in conanfile.conan_data: - for system in (system for system in conanfile.conan_data[key] if system in ("any", actual_os)): - for name, req in conanfile.conan_data[key][system].items(): - if name not in pip_requirements or Version(pip_requirements[name]["version"]) < Version(req["version"]): - pip_requirements[name] = req + try: + actual_package_version = Version(pip_requirements[package_name]["version"]) + except KeyError: + actual_package_version = None - if add_dependencies: - for name, dep in reversed(self.conanfile.dependencies.host.items()): - self._populate_pip_requirements(dep, key, pip_requirements, actual_os, add_dependencies = False) + new_package_version = Version(package_desc["version"]) if "version" in package_desc else None + if (actual_package_version is None or + (actual_package_version is not None and new_package_version is not None and new_package_version > actual_package_version)): + pip_requirements[package_name] = package_desc - def _install_pip_requirements(self, file_suffix, file_content, output_folder, env_vars, py_interp_venv): - if len(file_content) > 0: - pip_file_path = os.path.join(output_folder, 'conan', f'requirements_{file_suffix}.txt') - save(self.conanfile, pip_file_path, "\n".join(file_content)) - with env_vars.apply(): - subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", pip_file_path]) + return pip_requirements From 4cbd551c219dc36d2ab8326479043caf8de916a6 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 21 Nov 2024 08:40:09 +0100 Subject: [PATCH 36/64] Use 'core' for base pip dependencies --- extensions/generators/VirtualPythonEnv.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index 5ed7640..308817d 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -77,11 +77,11 @@ class VirtualPythonEnv: # "activate")) # save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) - requirements_base = self._make_pip_requirements_files() + requirements_core = self._make_pip_requirements_files("core") requirements_dev = self._make_pip_requirements_files("dev") requirements_installer = self._make_pip_requirements_files("installer") - self._install_pip_requirements(requirements_base, env_vars, py_interp_venv) + self._install_pip_requirements(requirements_core, env_vars, py_interp_venv) if self.conanfile.conf.get("user.generator.virtual_python_env:dev_tools", default=False, check_type=bool): self._install_pip_requirements(requirements_dev, env_vars, py_interp_venv) @@ -97,7 +97,7 @@ class VirtualPythonEnv: subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", file_path]) - def _make_pip_requirements_files(self, suffix = None): + def _make_pip_requirements_files(self, suffix): actual_os = str(self.conanfile.settings.os) pip_requirements = VirtualPythonEnv._populate_pip_requirements(self.conanfile, suffix, actual_os) @@ -133,8 +133,7 @@ class VirtualPythonEnv: def _make_pip_requirements_file(self, requirements_txt, requirements_type, suffix, generated_files): if len(requirements_txt) > 0: file_suffixes = [file_suffix for file_suffix in [suffix, requirements_type] if file_suffix is not None] - file_basename = "_".join(["pip", "requirements"] + file_suffixes) - file_path = os.path.abspath(f"{file_basename}.txt") + file_path = os.path.abspath(f"pip_requirements_{suffix}_{requirements_type}.txt") self.conanfile.output.info(f"Generating pip requirements file at '{file_path}'") save(self.conanfile, file_path, "\n".join(requirements_txt)) generated_files.append(file_path) @@ -143,7 +142,7 @@ class VirtualPythonEnv: @staticmethod def _populate_pip_requirements(conanfile, suffix, actual_os): pip_requirements = {} - data_key = "pip_requirements" + (f"_{suffix}" if suffix is not None else "") + data_key = f"pip_requirements_{suffix}" if hasattr(conanfile, "conan_data") and data_key in conanfile.conan_data: pip_requirements_data = conanfile.conan_data[data_key] From 59c86c0e305bfc8e5283d5c1df7c898e81bdcfe5 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 21 Nov 2024 17:13:05 +0100 Subject: [PATCH 37/64] Add installer profile --- profiles/installer.jinja | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 profiles/installer.jinja diff --git a/profiles/installer.jinja b/profiles/installer.jinja new file mode 100644 index 0000000..403cb39 --- /dev/null +++ b/profiles/installer.jinja @@ -0,0 +1,14 @@ +include(cura_build) + +[conf] +tools.build:skip_test=True + +[options] +curaengine/*:enable_sentry=True +arcus/*:enable_sentry=True +clipper/*:enable_sentry=True + +[settings] +curaengine/*:build_type=RelWithDebInfo +arcus/*:build_type=RelWithDebInfo -s +clipper/*:build_type=RelWithDebInfo From 096c9efa236f7533cbfa15050c7d4f7b4161c713 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 21 Nov 2024 17:13:29 +0100 Subject: [PATCH 38/64] Add warning for possibly unused scripts --- extensions/generators/EnvScriptBuilder.py | 1 + extensions/generators/GitHubActionsBuildEnv.py | 2 +- extensions/generators/GitHubActionsRunEnv.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/generators/EnvScriptBuilder.py b/extensions/generators/EnvScriptBuilder.py index 1a225ee..c47e104 100644 --- a/extensions/generators/EnvScriptBuilder.py +++ b/extensions/generators/EnvScriptBuilder.py @@ -1,5 +1,6 @@ from conan.tools.files import save +#FIXME Unused for now, remove it if still unused class EnvScriptBuilder: def __init__(self): self._variables = {} diff --git a/extensions/generators/GitHubActionsBuildEnv.py b/extensions/generators/GitHubActionsBuildEnv.py index 6beb39a..373793a 100644 --- a/extensions/generators/GitHubActionsBuildEnv.py +++ b/extensions/generators/GitHubActionsBuildEnv.py @@ -5,7 +5,7 @@ from conan.tools.env import VirtualBuildEnv from EnvScriptBuilder import EnvScriptBuilder - +#FIXME Unused for now, remove it if still unused class GitHubActionsBuildEnv: def __init__(self, conanfile: ConanFile): self.conanfile: ConanFile = conanfile diff --git a/extensions/generators/GitHubActionsRunEnv.py b/extensions/generators/GitHubActionsRunEnv.py index 8fa9c28..cd3cfef 100644 --- a/extensions/generators/GitHubActionsRunEnv.py +++ b/extensions/generators/GitHubActionsRunEnv.py @@ -5,7 +5,7 @@ from conan.tools.env import VirtualRunEnv from EnvScriptBuilder import EnvScriptBuilder - +#FIXME Unused for now, remove it if still unused class GitHubActionsRunEnv: def __init__(self, conanfile: ConanFile): self.conanfile: ConanFile = conanfile From 30723a5f41c86bad570513128e3f0dadb554ac20 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 21 Nov 2024 17:22:14 +0100 Subject: [PATCH 39/64] Fix installer profile source --- profiles/installer.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/installer.jinja b/profiles/installer.jinja index 403cb39..55d8c2b 100644 --- a/profiles/installer.jinja +++ b/profiles/installer.jinja @@ -1,4 +1,4 @@ -include(cura_build) +include(cura_build.jinja) [conf] tools.build:skip_test=True From 69314da07e7b952dbff768effc2dcc254a6dd591 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 21 Nov 2024 17:23:54 +0100 Subject: [PATCH 40/64] Fix the setting value --- profiles/installer.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/installer.jinja b/profiles/installer.jinja index 55d8c2b..79b64db 100644 --- a/profiles/installer.jinja +++ b/profiles/installer.jinja @@ -10,5 +10,5 @@ clipper/*:enable_sentry=True [settings] curaengine/*:build_type=RelWithDebInfo -arcus/*:build_type=RelWithDebInfo -s +arcus/*:build_type=RelWithDebInfo clipper/*:build_type=RelWithDebInfo From 49b6f44effed982f681e2e637ccfc4f553d32d3f Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 21 Nov 2024 17:29:27 +0100 Subject: [PATCH 41/64] Make installer profile use cura instead of cura_build --- profiles/installer.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/installer.jinja b/profiles/installer.jinja index 79b64db..b3c02fb 100644 --- a/profiles/installer.jinja +++ b/profiles/installer.jinja @@ -1,4 +1,4 @@ -include(cura_build.jinja) +include(cura.jinja) [conf] tools.build:skip_test=True From f646a50080ddaa6b4660e5282692661edbf2fa9b Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 22 Nov 2024 10:56:03 +0100 Subject: [PATCH 42/64] Add sentry-related options --- profiles/installer.jinja | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/profiles/installer.jinja b/profiles/installer.jinja index b3c02fb..1e47e2f 100644 --- a/profiles/installer.jinja +++ b/profiles/installer.jinja @@ -2,6 +2,8 @@ include(cura.jinja) [conf] tools.build:skip_test=True +user.sentry:url=https://734f9ec9024f73e53701d59c3ffddfe3@o323038.ingest.sentry.io/4506257745510401 +user.sentry:organization=ultimaker-o7 [options] curaengine/*:enable_sentry=True @@ -12,3 +14,5 @@ clipper/*:enable_sentry=True curaengine/*:build_type=RelWithDebInfo arcus/*:build_type=RelWithDebInfo clipper/*:build_type=RelWithDebInfo +arcus/*:sentry_project=curaengine +clipper/*:sentry_project=curaengine From 5e1ae62705d3af12d4426d9972a1a11aec381f43 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 22 Nov 2024 11:00:10 +0100 Subject: [PATCH 43/64] Fix sentry_project not being a setting but an option --- profiles/installer.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiles/installer.jinja b/profiles/installer.jinja index 1e47e2f..5f67a51 100644 --- a/profiles/installer.jinja +++ b/profiles/installer.jinja @@ -8,11 +8,11 @@ user.sentry:organization=ultimaker-o7 [options] curaengine/*:enable_sentry=True arcus/*:enable_sentry=True +arcus/*:sentry_project=curaengine clipper/*:enable_sentry=True +clipper/*:sentry_project=curaengine [settings] curaengine/*:build_type=RelWithDebInfo arcus/*:build_type=RelWithDebInfo clipper/*:build_type=RelWithDebInfo -arcus/*:sentry_project=curaengine -clipper/*:sentry_project=curaengine From 401c0c16e5e687403f2bf5e21a0762bd8c1ebea0 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 22 Nov 2024 13:50:33 +0100 Subject: [PATCH 44/64] Create Sentry release with curaengine --- profiles/installer.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/profiles/installer.jinja b/profiles/installer.jinja index 5f67a51..10518c2 100644 --- a/profiles/installer.jinja +++ b/profiles/installer.jinja @@ -7,6 +7,7 @@ user.sentry:organization=ultimaker-o7 [options] curaengine/*:enable_sentry=True +curaengine/*:sentry_create_release=True arcus/*:enable_sentry=True arcus/*:sentry_project=curaengine clipper/*:enable_sentry=True From 664f4d6599af3c06e161532c723b757b02d24d15 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Mon, 25 Nov 2024 09:10:59 +0100 Subject: [PATCH 45/64] Send binaries to Sentry when building an installer --- profiles/installer.jinja | 3 +++ 1 file changed, 3 insertions(+) diff --git a/profiles/installer.jinja b/profiles/installer.jinja index 10518c2..40fd3f8 100644 --- a/profiles/installer.jinja +++ b/profiles/installer.jinja @@ -7,10 +7,13 @@ user.sentry:organization=ultimaker-o7 [options] curaengine/*:enable_sentry=True +curaengine/*:sentry_send_binaries=True curaengine/*:sentry_create_release=True arcus/*:enable_sentry=True +arcus/*:sentry_send_binaries=True arcus/*:sentry_project=curaengine clipper/*:enable_sentry=True +clipper/*:sentry_send_binaries=True clipper/*:sentry_project=curaengine [settings] From 47d3fc96ae80ea4893d3f3c5e4c90fac921c0d93 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 26 Nov 2024 15:21:45 +0100 Subject: [PATCH 46/64] Fail script is some command fails --- extensions/generators/VirtualPythonEnv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index 308817d..a05b07b 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -39,7 +39,7 @@ class VirtualPythonEnv: self.conanfile.output.info(f"Using Python interpreter '{py_interp}' to create Virtual Environment in '{venv_folder}'") with env_vars.apply(): - subprocess.run([py_interp, "-m", "venv", "--copies", venv_folder]) + subprocess.run([py_interp, "-m", "venv", "--copies", venv_folder], check=True) # Make sure there executable is named the same on all three OSes this allows it to be called with `python` # simplifying GH Actions steps @@ -54,7 +54,7 @@ class VirtualPythonEnv: # Generate a script that mimics the venv activate script but is callable easily in Conan commands with env_vars.apply(): - buffer = subprocess.run([py_interp_venv, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], capture_output=True, encoding="utf-8").stdout + buffer = subprocess.run([py_interp_venv, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], capture_output=True, encoding="utf-8", check=True).stdout pythonpath = buffer.splitlines()[-1] env.define_path("VIRTUAL_ENV", venv_folder) @@ -67,7 +67,7 @@ class VirtualPythonEnv: # Install some base packages with env_vars.apply(): - subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"]) + subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"], check=True) # if self.conanfile.settings.os != "Windows": # content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(self.conanfile, @@ -94,7 +94,7 @@ class VirtualPythonEnv: with env_vars.apply(): for file_path in files_paths: self.conanfile.output.info(f"Installing pip requirements from {file_path}") - subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", file_path]) + subprocess.run([py_interp_venv, "-m", "pip", "install", "-r", file_path], check=True) def _make_pip_requirements_files(self, suffix): From 43c8d0069cebed1fc9e08a98c5cae87b9974ec28 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 26 Nov 2024 15:22:22 +0100 Subject: [PATCH 47/64] Remove default options for subdirectory at build --- global.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/global.conf b/global.conf index f122ea8..04d10c8 100644 --- a/global.conf +++ b/global.conf @@ -3,4 +3,3 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True -tools.cmake.cmake_layout:build_folder_vars=['settings.os', 'settings.compiler'] From e2a5f902116594dbbd113cf0ef05fd8e56fdea83 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Tue, 26 Nov 2024 15:28:25 +0100 Subject: [PATCH 48/64] User powershell by default on Windows --- global.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/global.conf b/global.conf index 04d10c8..8a3e51a 100644 --- a/global.conf +++ b/global.conf @@ -3,3 +3,4 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True +tools.env.virtualenv:powershell=True From dd5e3c096e5b10dc7a7441fc996944e2ae32d518 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 27 Nov 2024 11:32:31 +0100 Subject: [PATCH 49/64] Activate powershell specifically for Windows --- global.conf | 1 - profiles/cura.jinja | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/global.conf b/global.conf index 8a3e51a..04d10c8 100644 --- a/global.conf +++ b/global.conf @@ -3,4 +3,3 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True -tools.env.virtualenv:powershell=True diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 623c4ad..95d2202 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -10,6 +10,9 @@ scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 +{% if platform.system() == 'Windows' %} +tools.env.virtualenv:powershell=True +{% endif %} [options] asio-grpc/*:local_allocator=recycling_allocator From 7f5f9fd810af270baae1175fe85021b8292d7436 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 27 Nov 2024 18:03:18 +0100 Subject: [PATCH 50/64] Add wasm profile --- profiles/cura_wasm.jinja | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 profiles/cura_wasm.jinja diff --git a/profiles/cura_wasm.jinja b/profiles/cura_wasm.jinja new file mode 100644 index 0000000..bfbca96 --- /dev/null +++ b/profiles/cura_wasm.jinja @@ -0,0 +1,17 @@ +include(cura.jinja) + +[tool_requires] +# change to main once merged https://github.com/Ultimaker/conan-ultimaker-index/pull/14 +emsdk/3.1.65@ultimaker/stable +nodejs/20.16.0@ultimaker/stable + +[settings] +os=Emscripten +arch=wasm + +[conf] +tools.build:skip_test=True + +[options] +curaengine:enable_plugins=False +curaengine:enable_arcus=False From 2146751d72bc76ef8bd479c0f1846cd3603241c0 Mon Sep 17 00:00:00 2001 From: Remco Burema Date: Wed, 27 Nov 2024 20:16:04 +0100 Subject: [PATCH 51/64] (windows installer) since we run this part in bash, OS is detected as msys this is really ugly -- put up for debate later, just get it to work now CURA-11622 --- profiles/cura.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 95d2202..9090779 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -10,7 +10,7 @@ scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 -{% if platform.system() == 'Windows' %} +{% if platform.system() == 'msys' %} tools.env.virtualenv:powershell=True {% endif %} From c7b8efd5eec0377cfa483e5d4df70d607c50f219 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 28 Nov 2024 08:58:07 +0100 Subject: [PATCH 52/64] Restore Powershell option (debug purposes) --- profiles/cura.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 9090779..95d2202 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -10,7 +10,7 @@ scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 -{% if platform.system() == 'msys' %} +{% if platform.system() == 'Windows' %} tools.env.virtualenv:powershell=True {% endif %} From 21b759a612d6d8e84481ca4bc1032f705926a3bb Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 28 Nov 2024 14:36:19 +0100 Subject: [PATCH 53/64] Restore Powershell for everyone (will break things) --- profiles/cura.jinja | 2 -- 1 file changed, 2 deletions(-) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 95d2202..5cb953f 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -10,9 +10,7 @@ scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 -{% if platform.system() == 'Windows' %} tools.env.virtualenv:powershell=True -{% endif %} [options] asio-grpc/*:local_allocator=recycling_allocator From ace62584fc7c384eebdd2684b1deafdf1dccdc15 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 28 Nov 2024 14:42:57 +0100 Subject: [PATCH 54/64] Revert "Restore Powershell for everyone (will break things)" This reverts commit 21b759a612d6d8e84481ca4bc1032f705926a3bb. --- profiles/cura.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 5cb953f..95d2202 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -10,7 +10,9 @@ scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 +{% if platform.system() == 'Windows' %} tools.env.virtualenv:powershell=True +{% endif %} [options] asio-grpc/*:local_allocator=recycling_allocator From 08a7f3a04b5f4d4554fef588fad2eca8470a313e Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Thu, 28 Nov 2024 14:56:24 +0100 Subject: [PATCH 55/64] Active powershell with conf, not setting --- profiles/cura.jinja | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 95d2202..d29d73b 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -1,5 +1,10 @@ include(default) +[conf] +{% if platform.system() == 'Windows' %} +tools.env.virtualenv:powershell=True +{% endif %} + [settings] compiler.cppstd=17 curaengine*:compiler.cppstd=20 @@ -10,9 +15,6 @@ scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 cura_settings*:compiler.cppstd=20 -{% if platform.system() == 'Windows' %} -tools.env.virtualenv:powershell=True -{% endif %} [options] asio-grpc/*:local_allocator=recycling_allocator From e82608a30e309c26f4b54c801576a6bc6b2a1107 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 29 Nov 2024 07:51:59 +0100 Subject: [PATCH 56/64] Disable powershell for cpython --- global.conf | 3 +++ profiles/cura.jinja | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/global.conf b/global.conf index 04d10c8..6584285 100644 --- a/global.conf +++ b/global.conf @@ -3,3 +3,6 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True +{% if platform.system() == 'Windows' %} +tools.env.virtualenv:powershell=True +{% endif %} diff --git a/profiles/cura.jinja b/profiles/cura.jinja index d29d73b..2c7a219 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -1,9 +1,7 @@ include(default) [conf] -{% if platform.system() == 'Windows' %} -tools.env.virtualenv:powershell=True -{% endif %} +cpython/*:tools.env.virtualenv:powershell=False [settings] compiler.cppstd=17 From b7269cb58580e32872c26d77ac8670ac1813dc04 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 29 Nov 2024 09:05:48 +0100 Subject: [PATCH 57/64] Remove powershell specific for CPython (doesn't work) --- profiles/cura.jinja | 3 --- 1 file changed, 3 deletions(-) diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 2c7a219..623c4ad 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -1,8 +1,5 @@ include(default) -[conf] -cpython/*:tools.env.virtualenv:powershell=False - [settings] compiler.cppstd=17 curaengine*:compiler.cppstd=20 From 29104655a6a4533924a8b5922babc114ea0a660b Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 29 Nov 2024 10:47:11 +0100 Subject: [PATCH 58/64] Disable powershell, CPython build doesn't support it --- global.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/global.conf b/global.conf index 6584285..04d10c8 100644 --- a/global.conf +++ b/global.conf @@ -3,6 +3,3 @@ core:default_build_profile = cura_build.jinja tools.cmake.cmaketoolchain:generator = Ninja tools.gnu:define_libcxx11_abi = True core.version_ranges:resolve_prereleases=True -{% if platform.system() == 'Windows' %} -tools.env.virtualenv:powershell=True -{% endif %} From b67fcb10f00c0156aa9437b14c7ebaa41f454940 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 29 Nov 2024 12:06:08 +0100 Subject: [PATCH 59/64] Support pip packages with both url and hash --- extensions/generators/VirtualPythonEnv.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index a05b07b..b978f51 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -110,18 +110,22 @@ class VirtualPythonEnv: requirements_hashes_txt = [] for package_name, package_desc in pip_requirements.items(): - package_requirement = package_name + (f"=={package_desc['version']}" if "version" in package_desc else "") - - if "hashes" in package_desc: - package_requirement_with_hashes = [package_requirement] - for hash_str in package_desc['hashes']: - package_requirement_with_hashes.append(f"--hash={hash_str}") - requirements_hashes_txt.append(" ".join(package_requirement_with_hashes)) + package_requirement = "" + packages_hashes = [] if "url" in package_desc: - requirements_basic_txt.append(package_desc['url']) + package_requirement = package_desc["url"] + elif "version" in package_desc: + package_requirement = f"{package_name}=={package_desc['version']}" else: - requirements_basic_txt.append(package_requirement) + package_requirement = package_name + + if "hashes" in package_desc: + for hash_str in package_desc["hashes"]: + packages_hashes.append(f"--hash={hash_str}") + + destination_file = requirements_hashes_txt if len(packages_hashes) > 0 else requirements_basic_txt + destination_file.append(' '.join([package_requirement] + packages_hashes)) generated_files = [] self._make_pip_requirements_file(requirements_basic_txt, "basic", suffix, generated_files) From 3117a0bd96121c16cc0983155a92c3bf588dcbbc Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 29 Nov 2024 13:43:03 +0100 Subject: [PATCH 60/64] Properly handle package with urls+hashes --- extensions/generators/VirtualPythonEnv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index b978f51..f37d6db 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -67,6 +67,7 @@ class VirtualPythonEnv: # Install some base packages with env_vars.apply(): + subprocess.run([py_interp_venv, "-m", "pip", "install", "--upgrade", "pip"], check=True) subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"], check=True) # if self.conanfile.settings.os != "Windows": @@ -114,7 +115,7 @@ class VirtualPythonEnv: packages_hashes = [] if "url" in package_desc: - package_requirement = package_desc["url"] + package_requirement = f"{package_name}@{package_desc['url']}" elif "version" in package_desc: package_requirement = f"{package_name}=={package_desc['version']}" else: From ccaa511059ea390118a6f22d47e056c82b0c9e23 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Fri, 29 Nov 2024 13:53:45 +0100 Subject: [PATCH 61/64] Use Conan Center 2 --- remotes.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remotes.json b/remotes.json index 5e2e0bd..e0f7f4a 100644 --- a/remotes.json +++ b/remotes.json @@ -2,7 +2,7 @@ "remotes": [ { "name": "conancenter", - "url": "https://center.conan.io", + "url": "https://center2.conan.io", "verify_ssl": true }, { From 67a68c30ffd297b3964f353f179472c092da12eb Mon Sep 17 00:00:00 2001 From: Jelle Spijker Date: Sat, 30 Nov 2024 14:52:15 +0100 Subject: [PATCH 62/64] Add/Update profiles for wasm Introduce a new 'cura_clang.jinja' profile for using Clang 18 with specific compiler settings. Update 'cura_wasm.jinja' by modifying plugin options and adding new settings for curator and libzip. Additionally, adjust the C++ standard setting for the 'curator' component in 'cura.jinja'. Contribute to CURA-11622 --- profiles/cura.jinja | 2 +- profiles/cura_clang.jinja | 13 +++++++++++++ profiles/cura_wasm.jinja | 8 +++++--- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 profiles/cura_clang.jinja diff --git a/profiles/cura.jinja b/profiles/cura.jinja index 623c4ad..bc2c743 100644 --- a/profiles/cura.jinja +++ b/profiles/cura.jinja @@ -9,7 +9,7 @@ curaengine_grpc_definitions*:compiler.cppstd=20 scripta*:compiler.cppstd=20 umspatial*:compiler.cppstd=20 dulcificum*:compiler.cppstd=20 -cura_settings*:compiler.cppstd=20 +curator*:compiler.cppstd=20 [options] asio-grpc/*:local_allocator=recycling_allocator diff --git a/profiles/cura_clang.jinja b/profiles/cura_clang.jinja new file mode 100644 index 0000000..fc7eaaa --- /dev/null +++ b/profiles/cura_clang.jinja @@ -0,0 +1,13 @@ +include(cura.jinja) + +[tool_requires] + +[settings] +compiler=clang +compiler.version=18 +compiler.libcxx=libstdc++11 + +[options] + +[conf] +tools.build:compiler_executables={"c":"clang", "cpp":"clang++"} diff --git a/profiles/cura_wasm.jinja b/profiles/cura_wasm.jinja index bfbca96..372fc79 100644 --- a/profiles/cura_wasm.jinja +++ b/profiles/cura_wasm.jinja @@ -1,7 +1,6 @@ include(cura.jinja) [tool_requires] -# change to main once merged https://github.com/Ultimaker/conan-ultimaker-index/pull/14 emsdk/3.1.65@ultimaker/stable nodejs/20.16.0@ultimaker/stable @@ -13,5 +12,8 @@ arch=wasm tools.build:skip_test=True [options] -curaengine:enable_plugins=False -curaengine:enable_arcus=False +curaengine/*:enable_plugins=False +curaengine/*:enable_arcus=False +curator/*:with_cura_resources=True +curator/*:disable_logging=True +libzip/*:crypto=False From ec787f69831aa92b3d54f5866e716b95374c52f0 Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 4 Dec 2024 08:00:59 +0100 Subject: [PATCH 63/64] Remove now-useless generators --- extensions/generators/EnvScriptBuilder.py | 38 ------------------- .../generators/GitHubActionsBuildEnv.py | 22 ----------- extensions/generators/GitHubActionsRunEnv.py | 22 ----------- 3 files changed, 82 deletions(-) delete mode 100644 extensions/generators/EnvScriptBuilder.py delete mode 100644 extensions/generators/GitHubActionsBuildEnv.py delete mode 100644 extensions/generators/GitHubActionsRunEnv.py diff --git a/extensions/generators/EnvScriptBuilder.py b/extensions/generators/EnvScriptBuilder.py deleted file mode 100644 index c47e104..0000000 --- a/extensions/generators/EnvScriptBuilder.py +++ /dev/null @@ -1,38 +0,0 @@ -from conan.tools.files import save - -#FIXME Unused for now, remove it if still unused -class EnvScriptBuilder: - def __init__(self): - self._variables = {} - - def set_variable(self, name: str, value: str): - self._variables[name] = value - - def set_environment(self, env): - for name, value in env.items(): - self.set_variable(name, value) - - def save(self, path, conanfile, append_to=None) -> None: - file_path = path - - content = "" - for name, value in self._variables.items(): - set_variable = f'{name}={value}' - - if append_to is not None: - set_variable = f"echo {set_variable} >> {append_to}" - else: - set_variable = f"export {set_variable}" - - content += f"{set_variable}\n" - - if conanfile.settings.get_safe("os") == "Windows": - if conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool): - file_path += ".ps1" - else: - file_path += ".bat" - else: - file_path += ".sh" - - conanfile.output.info(f"Saving environment script to {file_path}") - save(conanfile, file_path, content) diff --git a/extensions/generators/GitHubActionsBuildEnv.py b/extensions/generators/GitHubActionsBuildEnv.py deleted file mode 100644 index 373793a..0000000 --- a/extensions/generators/GitHubActionsBuildEnv.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path - -from conan import ConanFile -from conan.tools.env import VirtualBuildEnv - -from EnvScriptBuilder import EnvScriptBuilder - -#FIXME Unused for now, remove it if still unused -class GitHubActionsBuildEnv: - def __init__(self, conanfile: ConanFile): - self.conanfile: ConanFile = conanfile - - def generate(self): - build_env = VirtualBuildEnv(self.conanfile) - env = build_env.environment() - envvars = env.vars(self.conanfile, scope="build") - env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_buildenv")) - - script_builder = EnvScriptBuilder() - script_builder.set_environment(envvars) - script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV") diff --git a/extensions/generators/GitHubActionsRunEnv.py b/extensions/generators/GitHubActionsRunEnv.py deleted file mode 100644 index cd3cfef..0000000 --- a/extensions/generators/GitHubActionsRunEnv.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path - -from conan import ConanFile -from conan.tools.env import VirtualRunEnv - -from EnvScriptBuilder import EnvScriptBuilder - -#FIXME Unused for now, remove it if still unused -class GitHubActionsRunEnv: - def __init__(self, conanfile: ConanFile): - self.conanfile: ConanFile = conanfile - - def generate(self): - run_env = VirtualRunEnv(self.conanfile) - env = run_env.environment() - envvars = env.vars(self.conanfile, scope="run") - env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else "" - filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_runenv")) - - script_builder = EnvScriptBuilder() - script_builder.set_environment(envvars) - script_builder.save(filepath, self.conanfile, f"${env_prefix}GITHUB_ENV") From 712ed439bb2fadbb19c5cbbda62f7054f36bb82a Mon Sep 17 00:00:00 2001 From: Erwan MATHIEU Date: Wed, 4 Dec 2024 08:02:48 +0100 Subject: [PATCH 64/64] Remove unused code --- extensions/generators/VirtualPythonEnv.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/extensions/generators/VirtualPythonEnv.py b/extensions/generators/VirtualPythonEnv.py index f37d6db..612f8b9 100644 --- a/extensions/generators/VirtualPythonEnv.py +++ b/extensions/generators/VirtualPythonEnv.py @@ -70,14 +70,6 @@ class VirtualPythonEnv: subprocess.run([py_interp_venv, "-m", "pip", "install", "--upgrade", "pip"], check=True) subprocess.run([py_interp_venv, "-m", "pip", "install", "wheel", "setuptools"], check=True) - # if self.conanfile.settings.os != "Windows": - # content = f"source {os.path.join(output_folder, 'conan', 'virtual_python_env.sh')}\n" + load(self.conanfile, - # os.path.join( - # output_folder, - # bin_venv_path, - # "activate")) - # save(self.conanfile, os.path.join(output_folder, bin_venv_path, "activate"), content) - requirements_core = self._make_pip_requirements_files("core") requirements_dev = self._make_pip_requirements_files("dev") requirements_installer = self._make_pip_requirements_files("installer")