Browse Source

Merge branch 'CURA-9177_Fix_CI_CD'

master
Ghostkeeper 4 years ago
parent
commit
5e79858c2e
No known key found for this signature in database
GPG Key ID: 68F39EA88EEED5FF
  1. 5
      README.md
  2. 22
      conan.conf
  3. 30
      generators/GitHubActionsBuildEnv.py
  4. 30
      generators/GitHubActionsRunEnv.py
  5. 48
      generators/PyCharmRunEnv.py
  6. 177
      generators/VirtualPythonEnv.py
  7. 10
      global.conf
  8. 17
      profiles/cura.jinja
  9. 19
      profiles/cura_build.jinja
  10. 7
      profiles/cura_debug.jinja
  11. 7
      profiles/cura_release.jinja
  12. 3
      remotes.txt

5
README.md

@ -0,0 +1,5 @@
# Ultimaker Conan configuration files
> Conan is preparing a major update to 2.0, to ensure future proofing our recipes make sure you set the `CONAN_V2_MODE=1`
> environment variable.
> https://docs.conan.io/en/latest/reference/conan_v2_mode.html

22
conan.conf

@ -1,22 +1,20 @@
[log]
run_to_output = True
run_to_file = True
level = warning
print_run_commands = True
run_to_output = True # environment CONAN_LOG_RUN_TO_OUTPUT
run_to_file = False # environment CONAN_LOG_RUN_TO_FILE
level = critical # environment CONAN_LOGGING_LEVEL
print_run_commands = False # environment CONAN_PRINT_RUN_COMMANDS
[general]
compression_level = 9
sysrequires_sudo = True
request_timeout = 60
default_package_id_mode = full_version_mode
default_profile = default
compression_level = 9 # environment CONAN_COMPRESSION_LEVEL
sysrequires_sudo = True # environment CONAN_SYSREQUIRES_SUDO
request_timeout = 60 # environment CONAN_REQUEST_TIMEOUT (seconds)
default_package_id_mode = full_package_mode # environment CONAN_DEFAULT_PACKAGE_ID_MODE
revisions_enabled = 1
scm_to_conandata = 1
full_transitive_package_id = 0
default_profile = cura_release.jinja
default_build_profile = cura_build.jinja
required_conan_version = >=1.44.1
[storage]
path = ./data
[proxies]

30
generators/GitHubActionsBuildEnv.py

@ -0,0 +1,30 @@
from pathlib import Path
from jinja2 import Template
from conan.tools.env import VirtualBuildEnv
from conans.model import Generator
class GitHubActionsBuildEnv(Generator):
@property
def filename(self):
filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_buildenv"))
if self.conanfile.settings.get_safe("os") == "Windows":
if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type = bool):
filepath += ".ps1"
else:
filepath += ".bat"
else:
filepath += ".sh"
return filepath
@property
def content(self):
template = Template("""{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""")
build_env = VirtualBuildEnv(self.conanfile)
env = build_env.environment()
envvars = env.vars(self.conanfile, scope = "build")
env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else ""
return template.render(envvars = envvars, env_prefix = env_prefix)

30
generators/GitHubActionsRunEnv.py

@ -0,0 +1,30 @@
from pathlib import Path
from jinja2 import Template
from conan.tools.env import VirtualRunEnv
from conans.model import Generator
class GitHubActionsRunEnv(Generator):
@property
def filename(self):
filepath = str(Path(self.conanfile.generators_folder).joinpath("activate_github_actions_runenv"))
if self.conanfile.settings.get_safe("os") == "Windows":
if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type = bool):
filepath += ".ps1"
else:
filepath += ".bat"
else:
filepath += ".sh"
return filepath
@property
def content(self):
template = Template("""{% for k, v in envvars.items() %}echo "{{ k }}={{ v }}" >> ${{ env_prefix }}GITHUB_ENV\n{% endfor %}""")
build_env = VirtualRunEnv(self.conanfile)
env = build_env.environment()
envvars = env.vars(self.conanfile, scope = "run")
env_prefix = "Env:" if self.conanfile.settings.os == "Windows" else ""
return template.render(envvars = envvars, env_prefix = env_prefix)

48
generators/PyCharmRunEnv.py

@ -0,0 +1,48 @@
from pathlib import Path
from jinja2 import Template
from conan.tools.env import VirtualRunEnv
from conans.model import Generator
class PyCharmRunEnv(Generator):
run_xml = Template(r"""<component name="ProjectRunConfigurationManager">
<configuration default="false" name="{{ name }}" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
<module name="{{ name }}" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />{% for key, value in envvars.items() %}
<env name="{{ key }}" value="{{ value }}" />{% endfor %}
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/{{ entrypoint }}" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
</component> """)
@property
def filename(self):
stem = Path(self.conanfile._um_data(self.conanfile.version)["runinfo"]["entrypoint"]).stem
return str(Path(self.conanfile.source_folder).joinpath(".run", f"{stem}.run.xml"))
@property
def content(self):
run_env = VirtualRunEnv(self.conanfile)
env = run_env.environment()
envvars = env.vars(self.conanfile, scope = "run")
entrypoint = self.conanfile._um_data(self.conanfile.version)["runinfo"]["entrypoint"]
return self.run_xml.render(name = entrypoint, envvars = envvars,
entrypoint = entrypoint)

177
generators/VirtualPythonEnv.py

@ -0,0 +1,177 @@
import sys
import os
from io import StringIO
from pathlib import Path
from jinja2 import Template
from conan import ConanFile
from conan.tools.env import VirtualRunEnv
from conans.model import Generator
from conans.errors import ConanException
class VirtualPythonEnv(Generator):
@property
def _script_ext(self):
if self.conanfile.settings.get_safe("os") == "Windows":
if self.conanfile.conf.get("tools.env.virtualenv:powershell", check_type = bool):
return ".ps1"
else:
return ".bat"
return ".sh"
@property
def _venv_path(self):
if self.settings.os == "Windows":
return "Scripts"
return "bin"
@property
def filename(self):
pass
@property
def content(self):
conanfile: ConanFile = self.conanfile
python_interpreter = Path(self.conanfile.deps_user_info["cpython"].python)
# When on Windows execute as Windows Path
if conanfile.settings.os == "Windows":
python_interpreter = Path(*[f'"{p}"' if " " in p else p for p in python_interpreter.parts])
# Create the virtual environment
if conanfile.install_folder is None:
if conanfile.build_folder is None:
venv_folder = Path(os.getcwd(), "venv")
else:
venv_folder = conanfile.build_folder
else:
venv_folder = conanfile.install_folder
conanfile.run(f"""{python_interpreter} -m venv {venv_folder}""", run_environment = True, env = "conanrun")
# Make sure there executable is named the same on all three OSes this allows it to be called with `python`
# simplifying GH Actions steps
if conanfile.settings.os != "Windows":
python_venv_interpreter = Path(venv_folder, self._venv_path, "python")
if not python_venv_interpreter.exists():
python_venv_interpreter.hardlink_to(
Path(venv_folder, self._venv_path, Path(sys.executable).stem + Path(sys.executable).suffix))
else:
python_venv_interpreter = Path(venv_folder, self._venv_path, Path(sys.executable).stem + Path(sys.executable).suffix)
if not python_venv_interpreter.exists():
raise ConanException(f"Virtual environment Python interpreter not found at: {python_venv_interpreter}")
if conanfile.settings.os == "Windows":
python_venv_interpreter = Path(*[f'"{p}"' if " " in p else p for p in python_venv_interpreter.parts])
buffer = StringIO()
outer = '"' if conanfile.settings.os == "Windows" else "'"
inner = "'" if conanfile.settings.os == "Windows" else '"'
conanfile.run(f"{python_venv_interpreter} -c {outer}import sysconfig; print(sysconfig.get_path({inner}purelib{inner})){outer}",
env = "conanrun",
output = buffer)
pythonpath = buffer.getvalue().splitlines()[-1]
run_env = VirtualRunEnv(conanfile)
env = run_env.environment()
env.define_path("VIRTUAL_ENV", venv_folder)
env.prepend_path("PATH", os.path.join(venv_folder, self._venv_path))
env.prepend_path("PYTHONPATH", pythonpath)
env.unset("PYTHONHOME")
envvars = env.vars(conanfile, scope = "run")
# Install some base_packages
conanfile.run(f"""{python_venv_interpreter} -m pip install wheel setuptools""", run_environment = True, env = "conanrun")
# Install pip_requirements from dependencies
for dep_name in reversed(conanfile.deps_user_info):
dep_user_info = conanfile.deps_user_info[dep_name]
if len(dep_user_info.vars) == 0:
continue
pip_req_paths = [conanfile.deps_cpp_info[dep_name].res_paths[i] for i, req_path in
enumerate(conanfile.deps_cpp_info[dep_name].resdirs) if req_path.endswith("pip_requirements")]
if len(pip_req_paths) != 1:
continue
pip_req_base_path = Path(pip_req_paths[0])
if hasattr(dep_user_info, "pip_requirements"):
req_txt = pip_req_base_path.joinpath(dep_user_info.pip_requirements)
if req_txt.exists():
conanfile.run(f"{python_venv_interpreter} -m pip install -r {req_txt} --force-reinstall", run_environment = True,
env = "conanrun")
conanfile.output.success(f"Dependency {dep_name} specifies pip_requirements in user_info installed!")
else:
conanfile.output.warn(f"Dependency {dep_name} specifies pip_requirements in user_info but {req_txt} can't be found!")
if hasattr(dep_user_info, "pip_requirements_git"):
req_txt = pip_req_base_path.joinpath(dep_user_info.pip_requirements_git)
if req_txt.exists():
conanfile.run(f"{python_venv_interpreter} -m pip install -r {req_txt} --force-reinstall", run_environment = True,
env = "conanrun")
conanfile.output.success(f"Dependency {dep_name} specifies pip_requirements_git in user_info installed!")
else:
conanfile.output.warn(
f"Dependency {dep_name} specifies pip_requirements_git in user_info but {req_txt} can't be found!")
if hasattr(dep_user_info, "pip_requirements_build"):
req_txt = pip_req_base_path.joinpath(dep_user_info.pip_requirements_build)
if req_txt.exists():
conanfile.run(f"{python_venv_interpreter} -m pip install -r {req_txt} --force-reinstall", run_environment = True,
env = "conanrun")
conanfile.output.success(f"Dependency {dep_name} specifies pip_requirements_build in user_info installed!")
else:
conanfile.output.warn(
f"Dependency {dep_name} specifies pip_requirements_build in user_info but {req_txt} can't be found!")
if not conanfile.in_local_cache:
# Install the Python requirements of the current conanfile requirements*.txt
pip_req_base_path = Path(conanfile.cpp_info.rootpath, conanfile.cpp_info.resdirs[-1])
# Add the dev reqs needed for pyinstaller
conanfile.run(
f"{python_venv_interpreter} -m pip install -r {pip_req_base_path.joinpath(conanfile.user_info.pip_requirements_build)} --force-reinstall",
run_environment = True, env = "conanrun")
# Install the requirements.text for cura
conanfile.run(
f"{python_venv_interpreter} -m pip install -r {pip_req_base_path.joinpath(conanfile.user_info.pip_requirements_git)} --force-reinstall",
run_environment = True, env = "conanrun")
# Do the final requirements last such that these dependencies takes precedence over possible previous installed Python modules.
# Since these are actually shipped with Cura and therefore require hashes and pinned version numbers in the requirements.txt
self.run(
f"{python_venv_interpreter} -m pip install -r {pip_req_base_path.joinpath(conanfile.user_info.pip_requirements)} --force-reinstall",
run_environment = True,
env = "conanrun")
# Add all dlls/dylibs/so found in site-packages to the PATH, DYLD_LIBRARY_PATH and LD_LIBRARY_PATH
dll_paths = list({ dll.parent for dll in Path(pythonpath).glob("**/*.dll") })
for dll_path in dll_paths:
env.append_path("PATH", str(dll_path))
dylib_paths = list({ dylib.parent for dylib in Path(pythonpath).glob("**/*.dylib") })
for dylib_path in dylib_paths:
env.append_path("DYLD_LIBRARY_PATH", str(dylib_path))
so_paths = list({ so.parent for so in Path(pythonpath).glob("**/*.dylib") })
for so_path in so_paths:
env.append_path("LD_LIBRARY_PATH", str(so_path))
full_envvars = env.vars(conanfile, scope = "run")
# Generate the Python Virtual Environment Script
full_envvars.save_sh(Path(venv_folder, self._venv_path, "activate"))
full_envvars.save_bat(Path(venv_folder, self._venv_path, "activate.bat"))
full_envvars.save_ps1(Path(venv_folder, self._venv_path, "Activate.ps1"))
# Generate the GitHub Action activation script
env_prefix = "Env:" if conanfile.settings.os == "Windows" else ""
activate_github_actions_buildenv = Template(r"""{% for var, value in envvars.items() %}echo "{{ var }}={{ value }}" >> ${{ env_prefix }}GITHUB_ENV
{% endfor %}""").render(envvars = full_envvars, env_prefix = env_prefix)
return {
str(Path(venv_folder, self._venv_path, f"activate_github_actions_env{self._script_ext}")): activate_github_actions_buildenv
}

10
global.conf

@ -0,0 +1,10 @@
core:default_profile = cura.jinja
core:default_build_profile = cura_build.jinja
tools.cmake.cmaketoolchain:generator = Ninja
tools.env.virtualenv:auto_use = True
tools.gnu:define_libcxx11_abi = True
# FIXME: Needs to be commented out for OpenSSL to work but if we wan't to create ps1 scripts it needs to be set to True
# Otherwise .bat files are created. Maybe we should define this on a recipe basis:
#
#{% if platform.system() == 'Windows' %}tools.env.virtualenv:powershell=True{% endif %}

17
profiles/cura.jinja

@ -3,22 +3,15 @@ include(default)
[build_requires]
[settings]
os={{ {"Darwin": "Macos"}.get(platform.system(), platform.system()) }}
os_build={{ {"Darwin": "Macos"}.get(platform.system(), platform.system()) }}
os='{{ {'Darwin': 'Macos'}.get(platform.system(), platform.system()) }}'
os_build='{{ {'Darwin': 'Macos'}.get(platform.system(), platform.system()) }}'
compiler.cppstd=17
{% if compiler == "gcc" %}
compiler.libcxx=libstdc++11
{% elif compiler == "apple-clang" %}
compiler.libcxx=libc++
{% elif compiler == "Visual Studio" %}
compiler.toolset=v142
{% if compiler == 'gcc' %}compiler.libcxx=libstdc++11
{% elif compiler == 'apple-clang' %}compiler.libcxx=libc++
{% elif compiler == 'Visual Studio' %}compiler.toolset=v143
{% endif %}
[options]
[env]
[conf]
tools.cmake.cmaketoolchain:find_package_prefer_config=True
{% if compiler == "Visual Studio" %}
tools.microsoft.msbuild:vs_version=compiler.version
{% endif %}

19
profiles/cura_build.jinja

@ -1,26 +1,17 @@
include(default)
[build_requires]
cmake/3.22.0
ninja/1.10.2
[settings]
os={{ {"Darwin": "Macos"}.get(platform.system(), platform.system()) }}
os_build={{ {"Darwin": "Macos"}.get(platform.system(), platform.system()) }}
os='{{ {'Darwin': 'Macos'}.get(platform.system(), platform.system()) }}'
os_build='{{ {'Darwin': 'Macos'}.get(platform.system(), platform.system()) }}'
compiler.cppstd=17
{% if compiler == "gcc" %}
compiler.libcxx=libstdc++11
{% elif compiler == "apple-clang" %}
compiler.libcxx=libc++
{% elif compiler == "Visual Studio" %}
compiler.toolset=v142
{% if compiler == 'gcc' %}compiler.libcxx=libstdc++11
{% elif compiler == 'apple-clang' %}compiler.libcxx=libc++
{% elif compiler == 'Visual Studio' %}compiler.toolset=v143
{% endif %}
[options]
[env]
[conf]
tools.cmake.cmaketoolchain:find_package_prefer_config=True
{% if compiler == "Visual Studio" %}
tools.microsoft.msbuild:vs_version=compiler.version
{% endif %}

7
profiles/cura_debug.jinja

@ -1,7 +0,0 @@
include(cura.jinja)
[settings]
build_type=Debug
{% if compiler == "Visual Studio" %}
compiler.runtime=MDd
{% endif %}

7
profiles/cura_release.jinja

@ -1,7 +0,0 @@
include(cura.jinja)
[settings]
build_type=Release
{% if compiler == "Visual Studio" %}
compiler.runtime=MD
{% endif %}

3
remotes.txt

@ -1,2 +1,3 @@
ultimaker https://peer23peer.jfrog.io/artifactory/api/conan/ultimaker-conan True
cura https://ultimaker.jfrog.io/artifactory/api/conan/cura-internal True
cura-ce https://ultimaker.jfrog.io/artifactory/api/conan/cura-community True
conan-center https://center.conan.io True

Loading…
Cancel
Save