Browse Source
The does some pruning of the dependencies' files to avoid a 100 MiB+ tarball. Even with that, the final tarball size is 17 MiB with xz, 25 MiB with gzip. See the file documentation at the top of `tools/make_src_dist.py` for more information.pull/3604/head
Gleb Mazovetskiy
4 years ago
3 changed files with 178 additions and 3 deletions
@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python |
||||
|
||||
""" |
||||
Makes a tarball suitable for distros. |
||||
|
||||
It contains the following: |
||||
|
||||
1. The repo source code. |
||||
|
||||
2. An additional `dist` directory with: |
||||
|
||||
1. `FetchContent` dependencies that currently must be vendored. |
||||
These are stripped from especially heavy bloat. |
||||
|
||||
2. `devilutionx.mpq`. |
||||
While this file can be generated by the build system, it requires |
||||
the `smpq` host dependency which may be missing in some distributions. |
||||
|
||||
3. `CMakeFlags.txt` - a file the cmake flags containing the version, |
||||
the path to `devilutionx.mpq` and the dependency path. |
||||
|
||||
This file is automatically used by the build system if present. |
||||
|
||||
The only stdout output of this script is the path to the generated tarball. |
||||
""" |
||||
|
||||
import logging |
||||
import pathlib |
||||
import re |
||||
import shutil |
||||
import subprocess |
||||
import sys |
||||
|
||||
# We only package the dependencies that are: |
||||
# 1. Uncommon in package managers (sdl_audiolib and simpleini). |
||||
# 2. Require devilutionx forks (all others). |
||||
_DEPS = ['asio', 'libmpq', 'libsmackerdec', |
||||
'libzt', 'sdl_audiolib', 'simpleini'] |
||||
|
||||
_ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent |
||||
_BUILD_DIR = _ROOT_DIR.joinpath('build-src-dist') |
||||
_ARCHIVE_DIR = _BUILD_DIR.joinpath('archive') |
||||
_DIST_DIR = _ARCHIVE_DIR.joinpath('dist') |
||||
|
||||
_LOGGER = logging.getLogger() |
||||
_LOGGER.setLevel(logging.INFO) |
||||
_LOGGER.addHandler(logging.StreamHandler(sys.stderr)) |
||||
|
||||
|
||||
def main(): |
||||
cmake(f'-S{_ROOT_DIR}', f'-B{_BUILD_DIR}', '-DBUILD_ASSETS_MPQ=ON') |
||||
cmake('--build', _BUILD_DIR, '--target', 'devilutionx_mpq') |
||||
|
||||
if _ARCHIVE_DIR.exists(): |
||||
shutil.rmtree(_ARCHIVE_DIR) |
||||
|
||||
_LOGGER.info(f'Copying repo files...') |
||||
for src_bytes in git('ls-files', '-z').rstrip(b'\0').split(b'\0'): |
||||
src = src_bytes.decode() |
||||
dst_path = _ARCHIVE_DIR.joinpath(src) |
||||
dst_path.parent.mkdir(parents=True, exist_ok=True) |
||||
if re.search('(^|/)\.gitkeep$', src): |
||||
continue |
||||
shutil.copy2(_ROOT_DIR.joinpath(src), dst_path, follow_symlinks=False) |
||||
|
||||
_LOGGER.info(f'Copying devilutionx.mpq...') |
||||
_DIST_DIR.mkdir(parents=True) |
||||
shutil.copy(_BUILD_DIR.joinpath('devilutionx.mpq'), _DIST_DIR) |
||||
|
||||
for dep in _DEPS: |
||||
_LOGGER.info(f'Copying {dep}...') |
||||
shutil.copytree( |
||||
src=_BUILD_DIR.joinpath('_deps', f'{dep}-src'), |
||||
dst=_DIST_DIR.joinpath(f'{dep}-src'), |
||||
ignore=ignore_dep_src) |
||||
|
||||
version_num, version_suffix = get_version() |
||||
write_dist_cmakelists(version_num, version_suffix) |
||||
print(make_archive(version_num, version_suffix)) |
||||
|
||||
|
||||
def cmake(*cmd_args): |
||||
_LOGGER.info(f'+ cmake {subprocess.list2cmdline(cmd_args)}') |
||||
subprocess.run(['cmake', *cmd_args], cwd=_ROOT_DIR, stdout=subprocess.DEVNULL) |
||||
|
||||
|
||||
def git(*cmd_args): |
||||
_LOGGER.debug(f'+ git {subprocess.list2cmdline(cmd_args)}') |
||||
return subprocess.run(['git', *cmd_args], cwd=_ROOT_DIR, capture_output=True).stdout |
||||
|
||||
|
||||
# Ignore files in dependencies that we don't need. |
||||
# Examples of some heavy ones: |
||||
# 48M libzt-src/ext |
||||
# 9.8M asio-src/asio/src/doc |
||||
_IGNORE_DEP_DIR_RE = re.compile( |
||||
r'(/|^)\.|(/|^)(tests?|other|vcx?proj|examples?|doxygen|docs?|asio-src/asio/src)(/|$)') |
||||
_IGNORE_DEP_FILE_RE = re.compile( |
||||
r'(^\.|Makefile|vcx?proj|example|doxygen|docs?|\.(doxy|cmd|png|html|ico|icns)$)') |
||||
|
||||
|
||||
def ignore_dep_src(src, names): |
||||
if 'sdl_audiolib' in src: |
||||
# SDL_audiolib currently fails to compile if any of the files are missing. |
||||
# TODO: Fix this in SDL_audiolib by making this optional: |
||||
# https://github.com/realnc/SDL_audiolib/blob/5a700ba556d3a5b5c531c2fa1f45fc0c3214a16b/CMakeLists.txt#L399-L401 |
||||
return [] |
||||
|
||||
if _IGNORE_DEP_DIR_RE.search(src): |
||||
_LOGGER.debug(f'Excluded directory {src}') |
||||
return names |
||||
|
||||
def ignore_name(name): |
||||
if _IGNORE_DEP_FILE_RE.search(name) or _IGNORE_DEP_DIR_RE.search(name): |
||||
_LOGGER.debug(f'Excluded file {src}/{name}') |
||||
return True |
||||
return False |
||||
|
||||
return filter(ignore_name, names) |
||||
|
||||
|
||||
def get_version(): |
||||
git_tag = git('describe', '--abbrev=0', '--tags').rstrip() |
||||
git_commit_sha = git('rev-parse', '--short', 'HEAD').rstrip() |
||||
git_tag_sha = git('rev-parse', '--short', git_tag).rstrip() |
||||
return git_tag, (git_commit_sha if git_tag_sha != git_commit_sha else None) |
||||
|
||||
|
||||
def write_dist_cmakelists(version_num, version_suffix): |
||||
version_num, version_suffix = get_version() |
||||
with open(_DIST_DIR.joinpath('CMakeLists.txt'), 'wb') as f: |
||||
f.write(b'# Generated by tools/make_src_dist.py\n') |
||||
f.write(b'set(VERSION_NUM "%s" PARENT_SCOPE)\n' % version_num) |
||||
if version_suffix: |
||||
f.write(b'set(VERSION_SUFFIX "%s" PARENT_SCOPE)\n' % version_suffix) |
||||
|
||||
f.write(b''' |
||||
# Pre-generated `devilutionx.mpq` is provided so that distributions do not have to depend on smpq. |
||||
set(DEVILUTIONX_MPQ "${CMAKE_CURRENT_SOURCE_DIR}/devilutionx.mpq" PARENT_SCOPE) |
||||
|
||||
# This would ensure that CMake does not attempt to connect to network. |
||||
# We do not set this to allow for builds for Windows and Android, which do fetch some |
||||
# dependencies even with this source distribution. |
||||
# set(FETCHCONTENT_FULLY_DISCONNECTED ON PARENT_SCOPE) |
||||
|
||||
# Set the path to each dependency that must be vendored: |
||||
''') |
||||
for dep in _DEPS: |
||||
f.write(b'set(FETCHCONTENT_SOURCE_DIR_%s "${CMAKE_CURRENT_SOURCE_DIR}/%s-src" CACHE STRING "")\n' % ( |
||||
dep.upper().encode(), dep.encode())) |
||||
|
||||
|
||||
def make_archive(version_num, version_suffix): |
||||
archive_base_name = f'devilutionx-{version_num.decode()}' |
||||
if version_suffix: |
||||
archive_base_name += f'-{version_suffix.decode()}' |
||||
_LOGGER.info(f'Compressing {_ARCHIVE_DIR}') |
||||
return shutil.make_archive( |
||||
format='xztar', |
||||
logger=_LOGGER, |
||||
base_name=_BUILD_DIR.joinpath(archive_base_name), |
||||
root_dir=_ARCHIVE_DIR, |
||||
base_dir='.') |
||||
|
||||
|
||||
main() |
||||
Loading…
Reference in new issue