From 28a88f125d189a6ee479f45431c1b7bb9c215b96 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Wed, 24 Nov 2021 01:21:52 +0000 Subject: [PATCH] A tool to build a self-contained source tarball The does some pruning of the dependencies' files to avoid a 100 MiB+ tarball. Even with that, the final tarball size is 17 MiB with xz, 25 MiB with gzip. See the file documentation at the top of `tools/make_src_dist.py` for more information. --- .gitignore | 3 + CMakeLists.txt | 28 +++++-- tools/make_src_dist.py | 165 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 5 deletions(-) create mode 100755 tools/make_src_dist.py diff --git a/.gitignore b/.gitignore index 5865b4cc1..8a574ae36 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ comparer-config.toml /build-*/ .vscode/tasks.json +# Extra files in the source distribution (see make_src_dist.py) +/dist/ + # ELF object file, shared library and object archive. *.o *.so diff --git a/CMakeLists.txt b/CMakeLists.txt index e9cee7960..4c242067e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,12 @@ if(POLICY CMP0111) cmake_policy(SET CMP0111 NEW) endif() +if(IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/dist") + message("-- Detected a source distribution with the required FetchContent dependencies and devilutionx.mpq included") + set(SRC_DIST ON) + add_subdirectory(dist) +endif() + include(CMakeDependentOption) include(CMake/out_of_tree.cmake) include(CMake/genex.cmake) @@ -41,6 +47,19 @@ mark_as_advanced(DISABLE_STREAMING_SOUNDS) option(STREAM_ALL_AUDIO "Stream all the audio. For extremely RAM-constrained platforms.") mark_as_advanced(STREAM_ALL_AUDIO) +# By default, devilutionx.mpq is built only if smpq is installed. +if(NOT DEFINED BUILD_ASSETS_MPQ AND NOT SRC_DIST) + find_program(SMPQ smpq) +elseif(BUILD_ASSETS_MPQ) + find_program(SMPQ smpq REQUIRED) +endif() +if(SMPQ) + set(_has_smpq ON) +else() + set(_has_smpq OFF) +endif() +option(BUILD_ASSETS_MPQ "If true, assets are packaged into devilutionx.mpq." ${_has_smpq}) + # The gettext[tools] package takes a very long time to install if(CMAKE_TOOLCHAIN_FILE MATCHES "vcpkg.cmake$") option(USE_GETTEXT_FROM_VCPKG "Add vcpkg dependency for gettext[tools] for compiling translations" OFF) @@ -835,8 +854,7 @@ if (Gettext_FOUND) endforeach() endif() -find_program(SMPQ smpq) -if(SMPQ) +if(BUILD_ASSETS_MPQ) set(DEVILUTIONX_MPQ "${CMAKE_CURRENT_BINARY_DIR}/devilutionx.mpq") add_custom_command( COMMENT "Building devilutionx.mpq" @@ -1130,7 +1148,7 @@ if(VITA) set(VITA_MKSFOEX_FLAGS "${VITA_MKSFOEX_FLAGS} -d PARENTAL_LEVEL=1") set(VITA_MKSFOEX_FLAGS "${VITA_MKSFOEX_FLAGS} -d ATTRIBUTE2=12") vita_create_self(devilutionx.self devilutionx UNSAFE) - if(SMPQ) + if(BUILD_ASSETS_MPQ OR SRC_DIST) vita_create_vpk(devilutionx.vpk ${VITA_TITLEID} devilutionx.self VERSION ${VITA_VERSION} NAME ${VITA_APP_NAME} @@ -1180,7 +1198,7 @@ if(NINTENDO_3DS) add_custom_target(romfs_files COMMAND ${CMAKE_COMMAND} -E copy ${APP_ROMFS_FILES} ${APP_ROMFS} DEPENDS ${APP_ROMFS_FILES}) - + add_dependencies(romfs_files romfs_directory devilutionx_mpq) include(Tools3DS) @@ -1190,7 +1208,7 @@ if(NINTENDO_3DS) add_dependencies(${APP_TARGET_PREFIX}_cia romfs_files) endif() -if(CPACK AND (APPLE OR SMPQ)) +if(CPACK AND (APPLE OR BUILD_ASSETS_MPQ OR SRC_DIST)) if(WIN32) if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(SDL2_WIN32_DLLS_DIR "${CMAKE_BINARY_DIR}") diff --git a/tools/make_src_dist.py b/tools/make_src_dist.py new file mode 100755 index 000000000..4d23ce03d --- /dev/null +++ b/tools/make_src_dist.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python + +""" +Makes a tarball suitable for distros. + +It contains the following: + +1. The repo source code. + +2. An additional `dist` directory with: + + 1. `FetchContent` dependencies that currently must be vendored. + These are stripped from especially heavy bloat. + + 2. `devilutionx.mpq`. + While this file can be generated by the build system, it requires + the `smpq` host dependency which may be missing in some distributions. + + 3. `CMakeFlags.txt` - a file the cmake flags containing the version, + the path to `devilutionx.mpq` and the dependency path. + + This file is automatically used by the build system if present. + +The only stdout output of this script is the path to the generated tarball. +""" + +import logging +import pathlib +import re +import shutil +import subprocess +import sys + +# We only package the dependencies that are: +# 1. Uncommon in package managers (sdl_audiolib and simpleini). +# 2. Require devilutionx forks (all others). +_DEPS = ['asio', 'libzt', 'sdl_audiolib', 'simpleini'] + +_ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent +_BUILD_DIR = _ROOT_DIR.joinpath('build-src-dist') +_ARCHIVE_DIR = _BUILD_DIR.joinpath('archive') +_DIST_DIR = _ARCHIVE_DIR.joinpath('dist') + +_LOGGER = logging.getLogger() +_LOGGER.setLevel(logging.INFO) +_LOGGER.addHandler(logging.StreamHandler(sys.stderr)) + + +def main(): + cmake(f'-S{_ROOT_DIR}', f'-B{_BUILD_DIR}', '-DBUILD_ASSETS_MPQ=ON') + cmake('--build', _BUILD_DIR, '--target', 'devilutionx_mpq') + + if _ARCHIVE_DIR.exists(): + shutil.rmtree(_ARCHIVE_DIR) + + _LOGGER.info(f'Copying repo files...') + for src_bytes in git('ls-files', '-z').rstrip(b'\0').split(b'\0'): + src = src_bytes.decode() + dst_path = _ARCHIVE_DIR.joinpath(src) + dst_path.parent.mkdir(parents=True, exist_ok=True) + if re.search('(^|/)\.gitkeep$', src): + continue + shutil.copy2(_ROOT_DIR.joinpath(src), dst_path, follow_symlinks=False) + + _LOGGER.info(f'Copying devilutionx.mpq...') + _DIST_DIR.mkdir(parents=True) + shutil.copy(_BUILD_DIR.joinpath('devilutionx.mpq'), _DIST_DIR) + + for dep in _DEPS: + _LOGGER.info(f'Copying {dep}...') + shutil.copytree( + src=_BUILD_DIR.joinpath('_deps', f'{dep}-src'), + dst=_DIST_DIR.joinpath(f'{dep}-src'), + ignore=ignore_dep_src) + + version_num, version_suffix = get_version() + write_dist_cmakelists(version_num, version_suffix) + print(make_archive(version_num, version_suffix)) + + +def cmake(*cmd_args): + _LOGGER.info(f'+ cmake {subprocess.list2cmdline(cmd_args)}') + subprocess.run(['cmake', *cmd_args], cwd=_ROOT_DIR, stdout=subprocess.DEVNULL) + + +def git(*cmd_args): + _LOGGER.debug(f'+ git {subprocess.list2cmdline(cmd_args)}') + return subprocess.run(['git', *cmd_args], cwd=_ROOT_DIR, capture_output=True).stdout + + +# Ignore files in dependencies that we don't need. +# Examples of some heavy ones: +# 48M libzt-src/ext +# 9.8M asio-src/asio/src/doc +_IGNORE_DEP_DIR_RE = re.compile( + r'(/|^)\.|(/|^)(tests?|other|vcx?proj|examples?|doxygen|docs?|asio-src/asio/src)(/|$)') +_IGNORE_DEP_FILE_RE = re.compile( + r'(^\.|Makefile|vcx?proj|example|doxygen|docs?|\.(doxy|cmd|png|html|ico|icns)$)') + + +def ignore_dep_src(src, names): + if 'sdl_audiolib' in src: + # SDL_audiolib currently fails to compile if any of the files are missing. + # TODO: Fix this in SDL_audiolib by making this optional: + # https://github.com/realnc/SDL_audiolib/blob/5a700ba556d3a5b5c531c2fa1f45fc0c3214a16b/CMakeLists.txt#L399-L401 + return [] + + if _IGNORE_DEP_DIR_RE.search(src): + _LOGGER.debug(f'Excluded directory {src}') + return names + + def ignore_name(name): + if _IGNORE_DEP_FILE_RE.search(name) or _IGNORE_DEP_DIR_RE.search(name): + _LOGGER.debug(f'Excluded file {src}/{name}') + return True + return False + + return filter(ignore_name, names) + + +def get_version(): + git_tag = git('describe', '--abbrev=0', '--tags').rstrip() + git_commit_sha = git('rev-parse', '--short', 'HEAD').rstrip() + git_tag_sha = git('rev-parse', '--short', git_tag).rstrip() + return git_tag, (git_commit_sha if git_tag_sha != git_commit_sha else None) + + +def write_dist_cmakelists(version_num, version_suffix): + version_num, version_suffix = get_version() + with open(_DIST_DIR.joinpath('CMakeLists.txt'), 'wb') as f: + f.write(b'# Generated by tools/make_src_dist.py\n') + f.write(b'set(VERSION_NUM "%s" PARENT_SCOPE)\n' % version_num) + if version_suffix: + f.write(b'set(VERSION_SUFFIX "%s" PARENT_SCOPE)\n' % version_suffix) + + f.write(b''' +# Pre-generated `devilutionx.mpq` is provided so that distributions do not have to depend on smpq. +set(DEVILUTIONX_MPQ "${CMAKE_CURRENT_SOURCE_DIR}/devilutionx.mpq" PARENT_SCOPE) + +# This would ensure that CMake does not attempt to connect to network. +# We do not set this to allow for builds for Windows and Android, which do fetch some +# dependencies even with this source distribution. +# set(FETCHCONTENT_FULLY_DISCONNECTED ON PARENT_SCOPE) + +# Set the path to each dependency that must be vendored: +''') + for dep in _DEPS: + f.write(b'set(FETCHCONTENT_SOURCE_DIR_%s "${CMAKE_CURRENT_SOURCE_DIR}/%s-src" CACHE STRING "")\n' % ( + dep.upper().encode(), dep.encode())) + + +def make_archive(version_num, version_suffix): + archive_base_name = f'devilutionx-{version_num.decode()}' + if version_suffix: + archive_base_name += f'-{version_suffix.decode()}' + _LOGGER.info(f'Compressing {_ARCHIVE_DIR}') + return shutil.make_archive( + format='xztar', + logger=_LOGGER, + base_name=_BUILD_DIR.joinpath(archive_base_name), + root_dir=_ARCHIVE_DIR, + base_dir='.') + + +main()