203 lines
11 KiB
Python
203 lines
11 KiB
Python
"""Build a release package (dist/<name>.zip + .sha256) from source with clang-cl on Linux (CI).
|
|
|
|
usage: python3 tools/ci_build.py [--project ds2|se2] [--version TAG] [--toolchain DIR] [--winsysroot DIR]
|
|
python3 tools/ci_build.py --cache-key
|
|
--project (default ds2): ds2 = DS2Revival.zip (the game plugins, tools/package.py); se2 = SE2-Revival.zip
|
|
(DS2Toolkit + BWCompat for Siege Editor 2, tools/package_toolkit.py). --version TAG (se2) names the release in the
|
|
package's README.txt. The same package as mods/build.py + the packaging script on Windows with MSVC, cross-compiled:
|
|
1. xwin (pinned release, SHA-256 checked) downloads the x86 MSVC CRT and Windows SDK at pinned versions
|
|
(xwin checks every payload against the SHA-256 in Microsoft's manifest) into DIR/winsysroot-<crt>-<sdk>
|
|
(DIR defaults to build/ci-toolchain; reused when present). Running it accepts Microsoft's license for
|
|
those build tools. --winsysroot uses an existing /winsysroot-style directory instead.
|
|
2. Ultimate ASI Loader (pinned release, zip and dinput8.dll SHA-256 checked) -> build/asiloader/dinput8.dll.
|
|
3. mods/build.py --clang: the project's plugins (ds2: the shipped game plugins, not ds2dev; se2: DS2Toolkit and
|
|
BWCompat), same flags as the MSVC build.
|
|
4. Checks every plugin: PE32 x86 DLL, static CRT (no MSVCP/VCRUNTIME/UCRT/api-ms-win-crt imports).
|
|
5. The packaging script -> dist/<name>/ and dist/<name>.zip; dist/<name>.zip.sha256.
|
|
Needs python3, clang and lld 19 or newer (the MSVC STL rejects older Clang): clang-cl or clang, and lld-link,
|
|
on PATH or in /usr/lib/llvm-<n>/bin.
|
|
"""
|
|
import glob, hashlib, io, os, shutil, struct, subprocess, sys, tarfile, time, urllib.request, zipfile
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
XWIN_VERSION = '0.10.0'
|
|
XWIN_URL = ('https://github.com/Jake-Shadle/xwin/releases/download/%s/xwin-%s-x86_64-unknown-linux-musl.tar.gz'
|
|
% (XWIN_VERSION, XWIN_VERSION))
|
|
XWIN_SHA256 = 'd870eb4b2f390878af6da1ccd3cf321d22fcb72720984853b4be732ae597fc88'
|
|
CRT_VERSION = '14.44.17.14' # MSVC v143 (VS 2022 17.14) CRT/STL
|
|
SDK_VERSION = '10.0.26100' # Windows 11 SDK
|
|
LOADER_URL = 'https://github.com/ThirteenAG/Ultimate-ASI-Loader/releases/download/v9.7.4/Ultimate-ASI-Loader.zip'
|
|
LOADER_ZIP_SHA256 = '952cebfc30d525afc2bdbaca954329d405ded3aa688a83027354dae14dfd5c5f'
|
|
LOADER_DLL_SHA256 = 'd5a059aa467a7a7127c8f6169f79fa63ff0f55986ee9eb2fd9a281bebf2aa2e6' # x86 dinput8.dll
|
|
MIN_CLANG = 19
|
|
# project -> plugins (build/mods/<name>.asi), mods/build.py selection, packaging script (tools/), zip in dist/
|
|
PROJECTS = {
|
|
'ds2': dict(plugins=['DS2Multiplayer', 'DS2Widescreen', 'DS2Graphics', 'DS2Fixes', 'DS2BrokenWorld', 'DS2ModLoader', 'DS2Revival'],
|
|
select=['--release'], package='package.py', zip='DS2Revival.zip', versioned=False),
|
|
'se2': dict(plugins=['DS2Toolkit', 'BWCompat'], select=['--only', 'DS2Toolkit,BWCompat'],
|
|
package='package_toolkit.py', zip='SE2-Revival.zip', versioned=True),
|
|
}
|
|
CRT_DLLS = ('msvcp', 'msvcr', 'vcruntime', 'ucrtbase', 'api-ms-win-crt', 'concrt', 'libc++', 'libstdc++',
|
|
'libgcc', 'libwinpthread')
|
|
|
|
|
|
def fetch(url, sha256):
|
|
"""The bytes at url, checked against the pinned SHA-256 (3 tries)."""
|
|
for attempt in range(3):
|
|
try:
|
|
req = urllib.request.Request(url, headers={'User-Agent': 'ds2revival-ci'})
|
|
data = urllib.request.urlopen(req, timeout=120).read()
|
|
break
|
|
except OSError as e:
|
|
if attempt == 2:
|
|
raise
|
|
print('download failed (%s), retrying' % e)
|
|
time.sleep(5 * (attempt + 1))
|
|
got = hashlib.sha256(data).hexdigest()
|
|
if got != sha256:
|
|
sys.exit('SHA-256 mismatch for %s: got %s, expected %s' % (url, got, sha256))
|
|
return data
|
|
|
|
|
|
def winsysroot(toolchain):
|
|
"""The /winsysroot-style MSVC CRT + Windows SDK directory, downloaded with xwin when missing."""
|
|
out = os.path.join(toolchain, 'winsysroot-%s-%s' % (CRT_VERSION, SDK_VERSION))
|
|
if os.path.isfile(os.path.join(out, '.complete')):
|
|
print('using', out)
|
|
return out
|
|
if not sys.platform.startswith('linux'):
|
|
sys.exit('fetching the CRT/SDK needs Linux x86_64 (xwin); pass --winsysroot DIR instead')
|
|
xwin = os.path.join(toolchain, 'xwin-%s' % XWIN_VERSION, 'xwin')
|
|
if not os.path.isfile(xwin):
|
|
with tarfile.open(fileobj=io.BytesIO(fetch(XWIN_URL, XWIN_SHA256)), mode='r:gz') as tf:
|
|
member = next(m for m in tf.getmembers() if m.isfile() and os.path.basename(m.name) == 'xwin')
|
|
os.makedirs(os.path.dirname(xwin), exist_ok=True)
|
|
with open(xwin, 'wb') as f:
|
|
f.write(tf.extractfile(member).read())
|
|
os.chmod(xwin, 0o755)
|
|
cache = os.path.join(toolchain, 'xwin-cache')
|
|
shutil.rmtree(out, ignore_errors=True)
|
|
t = time.time()
|
|
subprocess.run([xwin, '--accept-license', '--log-level', 'warn', '--http-retry', '3', '--cache-dir', cache,
|
|
'--arch', 'x86', '--variant', 'desktop', '--crt-version', CRT_VERSION,
|
|
'--sdk-version', SDK_VERSION, 'splat', '--use-winsysroot-style', '--output', out], check=True)
|
|
shutil.rmtree(cache, ignore_errors=True) # the downloads and unpacked packages: ~1 GB, not needed again
|
|
# Never used by the plugins and large (keeps the CI cache smaller): the C++/WinRT projection headers and
|
|
# LLVM's sanitizer/profiling runtimes that the CRT package carries.
|
|
shutil.rmtree(os.path.join(out, 'Windows Kits', '10', 'Include', SDK_VERSION, 'cppwinrt'), ignore_errors=True)
|
|
for lib in glob.glob(os.path.join(out, 'VC', 'Tools', 'MSVC', CRT_VERSION, 'lib', 'x86', 'clang_rt.*')):
|
|
os.remove(lib)
|
|
open(os.path.join(out, '.complete'), 'w').close()
|
|
print('MSVC CRT %s + Windows SDK %s -> %s (%.0fs)' % (CRT_VERSION, SDK_VERSION, out, time.time() - t))
|
|
return out
|
|
|
|
|
|
def find_clang():
|
|
"""Put the newest Debian/Ubuntu LLVM (/usr/lib/llvm-<n>/bin, n >= MIN_CLANG, with lld) first on PATH,
|
|
then check the clang version found."""
|
|
dirs = [d for d in glob.glob('/usr/lib/llvm-*/bin') if os.path.isfile(os.path.join(d, 'lld-link'))
|
|
and d.split('llvm-')[1].split('/')[0].isdigit() and int(d.split('llvm-')[1].split('/')[0]) >= MIN_CLANG]
|
|
if dirs:
|
|
best = max(dirs, key=lambda d: int(d.split('llvm-')[1].split('/')[0]))
|
|
os.environ['PATH'] = best + os.pathsep + os.environ.get('PATH', '')
|
|
cc = shutil.which('clang-cl') or shutil.which('clang')
|
|
if not cc or not shutil.which('lld-link'):
|
|
sys.exit('clang-cl/clang and lld-link not found')
|
|
ver = subprocess.run([cc, '--version'], capture_output=True, text=True).stdout.splitlines()[0]
|
|
major = int(ver.split('version ')[1].split('.')[0])
|
|
if major < MIN_CLANG:
|
|
sys.exit('%s: Clang %d or newer needed (MSVC STL %s)' % (ver, MIN_CLANG, CRT_VERSION))
|
|
print(ver, '+', shutil.which('lld-link'))
|
|
|
|
|
|
def asi_loader():
|
|
data = fetch(LOADER_URL, LOADER_ZIP_SHA256)
|
|
dll = zipfile.ZipFile(io.BytesIO(data)).read('dinput8.dll')
|
|
if hashlib.sha256(dll).hexdigest() != LOADER_DLL_SHA256:
|
|
sys.exit('dinput8.dll in %s does not match the pinned SHA-256' % LOADER_URL)
|
|
out = os.path.join(ROOT, 'build', 'asiloader', 'dinput8.dll')
|
|
os.makedirs(os.path.dirname(out), exist_ok=True)
|
|
with open(out, 'wb') as f:
|
|
f.write(dll)
|
|
|
|
|
|
def pe_info(path):
|
|
"""(machine, is_dll, [imported DLL names], [exported names]) of a PE32 file."""
|
|
d = open(path, 'rb').read()
|
|
pe = struct.unpack_from('<I', d, 0x3C)[0]
|
|
if d[pe:pe + 4] != b'PE\0\0':
|
|
sys.exit('%s: not a PE file' % path)
|
|
machine, nsec, _, _, _, optsize, chars = struct.unpack_from('<HHIIIHH', d, pe + 4)
|
|
opt = pe + 24
|
|
if struct.unpack_from('<H', d, opt)[0] != 0x10B:
|
|
sys.exit('%s: not PE32' % path)
|
|
secs = [struct.unpack_from('<IIII', d, opt + optsize + 40 * i + 8) for i in range(nsec)] # vsize, va, rawsize, raw
|
|
|
|
def off(rva):
|
|
for vsize, va, rawsize, raw in secs:
|
|
if va <= rva < va + max(vsize, rawsize):
|
|
return rva - va + raw
|
|
raise ValueError('rva %x outside the sections' % rva)
|
|
|
|
def cstr(rva):
|
|
o = off(rva)
|
|
return d[o:d.index(b'\0', o)].decode('ascii', 'replace')
|
|
exp_rva, exp_size, imp_rva, imp_size = struct.unpack_from('<IIII', d, opt + 96)
|
|
imports, exports = [], []
|
|
if imp_rva:
|
|
o = off(imp_rva)
|
|
while True:
|
|
name_rva = struct.unpack_from('<I', d, o + 12)[0]
|
|
if not name_rva:
|
|
break
|
|
imports.append(cstr(name_rva))
|
|
o += 20
|
|
if exp_rva:
|
|
n_names, _, names_rva = struct.unpack_from('<III', d, off(exp_rva) + 24) # NumberOfNames, -, AddressOfNames
|
|
exports = [cstr(struct.unpack_from('<I', d, off(names_rva) + 4 * i)[0]) for i in range(n_names)]
|
|
return machine, bool(chars & 0x2000), imports, exports
|
|
|
|
|
|
def check_plugins(plugins):
|
|
for name in plugins:
|
|
path = os.path.join(ROOT, 'build', 'mods', name + '.asi')
|
|
machine, dll, imports, exports = pe_info(path)
|
|
bad = [i for i in imports if i.lower().startswith(CRT_DLLS)]
|
|
shown = ' '.join(exports) if len(exports) <= 8 else '%d (%s ... %s)' % (len(exports), exports[0], exports[-1])
|
|
print('%s.asi: %d bytes, imports %s, exports %s' % (name, os.path.getsize(path), ' '.join(imports),
|
|
shown or '-'))
|
|
if machine != 0x14C or not dll:
|
|
sys.exit('%s: not an x86 DLL (machine %#x)' % (path, machine))
|
|
if bad:
|
|
sys.exit('%s: imports CRT DLLs %s (the CRT must be static, /MT)' % (path, bad))
|
|
|
|
|
|
def main():
|
|
sys.stdout.reconfigure(line_buffering=True) # keep the log in order with the tools' output
|
|
a = sys.argv[1:]
|
|
if '--cache-key' in a: # for the CI cache of the toolchain directory: changes with every pinned version
|
|
print('winsysroot-%s-%s-xwin%s' % (CRT_VERSION, SDK_VERSION, XWIN_VERSION))
|
|
return
|
|
project = PROJECTS[a[a.index('--project') + 1] if '--project' in a else 'ds2']
|
|
toolchain = os.path.abspath(a[a.index('--toolchain') + 1]) if '--toolchain' in a else \
|
|
os.path.join(ROOT, 'build', 'ci-toolchain')
|
|
find_clang()
|
|
sysroot = os.path.abspath(a[a.index('--winsysroot') + 1]) if '--winsysroot' in a else winsysroot(toolchain)
|
|
asi_loader()
|
|
for name in project['plugins']: # never package a stale plugin
|
|
if os.path.exists(os.path.join(ROOT, 'build', 'mods', name + '.asi')):
|
|
os.remove(os.path.join(ROOT, 'build', 'mods', name + '.asi'))
|
|
subprocess.run([sys.executable, os.path.join(ROOT, 'mods', 'build.py')] + project['select'] +
|
|
['--clang', sysroot], check=True)
|
|
check_plugins(project['plugins'])
|
|
version = ['--version', a[a.index('--version') + 1]] if '--version' in a and project['versioned'] else []
|
|
subprocess.run([sys.executable, os.path.join(ROOT, 'tools', project['package'])] + version, check=True)
|
|
z = os.path.join(ROOT, 'dist', project['zip'])
|
|
digest = hashlib.sha256(open(z, 'rb').read()).hexdigest()
|
|
with open(z + '.sha256', 'w', newline='\n') as f:
|
|
f.write('%s %s\n' % (digest, project['zip']))
|
|
print('%s (%d bytes) sha256 %s' % (z, os.path.getsize(z), digest))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|