241 lines
12 KiB
Python
241 lines
12 KiB
Python
"""Read Dungeon Siege 2 Tank files (.ds2party, .ds2res, .ds2map: magic 'DSg2Tank') and Dungeon Siege 1 tanks
|
|
(.dsres, .dsmap: magic 'DSigTank'; read only - docs/formats/ds1-vs-ds2.md).
|
|
|
|
usage: python -I tools/tank.py list FILE
|
|
python -I tools/tank.py extract FILE OUTDIR
|
|
python -I tools/tank.py header FILE
|
|
python -I tools/tank.py pack TEMPLATE OUT NAME=FILE... (template's header and file order; NAME=FILE replaces
|
|
or adds a file; NAME= drops it; files stored uncompressed)
|
|
|
|
Layout as observed in party files (all little-endian):
|
|
0x00 'DSg2Tank', 0x08 header version, 0x0c dir set offset, 0x10 file set offset, 0x14 index size,
|
|
0x18 data offset, 0x34 priority, 0x3c creator '!GPG', 0x40 GUID, 0x58 build time (SYSTEMTIME), 0x68 copyright
|
|
(UTF-16), 0x314 product version and 0x31c minimum version (4 x u16 each: major, minor, sub, build).
|
|
DS1 ('DSigTank', header version 0x00010002, data at 0x324): the same fields up to 0x68 except that 0x1c and 0x28
|
|
hold the product and minimum versions (6 x u16 each, meaning not established); 0x1f8 title (UTF-16, e.g.
|
|
"Dungeon Siege Map: The World of Ehb"), 0x2c0 author. Index, CRCs and chunk format are the same.
|
|
File set: u32 count, u32 offsets[count] (relative to the set); each entry: parent, size, offset, crc32,
|
|
FILETIME, u16 format (0 raw, 1 zlib), u16 flags, u16 name length + NUL-terminated name padded to 4; zlib entries then have
|
|
u32 compressed size, u32 chunk size and one (size, compressed size, extra, offset) record per chunk.
|
|
Dir set: u32 count, u32 offsets[count]; the root entry is parent, child count, FILETIME, empty name (4 bytes),
|
|
u32 child offsets (into the dir set), 16 zero bytes. 0x50 index CRC32 = crc32(dir set .. end of file);
|
|
0x54 data CRC32 = crc32 of the uncompressed files concatenated in storage order. 16 zero bytes separate the
|
|
data from the dir set.
|
|
"""
|
|
import os, struct, sys, zlib
|
|
|
|
|
|
class Tank:
|
|
def __init__(self, path):
|
|
self.d = d = open(path, 'rb').read()
|
|
# 'DSg2Tank' = Dungeon Siege II; 'DSigTank' = Dungeon Siege 1 (.dsres/.dsmap, docs/formats/ds1-vs-ds2.md):
|
|
# the same index layout and storage formats; the product/minimum versions sit at 0x1c/0x28 (6 x u16)
|
|
# instead of 0x314/0x31c (4 x u16), and the header's title text (0x1f8) is filled in.
|
|
self.kind = {b'DSg2Tank': 'ds2', b'DSigTank': 'ds1'}.get(d[:8])
|
|
if not self.kind:
|
|
raise ValueError('%s: not a DSg2Tank/DSigTank file' % path)
|
|
(self.version, self.dirset, self.fileset, self.index_size,
|
|
self.data_offset) = struct.unpack_from('<5I', d, 8)
|
|
if self.kind == 'ds2':
|
|
self.product = struct.unpack_from('<4H', d, 0x314)
|
|
self.minimum = struct.unpack_from('<4H', d, 0x31c)
|
|
else:
|
|
self.product = struct.unpack_from('<6H', d, 0x1c)
|
|
self.minimum = struct.unpack_from('<6H', d, 0x28)
|
|
self.priority = struct.unpack_from('<I', d, 0x34)[0]
|
|
self.built = struct.unpack_from('<8H', d, 0x58)
|
|
self.title = d[0x1f8:0x2c0].decode('utf-16-le', 'replace').split('\0')[0] if self.kind == 'ds1' else ''
|
|
# Directories: entry offsets are relative to the dir set; a parent of 0 means the root.
|
|
self.dirs = {}
|
|
ndirs = struct.unpack_from('<I', d, self.dirset)[0]
|
|
for i in range(ndirs):
|
|
rel = struct.unpack_from('<I', d, self.dirset + 4 + 4 * i)[0]
|
|
parent = struct.unpack_from('<I', d, self.dirset + rel)[0]
|
|
nlen = struct.unpack_from('<H', d, self.dirset + rel + 16)[0]
|
|
self.dirs[rel] = (parent, d[self.dirset + rel + 18:self.dirset + rel + 18 + nlen].decode('latin1'))
|
|
self.files = []
|
|
count = struct.unpack_from('<I', d, self.fileset)[0]
|
|
for i in range(count):
|
|
o = self.fileset + struct.unpack_from('<I', d, self.fileset + 4 + 4 * i)[0]
|
|
parent, size, off, crc = struct.unpack_from('<4I', d, o)
|
|
fmt, flags, nlen = struct.unpack_from('<HHH', d, o + 24)
|
|
name = d[o + 30:o + 30 + nlen].decode('latin1')
|
|
p = o + 30 + nlen + 1 # NUL-terminated, padded to 4
|
|
p += (-(p - o)) % 4
|
|
chunks = []
|
|
if fmt:
|
|
csize, chunk = struct.unpack_from('<II', d, p)
|
|
p += 8
|
|
n = (size + chunk - 1) // chunk if chunk else 0
|
|
for k in range(n):
|
|
chunks.append(struct.unpack_from('<4I', d, p + 16 * k))
|
|
self.files.append(dict(name=name, path=self.dir_path(parent) + name, parent=parent, size=size,
|
|
offset=off, crc=crc, format=fmt, flags=flags, chunks=chunks))
|
|
|
|
def dir_path(self, rel):
|
|
"""'a/b/' for a directory entry offset; '' for the root."""
|
|
out, seen = '', set()
|
|
while rel and rel in self.dirs and rel not in seen:
|
|
seen.add(rel)
|
|
parent, name = self.dirs[rel]
|
|
if not name:
|
|
break
|
|
out = name + '/' + out
|
|
rel = parent
|
|
return out
|
|
|
|
def read(self, f):
|
|
base = self.data_offset + f['offset']
|
|
if not f['format']:
|
|
return self.d[base:base + f['size']]
|
|
out = b''
|
|
for usize, csize, extra, off in f['chunks']:
|
|
raw = self.d[base + off:base + off + csize + extra]
|
|
if usize == csize:
|
|
out += raw[:usize]
|
|
elif f['format'] != 1:
|
|
# DS1's TankStructure.h also defines LZO chunks (format 2); no retail DS1 or DS2 tank uses them
|
|
raise NotImplementedError('%s: storage format %d' % (f['path'], f['format']))
|
|
else:
|
|
# the last 'extra' bytes of a chunk are stored uncompressed after its zlib stream
|
|
out += (zlib.decompressobj().decompress(raw[:csize]) + raw[csize:])[:usize]
|
|
return out
|
|
|
|
|
|
def pack(template, files):
|
|
"""files: list of (name, data) in storage order, all in the root directory (party files have no
|
|
subdirectories). Header fields are copied from the template Tank."""
|
|
if any('/' in f['path'] for f in template.files):
|
|
raise ValueError('pack only writes flat tanks (party files); the template has subdirectories')
|
|
data, entries, off = b'', [], 0
|
|
for name, blob in files:
|
|
entries.append((name, len(blob), off, zlib.crc32(blob) & 0xffffffff))
|
|
data += blob
|
|
off += len(blob)
|
|
head = bytearray(template.d[:template.data_offset])
|
|
body = data + b'\0' * 16
|
|
dirset_at = template.data_offset + len(body)
|
|
byname = sorted(entries, key=lambda e: e[0].lower())
|
|
fe = []
|
|
for name, size, off, crc in byname:
|
|
n = name.encode('latin1')
|
|
e = struct.pack('<4I', 0, size, off, crc) + b'\0' * 8 + struct.pack('<HHH', 0, 0, len(n)) + n + b'\0'
|
|
fe.append(e + b'\0' * (-len(e) % 4))
|
|
fs_head = 4 + 4 * len(fe)
|
|
offs, pos = [], fs_head
|
|
for e in fe:
|
|
offs.append(pos)
|
|
pos += len(e)
|
|
fileset = struct.pack('<I', len(fe)) + b''.join(struct.pack('<I', o) for o in offs) + b''.join(fe)
|
|
dirlen = 8 + 20 + 4 * len(fe) + 16
|
|
root = (struct.pack('<2I', 0, len(fe)) + b'\0' * 8 + b'\0' * 4 +
|
|
b''.join(struct.pack('<I', dirlen + o) for o in offs) + b'\0' * 16)
|
|
dirset = struct.pack('<2I', 1, 8) + root
|
|
assert len(dirset) == dirlen
|
|
index = dirset + fileset
|
|
struct.pack_into('<4I', head, 0x0c, dirset_at, dirset_at + len(dirset), len(index), template.data_offset)
|
|
struct.pack_into('<2I', head, 0x50, zlib.crc32(index) & 0xffffffff, zlib.crc32(data) & 0xffffffff)
|
|
return bytes(head) + body + index
|
|
|
|
|
|
def pack_tree(template, files, priority=0, guid=None):
|
|
"""A tank with directories (maps, resources). files: list of (path 'a/b/c.gas', data). Files are stored
|
|
uncompressed (format 0). Header fields come from the template Tank; the priority word at 0x34 (0 for base
|
|
content, 0x2000 for Broken World's overlays) and the GUID at 0x40 are set here.
|
|
Layout as in GPG's tanks: dir set = count, entry offsets, entries (parent, child count, FILETIME, name,
|
|
child offsets - relative to the dir set, sorted by name, files and directories mixed), 16 zero bytes;
|
|
file set = count, entry offsets, entries (parent, size, data offset, CRC32, FILETIME, format, flags, name)."""
|
|
import os
|
|
files = sorted(((p.strip('/'), b) for p, b in files), key=lambda f: f[0].lower())
|
|
dirs = {'': []} # dir path -> children (name, 'd'|'f', key)
|
|
for p, _ in files:
|
|
parts = p.split('/')
|
|
for i in range(len(parts) - 1):
|
|
parent, here = '/'.join(parts[:i]), '/'.join(parts[:i + 1])
|
|
if here not in dirs:
|
|
dirs[here] = []
|
|
dirs[parent].append((parts[i], 'd', here))
|
|
dirs['/'.join(parts[:-1])].append((parts[-1], 'f', p))
|
|
order = sorted(dirs, key=lambda d: (d.count('/') + (d != ''), d.lower())) # root first, then by depth
|
|
# data, in path order
|
|
data, foff = b'', {}
|
|
for p, blob in files:
|
|
foff[p] = len(data)
|
|
data += blob
|
|
blobs = dict(files)
|
|
|
|
def entry_head(parent, count, name):
|
|
n = name.encode('latin1')
|
|
e = struct.pack('<2I', parent, count) + b'\0' * 8 + struct.pack('<H', len(n)) + n + b'\0'
|
|
return e + b'\0' * (-len(e) % 4)
|
|
|
|
# dir entry sizes are known up front, so offsets can be assigned before the child lists are filled in
|
|
rel, pos = {}, 4 + 4 * len(order)
|
|
for d in order:
|
|
rel[d] = pos
|
|
pos += len(entry_head(0, 0, os.path.basename(d))) + 4 * len(dirs[d])
|
|
dirset_len = pos + 16
|
|
# file entries
|
|
fentries, fpos = [], {}
|
|
fs_pos = 4 + 4 * len(files)
|
|
for p, blob in files:
|
|
parent = rel['/'.join(p.split('/')[:-1])]
|
|
n = p.split('/')[-1].encode('latin1')
|
|
e = struct.pack('<4I', parent, len(blob), foff[p], zlib.crc32(blob) & 0xffffffff) + b'\0' * 8 + \
|
|
struct.pack('<HHH', 0, 0, len(n)) + n + b'\0'
|
|
e += b'\0' * (-len(e) % 4)
|
|
fpos[p] = fs_pos
|
|
fentries.append(e)
|
|
fs_pos += len(e)
|
|
dirset = struct.pack('<I', len(order)) + b''.join(struct.pack('<I', rel[d]) for d in order)
|
|
for d in order:
|
|
kids = sorted(dirs[d], key=lambda k: k[0].lower())
|
|
parent = rel['/'.join(d.split('/')[:-1])] if d else 0
|
|
dirset += entry_head(parent, len(kids), os.path.basename(d))
|
|
dirset += b''.join(struct.pack('<I', rel[k[2]] if k[1] == 'd' else dirset_len + fpos[k[2]]) for k in kids)
|
|
dirset += b'\0' * 16
|
|
assert len(dirset) == dirset_len
|
|
fileset = struct.pack('<I', len(files)) + b''.join(struct.pack('<I', fpos[p]) for p, _ in files) + b''.join(fentries)
|
|
head = bytearray(template.d[:template.data_offset])
|
|
body = data + b'\0' * 16
|
|
dirset_at = template.data_offset + len(body)
|
|
index = dirset + fileset
|
|
struct.pack_into('<4I', head, 0x0c, dirset_at, dirset_at + len(dirset), len(index), template.data_offset)
|
|
struct.pack_into('<I', head, 0x34, priority)
|
|
head[0x40:0x50] = guid if guid else os.urandom(16)
|
|
struct.pack_into('<2I', head, 0x50, zlib.crc32(index) & 0xffffffff, zlib.crc32(data) & 0xffffffff)
|
|
return bytes(head) + body + index
|
|
|
|
|
|
def main():
|
|
cmd, path = sys.argv[1], sys.argv[2]
|
|
t = Tank(path)
|
|
if cmd == 'pack':
|
|
order = sorted(t.files, key=lambda f: f['offset'])
|
|
files = [[f['name'], t.read(f)] for f in order]
|
|
for arg in sys.argv[4:]:
|
|
name, src = arg.split('=', 1)
|
|
files = [f for f in files if f[0] != name]
|
|
if src:
|
|
files.append([name, open(src, 'rb').read()])
|
|
open(sys.argv[3], 'wb').write(pack(t, files))
|
|
print('wrote', sys.argv[3], [f[0] for f in files])
|
|
return
|
|
if cmd == 'header':
|
|
print('%s version %08X data %X priority %X product %s minimum %s built %04d-%02d-%02d%s' % (
|
|
t.kind, t.version, t.data_offset, t.priority, '.'.join(map(str, t.product)), '.'.join(map(str, t.minimum)),
|
|
t.built[0], t.built[1], t.built[3], (' "%s"' % t.title) if t.title else ''))
|
|
for f in t.files:
|
|
if cmd in ('list', 'header'):
|
|
print('%-40s %8d fmt=%d crc=%08X' % (f['path'], f['size'], f['format'], f['crc']))
|
|
elif cmd == 'extract':
|
|
data = t.read(f)
|
|
dest = os.path.join(sys.argv[3], *f['path'].split('/'))
|
|
os.makedirs(os.path.dirname(dest) or '.', exist_ok=True)
|
|
open(dest, 'wb').write(data)
|
|
ok = (zlib.crc32(data) & 0xffffffff) == f['crc']
|
|
print('%-40s %8d %s' % (f['path'], len(data), 'crc ok' if ok else 'crc differs'))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|