diff --git a/srlp/msys/__init__.py b/srlp/msys/__init__.py new file mode 100644 index 0000000..b8023d8 --- /dev/null +++ b/srlp/msys/__init__.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/srlp/msys/__main__.py b/srlp/msys/__main__.py new file mode 100755 index 0000000..faea166 --- /dev/null +++ b/srlp/msys/__main__.py @@ -0,0 +1,52 @@ +#! /usr/bin/env python3 + +import hashlib +import os +import sys + +import arguments +import local +import synchronization + + +def build(): + pass + + +def check(): + packages = [] + broken = [] + lo = local.Local() + for architecture in lo: + for subsystem in architecture: + for package in subsystem.catalog.packages.values(): + packages.append((subsystem, package)) + for index, items in enumerate(packages): + print('\r', index, '/', len(packages), '←', 'checked', end='') + subsystem, package = items + path = os.path.join(subsystem.path, package.filename) + with open(path, 'br') as f: + hash = hashlib.sha256(f.read()).hexdigest() + if hash != package.sha256sum: + broken.append(package) + print() + print('', str(len(broken)).rjust(len(str(len(packages)))), + '/', len(packages), '←', 'broken') + + +def info(): + print(local.Local()) + + +def sync(): + sync = synchronization.Synchronization() + print(sync) + sync.run() + + +def main(): + getattr(sys.modules[__name__], arguments.action)() + + +if __name__ == '__main__': + main() diff --git a/srlp/msys/architecture.py b/srlp/msys/architecture.py new file mode 100644 index 0000000..78b9fb2 --- /dev/null +++ b/srlp/msys/architecture.py @@ -0,0 +1,36 @@ +import os + +import arguments +import distribution +import subsystem + +MAIN = 'x86_64' + +ARCHITECTURES = { + MAIN: (64, [subsystem.MAIN, 'clang64', 'mingw64', 'ucrt64']), + 'i686': (32, [subsystem.MAIN, 'clang32', 'mingw32']), +} + + +class Architecture: + def __init__(self, repository, name): + self.repository = repository + self.name = name + self.bits, subsystems = ARCHITECTURES[self.name] + self.distribution = distribution.Distribution(self) + self.subsystems = {s: subsystem.SubSystem(self, s) + for s in [f if f == subsystem.MAIN + else f'{f}{self.bits}' + for f in arguments.subsystems] + if s in subsystems} + + def __iter__(self): + return self.subsystems.values().__iter__() + + def __str__(self): + lines = [ + f' Name: {self.name}', + f' Bits: {self.bits}', + f'Subsystems: {subsystem.name for subsystem in self}', + ] + return os.linesep.join(lines) diff --git a/srlp/msys/arguments.py b/srlp/msys/arguments.py new file mode 100644 index 0000000..84fd24b --- /dev/null +++ b/srlp/msys/arguments.py @@ -0,0 +1,97 @@ +import argparse +import os + +import architecture +import remote as r +import subsystem + +ACTION = 'action' +ARCHITECTURES = 'architectures' +COMPRESSION = 'compression' +DIRECTORY = 'directory' +FILESYSTEM = 'filesystem' +REMOTE = 'remote' +SUBSYSTEMS = 'subsystems' +TEMPORARY = 'temporary' +THREADS = 'threads' +VERBOSE = 'verbose' + + +class Formatter(argparse.RawTextHelpFormatter, + argparse.ArgumentDefaultsHelpFormatter): + pass + + +def parse(): + parser = argparse.ArgumentParser(formatter_class=Formatter) + + parser.add_argument(f'-{VERBOSE[0]}', f'--{VERBOSE}', type=bool, help='''\ +verbose output +''') + + parser.add_argument(f'-{DIRECTORY[0]}', f'--{DIRECTORY}', type=str, + help='''\ +msys repository's directory +''') + parser.add_argument(f'--{TEMPORARY}', type=str, help='''\ +msys repository's temporary directory +''') + parser.add_argument(f'-{THREADS[0]}', f'--{THREADS}', type=int, help='''\ +number of threads to use +''') + parser.add_argument(f'{ACTION}', type=str, nargs='?', + choices=['build', 'check', 'info', 'sync'], help='''\ +action to perform onto msys repository +''') + + sync = parser.add_argument_group('sync') + sync.add_argument(f'-{REMOTE[0]}', f'--{REMOTE}', type=str, help='''\ +msys remote repository's location +''') + sync.add_argument(f'-{ARCHITECTURES[0]}', f'--{ARCHITECTURES}', type=str, + nargs='+', choices=architecture.ARCHITECTURES.keys(), + help='''\ +list of architectures to sync +''') + sync.add_argument(f'-{SUBSYSTEMS[0]}', f'--{SUBSYSTEMS}', type=str, + nargs='+', choices=subsystem.FAMILIES, help='''\ +list of subsystems to sync +''') + + build = parser.add_argument_group('build') + parser.add_argument(f'-{FILESYSTEM[0]}', f'--{FILESYSTEM}', type=str, + help='''\ +directory containing modifications applying to filesystem +''') + build.add_argument(f'-{COMPRESSION[0]}', f'--{COMPRESSION}', type=str, + choices=['gz', 'xz', 'zst'], help='''\ +compression applying to archive +''') + + parser.set_defaults( + directory=os.curdir, + temporary='tmp', + threads=2, + + action='info', + + remote=r.MAIN, + architectures=[architecture.MAIN], + subsystems=[subsystem.MAIN, 'mingw'], + + filesystem='fs', + compression='zst', + ) + + return vars(parser.parse_args()) + + +D = parse() + +action = D[ACTION] +architectures = D[ARCHITECTURES] +directory = D[DIRECTORY] +remote = D[REMOTE] +subsystems = D[SUBSYSTEMS] +temporary = D[TEMPORARY] +threads = D[THREADS] diff --git a/srlp/msys/catalog.py b/srlp/msys/catalog.py new file mode 100644 index 0000000..a096d34 --- /dev/null +++ b/srlp/msys/catalog.py @@ -0,0 +1,36 @@ +import io +import os +import tarfile + +import package + +CATALOG = '.files' +FILES = 'files' +PACKAGE = 'desc' + + +class Catalog: + def __init__(self, subsystem): + self.subsystem = subsystem + self.path = os.path.join(self.subsystem.path, + f'{self.subsystem.name}{CATALOG}') + self.load() + + def load(self): + binary = self.subsystem.architecture.repository.get_file(self.path) + f = io.BytesIO(binary) + archive = tarfile.open(fileobj=f) + m = {} + packages = {} + for member in archive.getmembers(): + directory, *file = member.name.split(os.sep) + if file: + d = m[directory] + d[file[0]] = archive.extractfile(member).read() + if len(d) == 2: + p = package.Package(d[PACKAGE], d[FILES]) + packages[p.name] = p + else: + m[directory] = {} + archive.close() + self.packages = packages diff --git a/srlp/msys/distribution.py b/srlp/msys/distribution.py new file mode 100644 index 0000000..64be502 --- /dev/null +++ b/srlp/msys/distribution.py @@ -0,0 +1,23 @@ +import os + +ARCHIVE = '.tar.xz' +DISTRIBUTION = 'distrib' + + +class Distribution: + def __init__(self, architecture): + self.architecture = architecture + self.path = os.path.join(DISTRIBUTION, self.architecture.name) + self.load() + + def load(self): + files = self.architecture.repository.get_files(self.path) + self.archives = [f for f in files if f.endswith(ARCHIVE)] + self.archive = self.archives[-1] + + def __str__(self): + lines = [ + f'Architecture: {self.architecture.name}', + f' Path: {self.path}', + ] + return os.linesep.join(lines) diff --git a/srlp/msys/file.py b/srlp/msys/file.py new file mode 100644 index 0000000..8100b1e --- /dev/null +++ b/srlp/msys/file.py @@ -0,0 +1,20 @@ +import os + + +class File: + def __init__(self, remote, name, size, local, hash): + self.remote = remote + self.name = name + self.size = size + self.local = local + self.hash = hash + + def __str__(self): + lines = [ + f'Remote: {self.remote}', + f' Name: {self.name}', + f' Size: {self.size}', + f' Local: {self.local}', + f' Hash: {self.hash}', + ] + return os.linesep.join(lines) diff --git a/srlp/msys/hypertext.py b/srlp/msys/hypertext.py new file mode 100644 index 0000000..fad9918 --- /dev/null +++ b/srlp/msys/hypertext.py @@ -0,0 +1,27 @@ +import html.parser +import requests + +CHARSET = 'u8' + + +class Parser(html.parser.HTMLParser): + def __init__(self): + self.links = [] + super().__init__() + + def handle_starttag(self, tag, attributes): + if tag == 'a': + self.links.extend( + [v for k, v in attributes if k == 'href']) + + +class HyperText: + def __init__(self, location): + self.location = location + self.load() + + def load(self): + hypertext = requests.get(self.location).content.decode(CHARSET) + parser = Parser() + parser.feed(hypertext) + self.links = parser.links diff --git a/srlp/msys/local.py b/srlp/msys/local.py new file mode 100644 index 0000000..d1af64f --- /dev/null +++ b/srlp/msys/local.py @@ -0,0 +1,31 @@ +import datetime +import os + +import arguments +import repository + + +class Local(repository.Repository): + def __init__(self): + super().__init__(arguments.directory) + self.temporary = arguments.temporary + + def get_file(self, path): + with open(os.path.join(self.location, path), 'br') as f: + return f.read() + + def get_files(self, path): + *_, files = next(os.walk(os.path.join(self.location, path))) + return files + + def get_temporary(self): + return os.path.join(self.temporary, + datetime.datetime.now() + .strftime('%Y%m%d%H%M%S')) + + def __str__(self): + lines = [ + super().__str__(), + f'Temporary: {self.temporary}', + ] + return os.linesep.join(lines) diff --git a/srlp/msys/package.py b/srlp/msys/package.py new file mode 100644 index 0000000..c768be3 --- /dev/null +++ b/srlp/msys/package.py @@ -0,0 +1,28 @@ +import os + +CHARSET = 'u8' +KEY = '%' +SEPARATOR = f'{os.linesep}{os.linesep}' + + +class Package: + def __init__(self, package, files): + for binary in [package, files]: + text = binary.decode(CHARSET).strip() + for item in text.split(SEPARATOR): + line, *lines = item.split(os.linesep) + key = line.split(KEY)[1].lower() + if len(lines) == 1: + value = lines[0] + else: + value = lines + setattr(self, key, value) + + def __str__(self): + lines = [ + f' Name: {self.name}', + f'FileName: {self.filename}', + f' Size: {self.csize}', + f' Hash: {self.sha256sum}', + ] + return os.linesep.join(lines) diff --git a/srlp/msys/readme.md b/srlp/msys/readme.md new file mode 100644 index 0000000..6939b25 --- /dev/null +++ b/srlp/msys/readme.md @@ -0,0 +1,32 @@ +# Minimal System Repository + +A tool to handle local/remote msys/mingw repositories. + +## Features + +* [ ] display information +* [ ] synchronize data + * [ ] fetch remote catalogs + * [ ] download packages + * [ ] single threading + * [ ] multi threading + * [ ] generate signatures from catalogs +* [ ] check integrity + * [ ] catalogs + * [ ] packages + * [x] single threading + * [ ] multi threading + * [ ] progress bars +* [ ] build archive + * [ ] get base file system + * [ ] extract from distribution + * [ ] bootstrap mintty, msys, pacman + * [ ] apply configuration + * [ ] archive directory +* [ ] build dependency graph + * [ ] generate graphviz diagram + * [ ] render final image + +## Info + +* pacman needs .db catalog, but .files seems optional diff --git a/srlp/msys/remote.py b/srlp/msys/remote.py new file mode 100644 index 0000000..47bb8ce --- /dev/null +++ b/srlp/msys/remote.py @@ -0,0 +1,25 @@ +import os +import requests + +import arguments +import hypertext +import repository + +MAIN = 'https://repo.msys2.org' + + +class Remote(repository.Repository): + def __init__(self): + super().__init__(arguments.remote) + + def get_file(self, path): + return requests.get(os.path.join(self.location, path)).content + + def get_files(self, path): + return hypertext.HyperText(os.path.join(self.location, path)).links + + def __str__(self): + lines = [ + super().__str__(), + ] + return os.linesep.join(lines) diff --git a/srlp/msys/repository.py b/srlp/msys/repository.py new file mode 100644 index 0000000..c320d98 --- /dev/null +++ b/srlp/msys/repository.py @@ -0,0 +1,21 @@ +import os + +import architecture +import arguments + + +class Repository: + def __init__(self, location): + self.location = location + self.architectures = [architecture.Architecture(self, a) + for a in arguments.architectures] + + def __iter__(self): + return self.architectures.__iter__() + + def __str__(self): + lines = [ + f' Location: {self.location}', + f'Architectures: {[architecture.name for architecture in self]}', + ] + return os.linesep.join(lines) diff --git a/srlp/msys/subsystem.py b/srlp/msys/subsystem.py new file mode 100644 index 0000000..e22933f --- /dev/null +++ b/srlp/msys/subsystem.py @@ -0,0 +1,32 @@ +import os + +import catalog + +CRT = 'mingw' +MAIN = 'msys' + +FAMILIES = [MAIN, 'clang', 'mingw', 'ucrt'] + + +class SubSystem: + def __init__(self, architecture, name): + self.architecture = architecture + self.name = name + # path + list = [] + if self.name != MAIN: + list.append(CRT) + list.append(self.name) + if self.name == MAIN: + list.append(self.architecture.name) + self.path = os.path.join(*list) + # catalog + self.catalog = catalog.Catalog(self) + + def __str__(self): + lines = [ + f'Architecture: {self.architecture.name}', + f' Name: {self.name}', + f' Path: {self.path}', + ] + return os.linesep.join(lines) diff --git a/srlp/msys/synchronization.py b/srlp/msys/synchronization.py new file mode 100644 index 0000000..9339efc --- /dev/null +++ b/srlp/msys/synchronization.py @@ -0,0 +1,44 @@ +import os +import shutil + +import arguments +import file +import local +import remote + + +class Synchronization: + def __init__(self): + self.remote = remote.Remote() + self.repository = local.Local() + self.temporary = self.repository.get_temporary() + self.threads = arguments.threads + + def run(self): + for architecture in self.remote: + for subsystem in architecture: + for _, package in sorted(subsystem.catalog.packages.items()): + f = file.File( + os.path.join(self.remote.location, subsystem.path), + package.name, + package.csize, + os.path.join(self.repository.location, subsystem.path), + package.sha256sum, + ) + print() + print(f) + tmp = os.path.join(self.repository.location, + self.repository.get_temporary()) + os.makedirs(tmp) + # clean temporary directory + shutil.rmtree(tmp) + + def __str__(self): + lines = [ + str(self.remote), + str(), + str(self.repository), + str(), + f'Temporary: {self.temporary}', + ] + return os.linesep.join(lines)