merge branch msys

srlp/msys/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.0.1'

srlp/msys/__main__.py

@@ -0,0 +1,52 @@
+#! /usr/bin/env python3
+
+import hashlib
+import os
+import sys
+
+import arguments
+import local
+import synchronization
+
+
+def build():
+    pass
+
+
+def check():
+    packages = []
+    broken = []
+    lo = local.Local()
+    for architecture in lo:
+        for subsystem in architecture:
+            for package in subsystem.catalog.packages.values():
+                packages.append((subsystem, package))
+    for index, items in enumerate(packages):
+        print('\r', index, '/', len(packages), '←', 'checked', end='')
+        subsystem, package = items
+        path = os.path.join(subsystem.path, package.filename)
+        with open(path, 'br') as f:
+            hash = hashlib.sha256(f.read()).hexdigest()
+            if hash != package.sha256sum:
+                broken.append(package)
+    print()
+    print('', str(len(broken)).rjust(len(str(len(packages)))),
+          '/', len(packages), '←', 'broken')
+
+
+def info():
+    print(local.Local())
+
+
+def sync():
+    sync = synchronization.Synchronization()
+    print(sync)
+    sync.run()
+
+
+def main():
+    getattr(sys.modules[__name__], arguments.action)()
+
+
+if __name__ == '__main__':
+    main()

srlp/msys/architecture.py

@@ -0,0 +1,36 @@
+import os
+
+import arguments
+import distribution
+import subsystem
+
+MAIN = 'x86_64'
+
+ARCHITECTURES = {
+    MAIN:   (64, [subsystem.MAIN, 'clang64', 'mingw64', 'ucrt64']),
+    'i686': (32, [subsystem.MAIN, 'clang32', 'mingw32']),
+}
+
+
+class Architecture:
+    def __init__(self, repository, name):
+        self.repository = repository
+        self.name = name
+        self.bits, subsystems = ARCHITECTURES[self.name]
+        self.distribution = distribution.Distribution(self)
+        self.subsystems = {s: subsystem.SubSystem(self, s)
+                           for s in [f if f == subsystem.MAIN
+                                     else f'{f}{self.bits}'
+                                     for f in arguments.subsystems]
+                           if s in subsystems}
+
+    def __iter__(self):
+        return self.subsystems.values().__iter__()
+
+    def __str__(self):
+        lines = [
+            f'      Name: {self.name}',
+            f'      Bits: {self.bits}',
+            f'Subsystems: {subsystem.name for subsystem in self}',
+        ]
+        return os.linesep.join(lines)

srlp/msys/arguments.py

@@ -0,0 +1,97 @@
+import argparse
+import os
+
+import architecture
+import remote as r
+import subsystem
+
+ACTION = 'action'
+ARCHITECTURES = 'architectures'
+COMPRESSION = 'compression'
+DIRECTORY = 'directory'
+FILESYSTEM = 'filesystem'
+REMOTE = 'remote'
+SUBSYSTEMS = 'subsystems'
+TEMPORARY = 'temporary'
+THREADS = 'threads'
+VERBOSE = 'verbose'
+
+
+class Formatter(argparse.RawTextHelpFormatter,
+                argparse.ArgumentDefaultsHelpFormatter):
+    pass
+
+
+def parse():
+    parser = argparse.ArgumentParser(formatter_class=Formatter)
+
+    parser.add_argument(f'-{VERBOSE[0]}', f'--{VERBOSE}', type=bool, help='''\
+verbose output
+''')
+
+    parser.add_argument(f'-{DIRECTORY[0]}', f'--{DIRECTORY}', type=str,
+                        help='''\
+msys repository's directory
+''')
+    parser.add_argument(f'--{TEMPORARY}', type=str, help='''\
+msys repository's temporary directory
+''')
+    parser.add_argument(f'-{THREADS[0]}', f'--{THREADS}', type=int, help='''\
+number of threads to use
+''')
+    parser.add_argument(f'{ACTION}', type=str, nargs='?',
+                        choices=['build', 'check', 'info', 'sync'], help='''\
+action to perform onto msys repository
+''')
+
+    sync = parser.add_argument_group('sync')
+    sync.add_argument(f'-{REMOTE[0]}', f'--{REMOTE}', type=str, help='''\
+msys remote repository's location
+''')
+    sync.add_argument(f'-{ARCHITECTURES[0]}', f'--{ARCHITECTURES}', type=str,
+                      nargs='+', choices=architecture.ARCHITECTURES.keys(),
+                      help='''\
+list of architectures to sync
+''')
+    sync.add_argument(f'-{SUBSYSTEMS[0]}', f'--{SUBSYSTEMS}', type=str,
+                      nargs='+', choices=subsystem.FAMILIES, help='''\
+list of subsystems to sync
+''')
+
+    build = parser.add_argument_group('build')
+    parser.add_argument(f'-{FILESYSTEM[0]}', f'--{FILESYSTEM}', type=str,
+                        help='''\
+directory containing modifications applying to filesystem
+''')
+    build.add_argument(f'-{COMPRESSION[0]}', f'--{COMPRESSION}', type=str,
+                       choices=['gz', 'xz', 'zst'], help='''\
+compression applying to archive
+''')
+
+    parser.set_defaults(
+        directory=os.curdir,
+        temporary='tmp',
+        threads=2,
+
+        action='info',
+
+        remote=r.MAIN,
+        architectures=[architecture.MAIN],
+        subsystems=[subsystem.MAIN, 'mingw'],
+
+        filesystem='fs',
+        compression='zst',
+    )
+
+    return vars(parser.parse_args())
+
+
+D = parse()
+
+action = D[ACTION]
+architectures = D[ARCHITECTURES]
+directory = D[DIRECTORY]
+remote = D[REMOTE]
+subsystems = D[SUBSYSTEMS]
+temporary = D[TEMPORARY]
+threads = D[THREADS]

srlp/msys/catalog.py

@@ -0,0 +1,36 @@
+import io
+import os
+import tarfile
+
+import package
+
+CATALOG = '.files'
+FILES = 'files'
+PACKAGE = 'desc'
+
+
+class Catalog:
+    def __init__(self, subsystem):
+        self.subsystem = subsystem
+        self.path = os.path.join(self.subsystem.path,
+                                 f'{self.subsystem.name}{CATALOG}')
+        self.load()
+
+    def load(self):
+        binary = self.subsystem.architecture.repository.get_file(self.path)
+        f = io.BytesIO(binary)
+        archive = tarfile.open(fileobj=f)
+        m = {}
+        packages = {}
+        for member in archive.getmembers():
+            directory, *file = member.name.split(os.sep)
+            if file:
+                d = m[directory]
+                d[file[0]] = archive.extractfile(member).read()
+                if len(d) == 2:
+                    p = package.Package(d[PACKAGE], d[FILES])
+                    packages[p.name] = p
+            else:
+                m[directory] = {}
+        archive.close()
+        self.packages = packages

srlp/msys/distribution.py

@@ -0,0 +1,23 @@
+import os
+
+ARCHIVE = '.tar.xz'
+DISTRIBUTION = 'distrib'
+
+
+class Distribution:
+    def __init__(self, architecture):
+        self.architecture = architecture
+        self.path = os.path.join(DISTRIBUTION, self.architecture.name)
+        self.load()
+
+    def load(self):
+        files = self.architecture.repository.get_files(self.path)
+        self.archives = [f for f in files if f.endswith(ARCHIVE)]
+        self.archive = self.archives[-1]
+
+    def __str__(self):
+        lines = [
+            f'Architecture: {self.architecture.name}',
+            f'        Path: {self.path}',
+        ]
+        return os.linesep.join(lines)

srlp/msys/file.py

@@ -0,0 +1,20 @@
+import os
+
+
+class File:
+    def __init__(self, remote, name, size, local, hash):
+        self.remote = remote
+        self.name = name
+        self.size = size
+        self.local = local
+        self.hash = hash
+
+    def __str__(self):
+        lines = [
+            f'Remote: {self.remote}',
+            f'  Name: {self.name}',
+            f'  Size: {self.size}',
+            f' Local: {self.local}',
+            f'  Hash: {self.hash}',
+        ]
+        return os.linesep.join(lines)

srlp/msys/hypertext.py

@@ -0,0 +1,27 @@
+import html.parser
+import requests
+
+CHARSET = 'u8'
+
+
+class Parser(html.parser.HTMLParser):
+    def __init__(self):
+        self.links = []
+        super().__init__()
+
+    def handle_starttag(self, tag, attributes):
+        if tag == 'a':
+            self.links.extend(
+                [v for k, v in attributes if k == 'href'])
+
+
+class HyperText:
+    def __init__(self, location):
+        self.location = location
+        self.load()
+
+    def load(self):
+        hypertext = requests.get(self.location).content.decode(CHARSET)
+        parser = Parser()
+        parser.feed(hypertext)
+        self.links = parser.links

srlp/msys/local.py

@@ -0,0 +1,31 @@
+import datetime
+import os
+
+import arguments
+import repository
+
+
+class Local(repository.Repository):
+    def __init__(self):
+        super().__init__(arguments.directory)
+        self.temporary = arguments.temporary
+
+    def get_file(self, path):
+        with open(os.path.join(self.location, path), 'br') as f:
+            return f.read()
+
+    def get_files(self, path):
+        *_, files = next(os.walk(os.path.join(self.location, path)))
+        return files
+
+    def get_temporary(self):
+        return os.path.join(self.temporary,
+                            datetime.datetime.now()
+                            .strftime('%Y%m%d%H%M%S'))
+
+    def __str__(self):
+        lines = [
+            super().__str__(),
+            f'Temporary: {self.temporary}',
+        ]
+        return os.linesep.join(lines)

srlp/msys/package.py

@@ -0,0 +1,28 @@
+import os
+
+CHARSET = 'u8'
+KEY = '%'
+SEPARATOR = f'{os.linesep}{os.linesep}'
+
+
+class Package:
+    def __init__(self, package, files):
+        for binary in [package, files]:
+            text = binary.decode(CHARSET).strip()
+            for item in text.split(SEPARATOR):
+                line, *lines = item.split(os.linesep)
+                key = line.split(KEY)[1].lower()
+                if len(lines) == 1:
+                    value = lines[0]
+                else:
+                    value = lines
+                setattr(self, key, value)
+
+    def __str__(self):
+        lines = [
+            f'    Name: {self.name}',
+            f'FileName: {self.filename}',
+            f'    Size: {self.csize}',
+            f'    Hash: {self.sha256sum}',
+        ]
+        return os.linesep.join(lines)

srlp/msys/readme.md

@@ -0,0 +1,32 @@
+# Minimal System Repository
+
+A tool to handle local/remote msys/mingw repositories.
+
+## Features
+
+* [ ] display information
+* [ ] synchronize data
+    * [ ] fetch remote catalogs
+    * [ ] download packages
+        * [ ] single threading
+        * [ ] multi threading
+    * [ ] generate signatures from catalogs
+* [ ] check integrity
+    * [ ] catalogs
+    * [ ] packages
+        * [x] single threading
+        * [ ] multi threading
+        * [ ] progress bars
+* [ ] build archive
+    * [ ] get base file system
+        * [ ] extract from distribution
+        * [ ] bootstrap mintty, msys, pacman
+    * [ ] apply configuration
+    * [ ] archive directory
+* [ ] build dependency graph
+    * [ ] generate graphviz diagram
+    * [ ] render final image
+
+## Info
+
+* pacman needs .db catalog, but .files seems optional

srlp/msys/remote.py

@@ -0,0 +1,25 @@
+import os
+import requests
+
+import arguments
+import hypertext
+import repository
+
+MAIN = 'https://repo.msys2.org'
+
+
+class Remote(repository.Repository):
+    def __init__(self):
+        super().__init__(arguments.remote)
+
+    def get_file(self, path):
+        return requests.get(os.path.join(self.location, path)).content
+
+    def get_files(self, path):
+        return hypertext.HyperText(os.path.join(self.location, path)).links
+
+    def __str__(self):
+        lines = [
+            super().__str__(),
+        ]
+        return os.linesep.join(lines)

srlp/msys/repository.py

@@ -0,0 +1,21 @@
+import os
+
+import architecture
+import arguments
+
+
+class Repository:
+    def __init__(self, location):
+        self.location = location
+        self.architectures = [architecture.Architecture(self, a)
+                              for a in arguments.architectures]
+
+    def __iter__(self):
+        return self.architectures.__iter__()
+
+    def __str__(self):
+        lines = [
+            f'     Location: {self.location}',
+            f'Architectures: {[architecture.name for architecture in self]}',
+        ]
+        return os.linesep.join(lines)

srlp/msys/subsystem.py

@@ -0,0 +1,32 @@
+import os
+
+import catalog
+
+CRT = 'mingw'
+MAIN = 'msys'
+
+FAMILIES = [MAIN, 'clang', 'mingw', 'ucrt']
+
+
+class SubSystem:
+    def __init__(self, architecture, name):
+        self.architecture = architecture
+        self.name = name
+        # path
+        list = []
+        if self.name != MAIN:
+            list.append(CRT)
+        list.append(self.name)
+        if self.name == MAIN:
+            list.append(self.architecture.name)
+        self.path = os.path.join(*list)
+        # catalog
+        self.catalog = catalog.Catalog(self)
+
+    def __str__(self):
+        lines = [
+            f'Architecture: {self.architecture.name}',
+            f'        Name: {self.name}',
+            f'        Path: {self.path}',
+        ]
+        return os.linesep.join(lines)

srlp/msys/synchronization.py

@@ -0,0 +1,44 @@
+import os
+import shutil
+
+import arguments
+import file
+import local
+import remote
+
+
+class Synchronization:
+    def __init__(self):
+        self.remote = remote.Remote()
+        self.repository = local.Local()
+        self.temporary = self.repository.get_temporary()
+        self.threads = arguments.threads
+
+    def run(self):
+        for architecture in self.remote:
+            for subsystem in architecture:
+                for _, package in sorted(subsystem.catalog.packages.items()):
+                    f = file.File(
+                        os.path.join(self.remote.location, subsystem.path),
+                        package.name,
+                        package.csize,
+                        os.path.join(self.repository.location, subsystem.path),
+                        package.sha256sum,
+                    )
+                    print()
+                    print(f)
+        tmp = os.path.join(self.repository.location,
+                           self.repository.get_temporary())
+        os.makedirs(tmp)
+        # clean temporary directory
+        shutil.rmtree(tmp)
+
+    def __str__(self):
+        lines = [
+            str(self.remote),
+            str(),
+            str(self.repository),
+            str(),
+            f'Temporary: {self.temporary}',
+        ]
+        return os.linesep.join(lines)