From d3e79120cbd952c3334b2313290ba82d2f0e0908 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Mon, 15 Mar 2021 02:21:41 +0300 Subject: [PATCH] docstrings everywhere --- src/ahriman/application/ahriman.py | 29 +++++ src/ahriman/application/application.py | 54 +++++++++ src/ahriman/application/lock.py | 33 ++++++ src/ahriman/core/alpm/pacman.py | 12 ++ src/ahriman/core/alpm/repo.py | 30 ++++- src/ahriman/core/build_tools/task.py | 30 +++++ src/ahriman/core/configuration.py | 38 ++++++- src/ahriman/core/exceptions.py | 52 +++++++++ src/ahriman/core/report/html.py | 34 +++++- src/ahriman/core/report/report.py | 22 ++++ src/ahriman/core/repository.py | 88 +++++++++++++-- src/ahriman/core/sign/gpg.py | 45 +++++++- src/ahriman/core/tree.py | 81 +++++++++----- src/ahriman/core/upload/rsync.py | 13 +++ src/ahriman/core/upload/s3.py | 13 +++ src/ahriman/core/upload/uploader.py | 22 ++++ src/ahriman/core/util.py | 14 +++ src/ahriman/core/watcher/client.py | 103 ++++++++---------- src/ahriman/core/watcher/watcher.py | 27 +++++ src/ahriman/core/watcher/web_client.py | 97 +++++++++++++++++ src/ahriman/models/build_status.py | 22 ++++ src/ahriman/models/package.py | 67 +++++++++++- src/ahriman/models/report_settings.py | 10 ++ src/ahriman/models/repository_paths.py | 9 ++ src/ahriman/models/sign_settings.py | 11 ++ src/ahriman/models/upload_settings.py | 11 ++ src/ahriman/version.py | 2 +- .../web/middlewares/exception_handler.py | 5 + src/ahriman/web/routes.py | 27 ++++- src/ahriman/web/views/base.py | 7 +- src/ahriman/web/views/index.py | 15 +++ src/ahriman/web/views/package.py | 34 +++++- src/ahriman/web/views/packages.py | 7 ++ src/ahriman/web/web.py | 43 ++++++-- 34 files changed, 980 insertions(+), 127 deletions(-) create mode 100644 src/ahriman/core/watcher/web_client.py diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 400fba9a..7f6b9256 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -27,29 +27,54 @@ from ahriman.core.configuration import Configuration def add(args: argparse.Namespace) -> None: + ''' + add packages callback + :param args: command line args + ''' Application.from_args(args).add(args.package, args.without_dependencies) def rebuild(args: argparse.Namespace) -> None: + ''' + world rebuild callback + :param args: command line args + ''' app = Application.from_args(args) packages = app.repository.packages() app.update(packages) def remove(args: argparse.Namespace) -> None: + ''' + remove packages callback + :param args: command line args + ''' Application.from_args(args).remove(args.package) def report(args: argparse.Namespace) -> None: + ''' + generate report callback + :param args: command line args + ''' Application.from_args(args).report(args.target) def sync(args: argparse.Namespace) -> None: + ''' + sync to remote server callback + :param args: command line args + ''' Application.from_args(args).sync(args.target) def update(args: argparse.Namespace) -> None: + ''' + update packages callback + :param args: command line args + ''' app = Application.from_args(args) + # typing workaround log_fn = lambda line: print(line) if args.dry_run else app.logger.info(line) packages = app.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn) if args.dry_run: @@ -58,6 +83,10 @@ def update(args: argparse.Namespace) -> None: def web(args: argparse.Namespace) -> None: + ''' + web server callback + :param args: command line args + ''' from ahriman.web.web import run_server, setup_service config = Configuration.from_path(args.config) app = setup_service(args.architecture, config) diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index 7f808639..5c21fa09 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -34,8 +34,20 @@ from ahriman.models.package import Package class Application: + ''' + base application class + :ivar architecture: repository architecture + :ivar config: configuration instance + :ivar logger: application logger + :ivar repository: repository instance + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' self.logger = logging.getLogger('root') self.config = config self.architecture = architecture @@ -43,10 +55,19 @@ class Application: @classmethod def from_args(cls: Type[Application], args: argparse.Namespace) -> Application: + ''' + constructor which has to be used to build instance from command line args + :param args: command line args + :return: application instance + ''' config = Configuration.from_path(args.config) return cls(args.architecture, config) def _known_packages(self) -> Set[str]: + ''' + load packages from repository and pacman repositories + :return: list of known packages + ''' known_packages: Set[str] = set() # local set for package in self.repository.packages(): @@ -55,11 +76,23 @@ class Application: return known_packages def _finalize(self) -> None: + ''' + generate report and sync to remote server + ''' self.report() self.sync() def get_updates(self, filter_packages: List[str], no_aur: bool, no_manual: bool, no_vcs: bool, log_fn: Callable[[str], None]) -> List[Package]: + ''' + get list of packages to run update process + :param filter_packages: do not check every package just specified in the list + :param no_aur: do not check for aur updates + :param no_manual: do not check for manual updates + :param no_vcs: do not check VCS packages + :param log_fn: logger function to log updates + :return: list of out-of-dated packages + ''' updates = [] if not no_aur: @@ -73,6 +106,11 @@ class Application: return updates def add(self, names: Iterable[str], without_dependencies: bool) -> None: + ''' + add packages for the next build + :param names: list of package bases to add + :param without_dependencies: if set, dependency check will be disabled + ''' known_packages = self._known_packages() def add_manual(name: str) -> str: @@ -102,18 +140,34 @@ class Application: process_single(name) def remove(self, names: Iterable[str]) -> None: + ''' + remove packages from repository + :param names: list of packages (either base or name) to remove + ''' self.repository.process_remove(names) self._finalize() def report(self, target: Optional[Iterable[str]] = None) -> None: + ''' + generate report + :param target: list of targets to run (e.g. html) + ''' targets = target or None self.repository.process_report(targets) def sync(self, target: Optional[Iterable[str]] = None) -> None: + ''' + sync to remote server + :param target: list of targets to run (e.g. s3) + ''' targets = target or None self.repository.process_sync(targets) def update(self, updates: Iterable[Package]) -> None: + ''' + run package updates + :param updates: list of packages to update + ''' def process_update(paths: Iterable[str]) -> None: self.repository.process_update(paths) self._finalize() diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py index 41a95ca0..f2a50c73 100644 --- a/src/ahriman/application/lock.py +++ b/src/ahriman/application/lock.py @@ -28,12 +28,29 @@ from ahriman.core.exceptions import DuplicateRun class Lock: + ''' + wrapper for application lock file + :ivar force: remove lock file on start if any + :ivar path: path to lock file if any + ''' def __init__(self, path: Optional[str], architecture: str, force: bool) -> None: + ''' + default constructor + :param path: optional path to lock file, if empty no file lock will be used + :param architecture: repository architecture + :param force: remove lock file on start if any + ''' self.path = f'{path}_{architecture}' if path is not None else None self.force = force def __enter__(self) -> Lock: + ''' + default workflow is the following + * remove lock file if force flag is set + * check if there is lock file + * create lock file + ''' if self.force: self.remove() self.check() @@ -42,21 +59,37 @@ class Lock: def __exit__(self, exc_type: Optional[Type[Exception]], exc_val: Optional[Exception], exc_tb: TracebackType) -> Literal[False]: + ''' + remove lock file when done + :param exc_type: exception type name if any + :param exc_val: exception raised if any + :param exc_tb: exception traceback if any + :return: always False (do not suppress any exception) + ''' self.remove() return False def check(self) -> None: + ''' + check if lock file exists, raise exception if it does + ''' if self.path is None: return if os.path.exists(self.path): raise DuplicateRun() def create(self) -> None: + ''' + create lock file + ''' if self.path is None: return open(self.path, 'w').close() def remove(self) -> None: + ''' + remove lock file + ''' if self.path is None: return if os.path.exists(self.path): diff --git a/src/ahriman/core/alpm/pacman.py b/src/ahriman/core/alpm/pacman.py index c0385a80..986dc6d4 100644 --- a/src/ahriman/core/alpm/pacman.py +++ b/src/ahriman/core/alpm/pacman.py @@ -24,8 +24,16 @@ from ahriman.core.configuration import Configuration class Pacman: + ''' + alpm wrapper + :ivar handle: pyalpm root `Handle` + ''' def __init__(self, config: Configuration) -> None: + ''' + default constructor + :param config: configuration instance + ''' root = config.get('alpm', 'root') pacman_root = config.get('alpm', 'database') self.handle = Handle(root, pacman_root) @@ -33,6 +41,10 @@ class Pacman: self.handle.register_syncdb(repository, 0) # 0 is pgp_level def all_packages(self) -> List[str]: + ''' + get list of packages known for alpm + :return: list of package names + ''' result: Set[str] = set() for database in self.handle.get_syncdbs(): result.update({package.name for package in database.pkgcache}) diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py index 7619c221..9c2ba319 100644 --- a/src/ahriman/core/alpm/repo.py +++ b/src/ahriman/core/alpm/repo.py @@ -28,8 +28,21 @@ from ahriman.models.repository_paths import RepositoryPaths class Repo: + ''' + repo-add and repo-remove wrapper + :ivar logger: class logger + :ivar name: repository name + :ivar paths: repository paths instance + :ivar sign_args: additional args which have to be used to sign repository archive + ''' def __init__(self, name: str, paths: RepositoryPaths, sign_args: List[str]) -> None: + ''' + default constructor + :param name: repository name + :param paths: repository paths instance + :param sign_args: additional args which have to be used to sign repository archive + ''' self.logger = logging.getLogger('build_details') self.name = name self.paths = paths @@ -37,19 +50,32 @@ class Repo: @property def repo_path(self) -> str: + ''' + :return: path to repository database + ''' return os.path.join(self.paths.repository, f'{self.name}.db.tar.gz') def add(self, path: str) -> None: + ''' + add new package to repository + :param path: path to archive to add + ''' check_output( 'repo-add', *self.sign_args, '-R', self.repo_path, path, exception=BuildFailed(path), cwd=self.paths.repository, logger=self.logger) - def remove(self, prefix: str, package: str) -> None: - for fn in filter(lambda f: f.startswith(prefix), os.listdir(self.paths.repository)): + def remove(self, package: str) -> None: + ''' + remove package from repository + :param package: package name to remove + ''' + # remove package and signature (if any) from filesystem + for fn in filter(lambda f: f.startswith(package), os.listdir(self.paths.repository)): full_path = os.path.join(self.paths.repository, fn) os.remove(full_path) + # remove package from registry check_output( 'repo-remove', *self.sign_args, self.repo_path, package, exception=BuildFailed(package), diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index 4c9296c2..6174b33d 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -31,8 +31,22 @@ from ahriman.models.repository_paths import RepositoryPaths class Task: + ''' + base package build task + :ivar build_logger: logger for build process + :ivar logger: class logger + :ivar package: package definitions + :ivar paths: repository paths instance + ''' def __init__(self, package: Package, architecture: str, config: Configuration, paths: RepositoryPaths) -> None: + ''' + default constructor + :param package: package definitions + :param architecture: repository architecture + :param config: configuration instance + :param paths: repository paths instance + ''' self.logger = logging.getLogger('builder') self.build_logger = logging.getLogger('build_details') self.package = package @@ -46,14 +60,26 @@ class Task: @property def git_path(self) -> str: + ''' + :return: path to clone package from git + ''' return os.path.join(self.paths.sources, self.package.base) @staticmethod def fetch(local: str, remote: str) -> None: + ''' + fetch package from git + :param local: local path to fetch + :param remote: remote target (from where to fetch) + ''' shutil.rmtree(local, ignore_errors=True) # remove in case if file exists check_output('git', 'clone', remote, local, exception=None) def build(self) -> List[str]: + ''' + run package build + :return: paths of produced packages + ''' cmd = [self.build_command, '-r', self.paths.chroot] cmd.extend(self.archbuild_flags) cmd.extend(['--'] + self.makechrootpkg_flags) @@ -72,5 +98,9 @@ class Task: cwd=self.git_path).splitlines() def clone(self, path: Optional[str] = None) -> None: + ''' + fetch package from git + :param path: optional local path to fetch. If not set default path will be used + ''' git_path = path or self.git_path return Task.fetch(git_path, self.package.git_url) diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index f76282cf..311ea932 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -26,40 +26,73 @@ from logging.config import fileConfig from typing import List, Optional, Type -# built-in configparser extension class Configuration(configparser.RawConfigParser): + ''' + extension for built-in configuration parser + :ivar path: path to root configuration file + ''' def __init__(self) -> None: + ''' + default constructor + ''' configparser.RawConfigParser.__init__(self, allow_no_value=True) self.path: Optional[str] = None @property def include(self) -> str: + ''' + :return: path to directory with configuration includes + ''' return self.get('settings', 'include') @classmethod def from_path(cls: Type[Configuration], path: str) -> Configuration: + ''' + constructor with full object initialization + :param path: path to root configuration file + :return: configuration instance + ''' config = cls() config.load(path) config.load_logging() return config def getlist(self, section: str, key: str) -> List[str]: + ''' + get space separated string list option + :param section: section name + :param key: key name + :return: list of string if option is set, empty list otherwise + ''' raw = self.get(section, key, fallback=None) if not raw: # empty string or none return [] return raw.split() def get_section_name(self, prefix: str, suffix: str) -> str: + ''' + check if there is `prefix`_`suffix` section and return it on success. Return `prefix` otherwise + :param prefix: section name prefix + :param suffix: section name suffix (e.g. architecture name) + :return: found section name + ''' probe = f'{prefix}_{suffix}' return probe if self.has_section(probe) else prefix def load(self, path: str) -> None: + ''' + fully load configuration + :param path: path to root configuration file + ''' self.path = path self.read(self.path) self.load_includes() def load_includes(self) -> None: + ''' + load configuration includes + ''' try: for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))): self.read(os.path.join(self.include, conf)) @@ -67,4 +100,7 @@ class Configuration(configparser.RawConfigParser): pass def load_logging(self) -> None: + ''' + setup logging settings from configuration + ''' fileConfig(self.get('settings', 'logging')) diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index 5aca92d0..bf7c9bfc 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -21,35 +21,87 @@ from typing import Any class BuildFailed(Exception): + ''' + base exception for failed builds + ''' + def __init__(self, package: str) -> None: + ''' + default constructor + :param package: package base raised exception + ''' Exception.__init__(self, f'Package {package} build failed, check logs for details') class DuplicateRun(Exception): + ''' + exception which will be raised if there is another application instance + ''' + def __init__(self) -> None: + ''' + default constructor + ''' Exception.__init__(self, 'Another application instance is run') class InitializeException(Exception): + ''' + base service initialization exception + ''' + def __init__(self) -> None: + ''' + default constructor + ''' Exception.__init__(self, 'Could not load service') class InvalidOption(Exception): + ''' + exception which will be raised on configuration errors + ''' + def __init__(self, value: Any) -> None: + ''' + default constructor + :param value: option value + ''' Exception.__init__(self, f'Invalid or unknown option value `{value}`') class InvalidPackageInfo(Exception): + ''' + exception which will be raised on package load errors + ''' + def __init__(self, details: Any) -> None: + ''' + default constructor + :param details: error details + ''' Exception.__init__(self, f'There are errors during reading package information: `{details}`') class ReportFailed(Exception): + ''' + report generation exception + ''' + def __init__(self) -> None: + ''' + default constructor + ''' Exception.__init__(self, 'Report failed') class SyncFailed(Exception): + ''' + remote synchronization exception + ''' + def __init__(self) -> None: + ''' + default constructor + ''' Exception.__init__(self, 'Sync failed') \ No newline at end of file diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index fd820624..e282ab61 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -29,8 +29,34 @@ from ahriman.models.sign_settings import SignSettings class HTML(Report): + ''' + html report generator + + It uses jinja2 templates for report generation, the following variables are allowed: + + homepage - link to homepage, string, optional + link_path - prefix fo packages to download, string, required + has_package_signed - True in case if package sign enabled, False otherwise, required + has_repo_signed - True in case if repository database sign enabled, False otherwise, required + packages - sorted list of packages properties: filename, name, version. Required + pgp_key - default PGP key ID, string, optional + repository - repository name, string, required + + :ivar homepage: homepage link if any (for footer) + :ivar link_path: prefix fo packages to download + :ivar name: repository name + :ivar pgp_key: default PGP key + :ivar report_path: output path to html report + :ivar sign_targets: targets to sign enabled in configuration + :ivar tempate_path: path to directory with jinja templates + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' Report.__init__(self, architecture, config) section = config.get_section_name('html', architecture) self.report_path = config.get(section, 'path') @@ -39,13 +65,17 @@ class HTML(Report): # base template vars self.homepage = config.get(section, 'homepage', fallback=None) - self.repository = config.get('repository', 'name') + self.name = config.get('repository', 'name') sign_section = config.get_section_name('sign', architecture) self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist(sign_section, 'target')] self.pgp_key = config.get(sign_section, 'key') if self.sign_targets else None def generate(self, packages: Iterable[Package]) -> None: + ''' + generate report for the specified packages + :param packages: list of packages to generate report + ''' # idea comes from https://stackoverflow.com/a/38642558 templates_dir, template_name = os.path.split(self.template_path) loader = jinja2.FileSystemLoader(searchpath=templates_dir) @@ -68,7 +98,7 @@ class HTML(Report): has_repo_signed=SignSettings.SignRepository in self.sign_targets, packages=sorted(content, key=comparator), pgp_key=self.pgp_key, - repository=self.repository) + repository=self.name) with open(self.report_path, 'w') as out: out.write(html) diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index 46716364..7f3dc376 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -28,14 +28,32 @@ from ahriman.models.report_settings import ReportSettings class Report: + ''' + base report generator + :ivar architecture: repository architecture + :ivar config: configuration instance + :ivar logger: class logger + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' self.logger = logging.getLogger('builder') self.architecture = architecture self.config = config @staticmethod def run(architecture: str, config: Configuration, target: str, packages: Iterable[Package]) -> None: + ''' + run report generation + :param architecture: repository architecture + :param config: configuration instance + :param target: target to generate report (e.g. html) + :param packages: list of packages to generate report + ''' provider = ReportSettings.from_option(target) if provider == ReportSettings.HTML: from ahriman.core.report.html import HTML @@ -50,4 +68,8 @@ class Report: raise ReportFailed() def generate(self, packages: Iterable[Package]) -> None: + ''' + generate report for the specified packages + :param packages: list of packages to generate report + ''' pass \ No newline at end of file diff --git a/src/ahriman/core/repository.py b/src/ahriman/core/repository.py index 1f7f3baf..ef66cc3c 100644 --- a/src/ahriman/core/repository.py +++ b/src/ahriman/core/repository.py @@ -37,8 +37,26 @@ from ahriman.models.repository_paths import RepositoryPaths class Repository: + ''' + base repository control class + :ivar architecture: repository architecture + :ivar aur_url: base AUR url + :ivar config: configuration instance + :ivar logger: class logger + :ivar name: repository name + :ivar pacman: alpm wrapper instance + :ivar paths: repository paths instance + :ivar repo: repo commands wrapper instance + :ivar reporter: build status reporter instance + :ivar sign: GPG wrapper instance + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' self.logger = logging.getLogger('builder') self.architecture = architecture self.config = config @@ -52,21 +70,34 @@ class Repository: self.pacman = Pacman(config) self.sign = GPG(architecture, config) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) - self.web = Client.load(architecture, config) + self.reporter = Client.load(architecture, config) def _clear_build(self) -> None: + ''' + clear sources directory + ''' for package in os.listdir(self.paths.sources): shutil.rmtree(os.path.join(self.paths.sources, package)) def _clear_manual(self) -> None: + ''' + clear directory with manual package updates + ''' for package in os.listdir(self.paths.manual): shutil.rmtree(os.path.join(self.paths.manual, package)) def _clear_packages(self) -> None: + ''' + clear directory with built packages (NOT repository itself) + ''' for package in self.packages_built(): os.remove(package) def packages(self) -> List[Package]: + ''' + generate list of repository packages + :return: list of packages properties + ''' result: Dict[str, Package] = {} for fn in os.listdir(self.paths.repository): if not package_like(fn): @@ -81,14 +112,23 @@ class Repository: return list(result.values()) def packages_built(self) -> List[str]: + ''' + get list of files in built packages directory + :return: list of filenames from the directory + ''' return [ os.path.join(self.paths.packages, fn) for fn in os.listdir(self.paths.packages) ] def process_build(self, updates: Iterable[Package]) -> List[str]: + ''' + build packages + :param updates: list of packages properties to build + :return: `packages_built` + ''' def build_single(package: Package) -> None: - self.web.set_building(package.base) + self.reporter.set_building(package.base) task = Task(package, self.architecture, self.config, self.paths) task.clone() built = task.build() @@ -100,7 +140,7 @@ class Repository: try: build_single(package) except Exception: - self.web.set_failed(package.base) + self.reporter.set_failed(package.base) self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True) continue self._clear_build() @@ -108,9 +148,14 @@ class Repository: return self.packages_built() def process_remove(self, packages: Iterable[str]) -> str: + ''' + remove packages from list + :param packages: list of package names or bases to rmeove + :return: path to repository database + ''' def remove_single(package: str) -> None: try: - self.repo.remove(package, package) + self.repo.remove(package) except Exception: self.logger.exception(f'could not remove {package}', exc_info=True) @@ -118,29 +163,42 @@ class Repository: for local in self.packages(): if local.base in packages: to_remove = set(local.packages.keys()) + self.reporter.remove(local.base) # we only update status page in case of base removal elif requested.intersection(local.packages.keys()): to_remove = requested.intersection(local.packages.keys()) else: to_remove = set() - self.web.remove(local.base, to_remove) for package in to_remove: remove_single(package) return self.repo.repo_path def process_report(self, targets: Optional[Iterable[str]]) -> None: + ''' + generate reports + :param targets: list of targets to generate reports. Configuration option will be used if it is not set + ''' if targets is None: targets = self.config.getlist('report', 'target') for target in targets: Report.run(self.architecture, self.config, target, self.packages()) def process_sync(self, targets: Optional[Iterable[str]]) -> None: + ''' + process synchronization to remote servers + :param targets: list of targets to sync. Configuration option will be used if it is not set + ''' if targets is None: targets = self.config.getlist('upload', 'target') for target in targets: Uploader.run(self.architecture, self.config, target, self.paths.repository) def process_update(self, packages: Iterable[str]) -> str: + ''' + sign packages, add them to repository and update repository database + :param packages: list of filenames to run + :return: path to repository database + ''' for package in packages: local = Package.load(package, self.pacman, self.aur_url) # we will use it for status reports try: @@ -150,15 +208,21 @@ class Repository: shutil.move(src, dst) package_fn = os.path.join(self.paths.repository, os.path.basename(package)) self.repo.add(package_fn) - self.web.set_success(local) + self.reporter.set_success(local) except Exception: self.logger.exception(f'could not process {package}', exc_info=True) - self.web.set_failed(local.base) + self.reporter.set_failed(local.base) self._clear_packages() return self.repo.repo_path def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]: + ''' + check AUR for updates + :param filter_packages: do not check every package just specified in the list + :param no_vcs: do not check VCS packages + :return: list of packages which are out-of-dated + ''' result: List[Package] = [] build_section = self.config.get_section_name('build', self.architecture) @@ -176,22 +240,26 @@ class Repository: remote = Package.load(local.base, self.pacman, self.aur_url) if local.is_outdated(remote): result.append(remote) - self.web.set_pending(local.base) + self.reporter.set_pending(local.base) except Exception: - self.web.set_failed(local.base) + self.reporter.set_failed(local.base) self.logger.exception(f'could not load remote package {local.base}', exc_info=True) continue return result def updates_manual(self) -> List[Package]: + ''' + check for packages for which manual update has been requested + :return: list of packages which are out-of-dated + ''' result: List[Package] = [] for fn in os.listdir(self.paths.manual): try: local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url) result.append(local) - self.web.set_unknown(local) + self.reporter.set_unknown(local) except Exception: self.logger.exception(f'could not add package from {fn}', exc_info=True) self._clear_manual() diff --git a/src/ahriman/core/sign/gpg.py b/src/ahriman/core/sign/gpg.py index 506f6b26..35563d45 100644 --- a/src/ahriman/core/sign/gpg.py +++ b/src/ahriman/core/sign/gpg.py @@ -29,8 +29,21 @@ from ahriman.models.sign_settings import SignSettings class GPG: + ''' + gnupg wrapper + :ivar architecture: repository architecture + :ivar config: configuration instance + :ivar default_key: default PGP key ID to use + :ivar logger: class logger + :ivar target: list of targets to sign (repository, package etc) + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' self.logger = logging.getLogger('build_details') self.config = config self.section = config.get_section_name('sign', architecture) @@ -39,11 +52,20 @@ class GPG: @property def repository_sign_args(self) -> List[str]: + ''' + :return: command line arguments for repo-add command to sign database + ''' if SignSettings.SignRepository not in self.target: return [] return ['--sign', '--key', self.default_key] def process(self, path: str, key: str) -> List[str]: + ''' + gpg command wrapper + :param path: path to file to sign + :param key: PGP key ID + :return: list of generated files including original file + ''' check_output( *self.sign_cmd(path, key), exception=BuildFailed(path), @@ -52,18 +74,33 @@ class GPG: return [path, f'{path}.sig'] def sign_cmd(self, path: str, key: str) -> List[str]: - cmd = ['gpg'] - cmd.extend(['-u', key]) - cmd.extend(['-b', path]) - return cmd + ''' + gpg command to run + :param path: path to file to sign + :param key: PGP key ID + :return: gpg command with all required arguments + ''' + return ['gpg', '-u', key, '-b', path] def sign_package(self, path: str, base: str) -> List[str]: + ''' + sign package if required by configuration + :param path: path to file to sign + :param base: package base required to check for key overrides + :return: list of generated files including original file + ''' if SignSettings.SignPackages not in self.target: return [path] key = self.config.get(self.section, f'key_{base}', fallback=self.default_key) return self.process(path, key) def sign_repository(self, path: str) -> List[str]: + ''' + sign repository if required by configuration + :note: more likely you just want to pass `repository_sign_args` to repo wrapper + :param path: path to repository database + :return: list of generated files including original file + ''' if SignSettings.SignRepository not in self.target: return [path] return self.process(path, self.default_key) \ No newline at end of file diff --git a/src/ahriman/core/tree.py b/src/ahriman/core/tree.py index 9ad6295e..6c928924 100644 --- a/src/ahriman/core/tree.py +++ b/src/ahriman/core/tree.py @@ -28,42 +28,33 @@ from ahriman.core.build_tools.task import Task from ahriman.models.package import Package -class Tree: - - def __init__(self) -> None: - self.packages: List[Leaf] = [] - - def levels(self) -> List[List[Package]]: - result: List[List[Package]] = [] - - unprocessed = [leaf for leaf in self.packages] - while unprocessed: - result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)]) - unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)] - - return result - - def load(self, packages: Iterable[Package]) -> None: - for package in packages: - leaf = Leaf(package) - leaf.load_dependencies() - self.packages.append(leaf) - - class Leaf: + ''' + tree leaf implementation + :ivar dependencies: list of package dependencies + :ivar package: leaf package properties + ''' def __init__(self, package: Package) -> None: + ''' + default constructor + :param package: package properties + ''' self.package = package self.dependencies: Set[str] = set() @property def items(self) -> Iterable[str]: + ''' + :return: packages containing in this leaf + ''' return self.package.packages.keys() def is_root(self, packages: Iterable[Leaf]) -> bool: ''' - :param packages: - :return: true if any of packages is dependency of the leaf, false otherwise + check if package depends on any other package from list of not + :param packages: list of known leaves + :return: True if any of packages is dependency of the leaf, False otherwise ''' for leaf in packages: if self.dependencies.intersection(leaf.items): @@ -71,9 +62,49 @@ class Leaf: return True def load_dependencies(self) -> None: + ''' + load dependencies for the leaf + ''' clone_dir = tempfile.mkdtemp() try: Task.fetch(clone_dir, self.package.git_url) self.dependencies = Package.dependencies(clone_dir) finally: - shutil.rmtree(clone_dir, ignore_errors=True) \ No newline at end of file + shutil.rmtree(clone_dir, ignore_errors=True) + + +class Tree: + ''' + dependency tree implementation + :ivar leaves: list of tree leaves + ''' + + def __init__(self) -> None: + ''' + default constructor + ''' + self.leaves: List[Leaf] = [] + + def levels(self) -> List[List[Package]]: + ''' + get build levels starting from the packages which do not require any other package to build + :return: list of packages lists + ''' + result: List[List[Package]] = [] + + unprocessed = [leaf for leaf in self.leaves] + while unprocessed: + result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)]) + unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)] + + return result + + def load(self, packages: Iterable[Package]) -> None: + ''' + load tree from packages + :param packages: packages list + ''' + for package in packages: + leaf = Leaf(package) + leaf.load_dependencies() + self.leaves.append(leaf) diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index d8808f21..2b0e0849 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -23,13 +23,26 @@ from ahriman.core.util import check_output class Rsync(Uploader): + ''' + rsync wrapper + :ivar remote: remote address to sync + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' Uploader.__init__(self, architecture, config) section = config.get_section_name('rsync', architecture) self.remote = config.get(section, 'remote') def sync(self, path: str) -> None: + ''' + sync data to remote server + :param path: local path to sync + ''' check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote, exception=None, logger=self.logger) diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index 00c6e60c..dfc733f2 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -23,13 +23,26 @@ from ahriman.core.util import check_output class S3(Uploader): + ''' + aws-cli wrapper + :ivar bucket: full bucket name + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' Uploader.__init__(self, architecture, config) section = config.get_section_name('s3', architecture) self.bucket = config.get(section, 'bucket') def sync(self, path: str) -> None: + ''' + sync data to remote server + :param path: local path to sync + ''' # TODO rewrite to boto, but it is bullshit check_output('aws', 's3', 'sync', '--delete', path, self.bucket, exception=None, diff --git a/src/ahriman/core/upload/uploader.py b/src/ahriman/core/upload/uploader.py index f4ae6281..884876aa 100644 --- a/src/ahriman/core/upload/uploader.py +++ b/src/ahriman/core/upload/uploader.py @@ -25,14 +25,32 @@ from ahriman.models.upload_settings import UploadSettings class Uploader: + ''' + base remote sync class + :ivar architecture: repository architecture + :ivar config: configuration instance + :ivar logger: application logger + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' self.logger = logging.getLogger('builder') self.architecture = architecture self.config = config @staticmethod def run(architecture: str, config: Configuration, target: str, path: str) -> None: + ''' + run remote sync + :param architecture: repository architecture + :param config: configuration instance + :param target: target to run sync (e.g. s3) + :param path: local path to sync + ''' provider = UploadSettings.from_option(target) if provider == UploadSettings.Rsync: from ahriman.core.upload.rsync import Rsync @@ -50,4 +68,8 @@ class Uploader: raise SyncFailed() def sync(self, path: str) -> None: + ''' + sync data to remote server + :param path: local path to sync + ''' pass diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 3afe2b67..0d8b88f4 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -26,6 +26,15 @@ from typing import Optional def check_output(*args: str, exception: Optional[Exception], cwd: Optional[str] = None, stderr: int = subprocess.STDOUT, logger: Optional[Logger] = None) -> str: + ''' + subprocess wrapper + :param args: command line arguments + :param exception: exception which has to be reraised instead of default subprocess exception + :param cwd: current working directory + :param stderr: standard error output mode + :param logger: logger to log command result if required + :return: command output + ''' try: result = subprocess.check_output(args, cwd=cwd, stderr=stderr).decode('utf8').strip() if logger is not None: @@ -40,4 +49,9 @@ def check_output(*args: str, exception: Optional[Exception], def package_like(filename: str) -> bool: + ''' + check if file looks like package + :param filename: name of file to check + :return: True in case if name contains `.pkg.` and not signature, False otherwise + ''' return '.pkg.' in filename and not filename.endswith('.sig') diff --git a/src/ahriman/core/watcher/client.py b/src/ahriman/core/watcher/client.py index bc14ebd5..9c4f5c4a 100644 --- a/src/ahriman/core/watcher/client.py +++ b/src/ahriman/core/watcher/client.py @@ -19,98 +19,87 @@ # from __future__ import annotations -import logging - -from typing import Any, Dict, Set - from ahriman.core.configuration import Configuration from ahriman.models.build_status import BuildStatusEnum from ahriman.models.package import Package class Client: + ''' + base build status reporter client + ''' def add(self, package: Package, status: BuildStatusEnum) -> None: + ''' + add new package with status + :param package: package properties + :param status: current package build status + ''' pass - def remove(self, base: str, packages: Set[str]) -> None: + def remove(self, base: str) -> None: + ''' + remove packages from watcher + :param base: basename to remove + ''' pass def update(self, base: str, status: BuildStatusEnum) -> None: + ''' + update package build status. Unlike `add` it does not update package properties + :param base: package base to update + :param status: current package build status + ''' pass def set_building(self, base: str) -> None: + ''' + set package status to building + :param base: package base to update + ''' return self.update(base, BuildStatusEnum.Building) def set_failed(self, base: str) -> None: + ''' + set package status to failed + :param base: package base to update + ''' return self.update(base, BuildStatusEnum.Failed) def set_pending(self, base: str) -> None: + ''' + set package status to pending + :param base: package base to update + ''' return self.update(base, BuildStatusEnum.Pending) def set_success(self, package: Package) -> None: + ''' + set package status to success + :param package: current package properties + ''' return self.add(package, BuildStatusEnum.Success) def set_unknown(self, package: Package) -> None: + ''' + set package status to unknown + :param package: current package properties + ''' return self.add(package, BuildStatusEnum.Unknown) @staticmethod def load(architecture: str, config: Configuration) -> Client: + ''' + load client from settings + :param architecture: repository architecture + :param config: configuration instance + :return: client according to current settings + ''' section = config.get_section_name('web', architecture) host = config.get(section, 'host', fallback=None) port = config.getint(section, 'port', fallback=None) if host is None or port is None: return Client() + + from ahriman.core.watcher.web_client import WebClient return WebClient(host, port) - - -class WebClient(Client): - - def __init__(self, host: str, port: int) -> None: - self.logger = logging.getLogger('http') - self.host = host - self.port = port - - def _url(self, base: str) -> str: - return f'http://{self.host}:{self.port}/api/v1/packages/{base}' - - def add(self, package: Package, status: BuildStatusEnum) -> None: - import requests - - payload: Dict[str, Any] = { - 'status': status.value, - 'package': { - 'base': package.base, - 'packages': [p for p in package.packages], - 'version': package.version, - 'aur_url': package.aur_url - } - } - - try: - response = requests.post(self._url(package.base), json=payload) - response.raise_for_status() - except: - self.logger.exception(f'could not add {package.base}', exc_info=True) - - def remove(self, base: str, packages: Set[str]) -> None: - if not packages: - return - import requests - - try: - response = requests.delete(self._url(base)) - response.raise_for_status() - except: - self.logger.exception(f'could not delete {base}', exc_info=True) - - def update(self, base: str, status: BuildStatusEnum) -> None: - import requests - - payload: Dict[str, Any] = {'status': status.value} - - try: - response = requests.post(self._url(base), json=payload) - response.raise_for_status() - except: - self.logger.exception(f'could not update {base}', exc_info=True) \ No newline at end of file diff --git a/src/ahriman/core/watcher/watcher.py b/src/ahriman/core/watcher/watcher.py index 3c6c896f..dea88fc4 100644 --- a/src/ahriman/core/watcher/watcher.py +++ b/src/ahriman/core/watcher/watcher.py @@ -26,8 +26,19 @@ from ahriman.models.package import Package class Watcher: + ''' + package status watcher + :ivar architecture: repository architecture + :ivar known: list of known packages. For the most cases `packages` should be used instead + :ivar repository: repository object + ''' def __init__(self, architecture: str, config: Configuration) -> None: + ''' + default constructor + :param architecture: repository architecture + :param config: configuration instance + ''' self.architecture = architecture self.repository = Repository(architecture, config) @@ -35,9 +46,15 @@ class Watcher: @property def packages(self) -> List[Tuple[Package, BuildStatus]]: + ''' + :return: list of packages together with their statuses + ''' return [pair for pair in self.known.values()] def load(self) -> None: + ''' + load packages from local repository. In case if last status is known, it will use it + ''' for package in self.repository.packages(): # get status of build or assign unknown current = self.known.get(package.base) @@ -48,9 +65,19 @@ class Watcher: self.known[package.base] = (package, status) def remove(self, base: str) -> None: + ''' + remove package base from known list if any + :param base: package base + ''' self.known.pop(base, None) def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None: + ''' + update package status and description + :param base: package base to update + :param status: new build status + :param package: optional new package description. In case if not set current properties will be used + ''' if package is None: package, _ = self.known[base] full_status = BuildStatus(status) diff --git a/src/ahriman/core/watcher/web_client.py b/src/ahriman/core/watcher/web_client.py new file mode 100644 index 00000000..fbeb1df4 --- /dev/null +++ b/src/ahriman/core/watcher/web_client.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import logging +import requests + +from dataclasses import asdict +from typing import Any, Dict + +from ahriman.core.watcher.client import Client +from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.package import Package + + +class WebClient(Client): + ''' + build status reporter web client + :ivar host: host of web service + :ivar logger: class logger + :ivar port: port of web service + ''' + + def __init__(self, host: str, port: int) -> None: + ''' + default constructor + :param host: host of web service + :param port: port of web service + ''' + self.logger = logging.getLogger('http') + self.host = host + self.port = port + + def _url(self, base: str) -> str: + ''' + url generator + :param base: package base to generate url + :return: full url of web service for specific package base + ''' + return f'http://{self.host}:{self.port}/api/v1/packages/{base}' + + def add(self, package: Package, status: BuildStatusEnum) -> None: + ''' + add new package with status + :param package: package properties + :param status: current package build status + ''' + payload: Dict[str, Any] = { + 'status': status.value, + 'package': asdict(package) + } + + try: + response = requests.post(self._url(package.base), json=payload) + response.raise_for_status() + except: + self.logger.exception(f'could not add {package.base}', exc_info=True) + + def remove(self, base: str) -> None: + ''' + remove packages from watcher + :param base: basename to remove + ''' + try: + response = requests.delete(self._url(base)) + response.raise_for_status() + except: + self.logger.exception(f'could not delete {base}', exc_info=True) + + def update(self, base: str, status: BuildStatusEnum) -> None: + ''' + update package build status. Unlike `add` it does not update package properties + :param base: package base to update + :param status: current package build status + ''' + payload: Dict[str, Any] = {'status': status.value} + + try: + response = requests.post(self._url(base), json=payload) + response.raise_for_status() + except: + self.logger.exception(f'could not update {base}', exc_info=True) \ No newline at end of file diff --git a/src/ahriman/models/build_status.py b/src/ahriman/models/build_status.py index f8a490ff..311f63e2 100644 --- a/src/ahriman/models/build_status.py +++ b/src/ahriman/models/build_status.py @@ -24,6 +24,15 @@ from typing import Optional, Union class BuildStatusEnum(Enum): + ''' + build status enumeration + :cvar Unknown: build status is unknown + :cvar Pending: package is out-of-dated and will be built soon + :cvar Building: package is building right now + :cvar Failed: package build failed + :cvar Success: package has been built without errors + ''' + Unknown = 'unknown' Pending = 'pending' Building = 'building' @@ -32,12 +41,25 @@ class BuildStatusEnum(Enum): class BuildStatus: + ''' + build status holder + :ivar status: build status + :ivar _timestamp: build status update time + ''' def __init__(self, status: Union[BuildStatusEnum, str, None] = None, timestamp: Optional[datetime.datetime] = None) -> None: + ''' + default constructor + :param status: current build status if known. `BuildStatusEnum.Unknown` will be used if not set + :param timestamp: build status timestamp. Current timestamp will be used if not set + ''' self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown self._timestamp = timestamp or datetime.datetime.utcnow() @property def timestamp(self) -> str: + ''' + :return: string representation of build status timestamp + ''' return self._timestamp.strftime('%Y-%m-%d %H:%M:%S') diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index aedefb38..4a3988bb 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -36,17 +36,31 @@ from ahriman.core.util import check_output @dataclass class Package: + ''' + package properties representation + :ivar aurl_url: AUR root url + :ivar base: package base name + :ivar packages: map of package names to archive name + :ivar version: package full version + ''' + base: str version: str aur_url: str - packages: Dict[str, str] # map of package name to archive name + packages: Dict[str, str] @property def git_url(self) -> str: + ''' + :return: package git url to clone + ''' return f'{self.aur_url}/{self.base}.git' @property def is_vcs(self) -> bool: + ''' + :return: True in case if package base looks like VCS package and false otherwise + ''' return self.base.endswith('-bzr') \ or self.base.endswith('-csv')\ or self.base.endswith('-darcs')\ @@ -56,10 +70,16 @@ class Package: @property def web_url(self) -> str: + ''' + :return: package AUR url + ''' return f'{self.aur_url}/packages/{self.base}' - # additional method to handle vcs versions def actual_version(self) -> str: + ''' + additional method to handle VCS package versions + :return: package version if package is not VCS and current version according to VCS otherwise + ''' if not self.is_vcs: return self.version @@ -82,16 +102,35 @@ class Package: @classmethod def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package: + ''' + construct package properties from package archive + :param path: path to package archive + :param pacman: alpm wrapper instance + :param aur_url: AUR root url + :return: package properties + ''' package = pacman.handle.load_pkg(path) return cls(package.base, package.version, aur_url, {package.name: os.path.basename(path)}) @classmethod def from_aur(cls: Type[Package], name: str, aur_url: str)-> Package: + ''' + construct package properties from AUR page + :param name: package name (either base or normal name) + :param aur_url: AUR root url + :return: package properties + ''' package = aur.info(name) return cls(package.package_base, package.version, aur_url, {package.name: ''}) @classmethod def from_build(cls: Type[Package], path: str, aur_url: str) -> Package: + ''' + construct package properties from sources directory + :param path: path to package sources directory + :param aur_url: AUR root url + :return: package properties + ''' with open(os.path.join(path, '.SRCINFO')) as fn: src_info, errors = parse_srcinfo(fn.read()) if errors: @@ -103,6 +142,11 @@ class Package: @staticmethod def dependencies(path: str) -> Set[str]: + ''' + load dependencies from package sources + :param path: path to package sources directory + :return: list of package dependencies including makedepends array, but excluding packages from this base + ''' with open(os.path.join(path, '.SRCINFO')) as fn: src_info, errors = parse_srcinfo(fn.read()) if errors: @@ -118,11 +162,25 @@ class Package: @staticmethod def full_version(epoch: Optional[str], pkgver: str, pkgrel: str) -> str: + ''' + generate full version from components + :param epoch: package epoch if any + :param pkgver: package version + :param pkgrel: package release version (archlinux specific) + :return: generated version + ''' prefix = f'{epoch}:' if epoch else '' return f'{prefix}{pkgver}-{pkgrel}' @staticmethod def load(path: str, pacman: Pacman, aur_url: str) -> Package: + ''' + package constructor from available sources + :param path: one of path to sources directory, path to archive or package name/base + :param pacman: alpm wrapper instance (required to load from archive) + :param aur_url: AUR root url + :return: package properties + ''' try: if os.path.isdir(path): package: Package = Package.from_build(path, aur_url) @@ -137,6 +195,11 @@ class Package: raise InvalidPackageInfo(str(e)) def is_outdated(self, remote: Package) -> bool: + ''' + check if package is out-of-dated + :param remote: package properties from remote source + :return: True if the package is out-of-dated and False otherwise + ''' remote_version = remote.actual_version() # either normal version or updated VCS result: int = vercmp(self.version, remote_version) return result < 0 diff --git a/src/ahriman/models/report_settings.py b/src/ahriman/models/report_settings.py index 2411c56d..b4abfa95 100644 --- a/src/ahriman/models/report_settings.py +++ b/src/ahriman/models/report_settings.py @@ -25,10 +25,20 @@ from ahriman.core.exceptions import InvalidOption class ReportSettings(Enum): + ''' + report targets enumeration + :ivar HTML: html report generation + ''' + HTML = auto() @staticmethod def from_option(value: str) -> ReportSettings: + ''' + construct value from configuration + :param value: configuration value + :return: parsed value + ''' if value.lower() in ('html',): return ReportSettings.HTML raise InvalidOption(value) diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index 02e4e04e..c5d1f4c7 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -24,6 +24,12 @@ from dataclasses import dataclass @dataclass class RepositoryPaths: + ''' + repository paths holder + :ivar root: repository root (i.e. ahriman home) + :ivar architecture: repository architecture + ''' + root: str architecture: str @@ -63,6 +69,9 @@ class RepositoryPaths: return os.path.join(self.root, 'sources') def create_tree(self) -> None: + ''' + create ahriman working tree + ''' os.makedirs(self.chroot, mode=0o755, exist_ok=True) os.makedirs(self.manual, mode=0o755, exist_ok=True) os.makedirs(self.packages, mode=0o755, exist_ok=True) diff --git a/src/ahriman/models/sign_settings.py b/src/ahriman/models/sign_settings.py index f329d8ee..d70b5378 100644 --- a/src/ahriman/models/sign_settings.py +++ b/src/ahriman/models/sign_settings.py @@ -25,11 +25,22 @@ from ahriman.core.exceptions import InvalidOption class SignSettings(Enum): + ''' + sign targets enumeration + :ivar SignPackages: sign each package + :ivar SignRepository: sign repository database file + ''' + SignPackages = auto() SignRepository = auto() @staticmethod def from_option(value: str) -> SignSettings: + ''' + construct value from configuration + :param value: configuration value + :return: parsed value + ''' if value.lower() in ('package', 'packages', 'sign-package'): return SignSettings.SignPackages elif value.lower() in ('repository', 'sign-repository'): diff --git a/src/ahriman/models/upload_settings.py b/src/ahriman/models/upload_settings.py index 7b9a684c..9a9484c5 100644 --- a/src/ahriman/models/upload_settings.py +++ b/src/ahriman/models/upload_settings.py @@ -25,11 +25,22 @@ from ahriman.core.exceptions import InvalidOption class UploadSettings(Enum): + ''' + remote synchronization targets enumeration + :ivar Rsync: sync via rsync + :ivar S3: sync to Amazon S3 + ''' + Rsync = auto() S3 = auto() @staticmethod def from_option(value: str) -> UploadSettings: + ''' + construct value from configuration + :param value: configuration value + :return: parsed value + ''' if value.lower() in ('rsync',): return UploadSettings.Rsync elif value.lower() in ('s3',): diff --git a/src/ahriman/version.py b/src/ahriman/version.py index 1c8ec579..a50dced8 100644 --- a/src/ahriman/version.py +++ b/src/ahriman/version.py @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -__version__ = '0.11.7' \ No newline at end of file +__version__ = '0.11.7' diff --git a/src/ahriman/web/middlewares/exception_handler.py b/src/ahriman/web/middlewares/exception_handler.py index 202fa111..df6a2490 100644 --- a/src/ahriman/web/middlewares/exception_handler.py +++ b/src/ahriman/web/middlewares/exception_handler.py @@ -28,6 +28,11 @@ HandlerType = Callable[[Request], Awaitable[StreamResponse]] def exception_handler(logger: Logger) -> Callable[[Request, HandlerType], Awaitable[StreamResponse]]: + ''' + exception handler middleware. Just log any exception (except for client ones) + :param logger: class logger + :return: built middleware + ''' @middleware async def handle(request: Request, handler: HandlerType) -> StreamResponse: try: diff --git a/src/ahriman/web/routes.py b/src/ahriman/web/routes.py index ce893ca1..bde993e9 100644 --- a/src/ahriman/web/routes.py +++ b/src/ahriman/web/routes.py @@ -24,11 +24,26 @@ from ahriman.web.views.package import PackageView from ahriman.web.views.packages import PackagesView -def setup_routes(app: Application) -> None: - app.router.add_get('/', IndexView) - app.router.add_get('/index.html', IndexView) +def setup_routes(application: Application) -> None: + ''' + setup all defined routes - app.router.add_post('/api/v1/packages', PackagesView) + Available routes are: - app.router.add_delete('/api/v1/packages/{package}', PackageView) - app.router.add_post('/api/v1/packages/{package}', PackageView) \ No newline at end of file + GET / get build status page + GET /index.html same as above + + POST /api/v1/packages force update every package from repository + + POST /api/v1/package/:base update package base status + DELETE /api/v1/package/:base delete package base from status page + + :param application: web application instance + ''' + application.router.add_get('/', IndexView) + application.router.add_get('/index.html', IndexView) + + application.router.add_post('/api/v1/packages', PackagesView) + + application.router.add_delete('/api/v1/packages/{package}', PackageView) + application.router.add_post('/api/v1/packages/{package}', PackageView) \ No newline at end of file diff --git a/src/ahriman/web/views/base.py b/src/ahriman/web/views/base.py index 7399cb9b..c6165f8c 100644 --- a/src/ahriman/web/views/base.py +++ b/src/ahriman/web/views/base.py @@ -22,10 +22,15 @@ from aiohttp.web import View from ahriman.core.watcher.watcher import Watcher -# special class to make it typed class BaseView(View): + ''' + base web view to make things typed + ''' @property def service(self) -> Watcher: + ''' + :return: build status watcher instance + ''' watcher: Watcher = self.request.app['watcher'] return watcher diff --git a/src/ahriman/web/views/index.py b/src/ahriman/web/views/index.py index 75ef14e3..0b9e76a0 100644 --- a/src/ahriman/web/views/index.py +++ b/src/ahriman/web/views/index.py @@ -27,9 +27,24 @@ from ahriman.web.views.base import BaseView class IndexView(BaseView): + ''' + root view + + It uses jinja2 templates for report generation, the following variables are allowed: + + architecture - repository architecture, string, required + packages - sorted list of packages properties: base, packages (sorted list), status, + timestamp, version, web_url. Required + repository - repository name, string, required + version - ahriman version, string, required + ''' @aiohttp_jinja2.template("build-status.jinja2") # type: ignore async def get(self) -> Dict[str, Any]: + ''' + process get request. No parameters supported here + :return: parameters for jinja template + ''' # some magic to make it jinja-friendly packages = [ { diff --git a/src/ahriman/web/views/package.py b/src/ahriman/web/views/package.py index 1ecd8463..dea832f2 100644 --- a/src/ahriman/web/views/package.py +++ b/src/ahriman/web/views/package.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from aiohttp.web import HTTPOk, Response +from aiohttp.web import HTTPBadRequest, HTTPOk, Response from ahriman.models.build_status import BuildStatusEnum from ahriman.models.package import Package @@ -25,19 +25,45 @@ from ahriman.web.views.base import BaseView class PackageView(BaseView): + ''' + package base specific web view + ''' async def delete(self) -> Response: + ''' + delete package base from status page + :return: 200 on success + ''' base = self.request.match_info['package'] self.service.remove(base) return HTTPOk() async def post(self) -> Response: + ''' + update package build status + + JSON body must be supplied, the following model is used: + { + "status": "unknown", # package build status string, must be valid `BuildStatusEnum` + "package": {} # package body (use `dataclasses.asdict` to generate one), optional. + # Must be supplied in case if package base is unknown + } + + :return: 200 on success + ''' base = self.request.match_info['package'] data = await self.request.json() - package = Package(**data['package']) if 'package' in data else None - status = BuildStatusEnum(data['status']) - self.service.update(base, status, package) + try: + package = Package(**data['package']) if 'package' in data else None + status = BuildStatusEnum(data['status']) + except Exception as e: + raise HTTPBadRequest(text=str(e)) + + try: + self.service.update(base, status, package) + except KeyError: + raise HTTPBadRequest(text=f'Package {base} is unknown, but no package body set') return HTTPOk() diff --git a/src/ahriman/web/views/packages.py b/src/ahriman/web/views/packages.py index 977ea3f8..ea1ae343 100644 --- a/src/ahriman/web/views/packages.py +++ b/src/ahriman/web/views/packages.py @@ -23,8 +23,15 @@ from ahriman.web.views.base import BaseView class PackagesView(BaseView): + ''' + global watcher view + ''' async def post(self) -> Response: + ''' + reload all packages from repository. No parameters supported here + :return: 200 on success + ''' self.service.load() return HTTPOk() diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index fa5bab6f..1ae5dc6c 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -30,30 +30,49 @@ from ahriman.web.middlewares.exception_handler import exception_handler from ahriman.web.routes import setup_routes -async def on_shutdown(app: web.Application) -> None: - app.logger.warning('server terminated') +async def on_shutdown(application: web.Application) -> None: + ''' + web application shutdown handler + :param application: web application instance + ''' + application.logger.warning('server terminated') -async def on_startup(app: web.Application) -> None: - app.logger.info('server started') +async def on_startup(application: web.Application) -> None: + ''' + web application start handler + :param application: web application instance + ''' + application.logger.info('server started') try: - app['watcher'].load() + application['watcher'].load() except Exception: - app.logger.exception('could not load packages', exc_info=True) + application.logger.exception('could not load packages', exc_info=True) raise InitializeException() -def run_server(app: web.Application, architecture: str) -> None: - app.logger.info('start server') +def run_server(application: web.Application, architecture: str) -> None: + ''' + run web application + :param application: web application instance + :param architecture: repository architecture + ''' + application.logger.info('start server') - section = app['config'].get_section_name('web', architecture) - host = app['config'].get(section, 'host') - port = app['config'].getint(section, 'port') + section = application['config'].get_section_name('web', architecture) + host = application['config'].get(section, 'host') + port = application['config'].getint(section, 'port') - web.run_app(app, host=host, port=port, handle_signals=False) + web.run_app(application, host=host, port=port, handle_signals=False) def setup_service(architecture: str, config: Configuration) -> web.Application: + ''' + create web application + :param architecture: repository architecture + :param config: configuration instance + :return: web application instance + ''' app = web.Application(logger=logging.getLogger('http')) app.on_shutdown.append(on_shutdown) app.on_startup.append(on_startup)