From d4222eca25ff7f6c18945c071684d9163f09df29 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Tue, 16 Mar 2021 05:24:07 +0300 Subject: [PATCH] add dump config option, change all timestamp objects to int, check git directory --- package/archlinux/PKGBUILD | 2 +- src/ahriman/application/ahriman.py | 30 +++++++++++++++++++----- src/ahriman/core/build_tools/task.py | 3 ++- src/ahriman/core/configuration.py | 28 ++++++++++++++++++++-- src/ahriman/core/util.py | 4 ++-- src/ahriman/models/build_status.py | 4 ++-- src/ahriman/models/package.py | 4 +--- src/ahriman/models/package_desciption.py | 4 +--- src/ahriman/web/web.py | 28 +++++++++++----------- 9 files changed, 73 insertions(+), 34 deletions(-) diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index 3894827b..29d22dd4 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3' source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz" 'ahriman.sysusers' 'ahriman.tmpfiles') -sha512sums=('d7c4c0808eef7a1ebd7a137777e4855eebd4666304e18ea6ece9499f1ed8cf459299561d9ec4917f6c454e5fff7eee0b97e1d6efcc80be7308aac75141584cd5' +sha512sums=('d88810f7c90b1cbce7c601649c5d5f06026427e2b5260e8f61d75cefec38a5fad98f1bce5468bbd040a1af0472523547addb39675b1980baf4a695a678285903' '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075' '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4') backup=('etc/ahriman.ini' diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index c440722c..20e9902c 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -68,6 +68,21 @@ def clean(args: argparse.Namespace, architecture: str, config: Configuration) -> args.no_manual, args.no_packages) +def dump_config(args: argparse.Namespace, architecture: str, config: Configuration) -> None: + ''' + configuration dump callback + :param args: command line args + :param architecture: repository architecture + :param config: configuration instance + ''' + result = config.dump(architecture) + for section, values in sorted(result.items()): + print(f'[{section}]') + for key, value in sorted(values.items()): + print(f'{key} = {value}') + print() + + def rebuild(args: argparse.Namespace, architecture: str, config: Configuration) -> None: ''' world rebuild callback @@ -119,14 +134,14 @@ def update(args: argparse.Namespace, architecture: str, config: Configuration) - ''' # typing workaround def log_fn(line: str) -> None: - return print(line) if args.dry_run else app.logger.info(line) + return print(line) if args.dry_run else application.logger.info(line) - app = Application(architecture, config) - packages = app.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn) + application = Application(architecture, config) + packages = application.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn) if args.dry_run: return - app.update(packages) + application.update(packages) def web(args: argparse.Namespace, architecture: str, config: Configuration) -> None: @@ -137,8 +152,8 @@ def web(args: argparse.Namespace, architecture: str, config: Configuration) -> N :param config: configuration instance ''' from ahriman.web.web import run_server, setup_service - app = setup_service(architecture, config) - run_server(app, architecture) + application = setup_service(architecture, config) + run_server(application, architecture) if __name__ == '__main__': @@ -176,6 +191,9 @@ if __name__ == '__main__': clean_parser.add_argument('--no-packages', help='do not clear directory with built packages', action='store_true') clean_parser.set_defaults(fn=clean) + config_parser = subparsers.add_parser('config', description='dump configuration for specified architecture') + config_parser.set_defaults(fn=dump_config) + rebuild_parser = subparsers.add_parser('rebuild', description='rebuild whole repository') rebuild_parser.set_defaults(fn=rebuild) diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index 6ea5aef8..5fa85fc3 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -81,7 +81,8 @@ class Task: :param branch: branch name to checkout, master by default ''' logger = logging.getLogger('build_details') - if os.path.isdir(local): + # local directory exists and there is .git directory + if os.path.isdir(os.path.join(local, '.git')): check_output('git', 'fetch', 'origin', branch, exception=None, cwd=local, logger=logger) else: check_output('git', 'clone', remote, local, exception=None, logger=logger) diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index 285b7692..e1a71bda 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -24,20 +24,25 @@ import logging import os from logging.config import fileConfig -from typing import List, Optional, Type +from typing import Dict, List, Optional, Type class Configuration(configparser.RawConfigParser): ''' extension for built-in configuration parser :ivar path: path to root configuration file + :cvar ARCHITECTURE_SPECIFIC_SECTIONS: known sections which can be architecture specific (required by dump) :cvar DEFAULT_LOG_FORMAT: default log format (in case of fallback) :cvar DEFAULT_LOG_LEVEL: default log level (in case of fallback) + :cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump) ''' DEFAULT_LOG_FORMAT = '%(asctime)s : %(levelname)s : %(funcName)s : %(message)s' DEFAULT_LOG_LEVEL = logging.DEBUG + STATIC_SECTIONS = ['alpm', 'report', 'repository', 'settings', 'upload'] + ARCHITECTURE_SPECIFIC_SECTIONS = ['build', 'html', 'rsync', 's3', 'sign', 'web'] + def __init__(self) -> None: ''' default constructor @@ -64,6 +69,25 @@ class Configuration(configparser.RawConfigParser): config.load_logging() return config + def dump(self, architecture: str) -> Dict[str, Dict[str, str]]: + ''' + dump configuration to dictionary + :param architecture: repository architecture + :return: configuration dump for specific architecture + ''' + result: Dict[str, Dict[str, str]] = {} + for section in Configuration.STATIC_SECTIONS: + if not self.has_section(section): + continue + result[section] = dict(self[section]) + for group in Configuration.ARCHITECTURE_SPECIFIC_SECTIONS: + section = self.get_section_name(group, architecture) + if not self.has_section(section): + continue + result[section] = dict(self[section]) + + return result + def getlist(self, section: str, key: str) -> List[str]: ''' get space separated string list option @@ -103,7 +127,7 @@ class Configuration(configparser.RawConfigParser): for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))): self.read(os.path.join(self.include, conf)) except (FileNotFoundError, configparser.NoOptionError): - passDEFAULT_LOG_LEVEL + pass def load_logging(self) -> None: ''' diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 1ebf505d..867e182c 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -60,13 +60,13 @@ def package_like(filename: str) -> bool: return '.pkg.' in filename and not filename.endswith('.sig') -def pretty_datetime(timestamp: Optional[datetime.datetime]) -> str: +def pretty_datetime(timestamp: Optional[int]) -> str: ''' convert datetime object to string :param timestamp: datetime to convert :return: pretty printable datetime as string ''' - return '' if timestamp is None else timestamp.strftime('%Y-%m-%d %H:%M:%S') + return '' if timestamp is None else datetime.datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') def pretty_size(size: Optional[float], level: int = 0) -> str: diff --git a/src/ahriman/models/build_status.py b/src/ahriman/models/build_status.py index 5b417793..aca5c0e6 100644 --- a/src/ahriman/models/build_status.py +++ b/src/ahriman/models/build_status.py @@ -63,11 +63,11 @@ class BuildStatus: ''' def __init__(self, status: Union[BuildStatusEnum, str, None] = None, - timestamp: Optional[datetime.datetime] = None) -> None: + timestamp: Optional[int] = None) -> None: ''' default constructor :param status: current build status if known. `BuildStatusEnum.Unknown` will be used if not set :param timestamp: build status timestamp. Current timestamp will be used if not set ''' self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown - self.timestamp = timestamp or datetime.datetime.utcnow() + self.timestamp = timestamp or int(datetime.datetime.utcnow().timestamp()) diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 404fbadd..542089c3 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -22,7 +22,6 @@ from __future__ import annotations import logging import aur # type: ignore -import datetime import os from dataclasses import dataclass @@ -113,8 +112,7 @@ class Package: :return: package properties ''' package = pacman.handle.load_pkg(path) - build_date = datetime.datetime.fromtimestamp(package.builddate) - properties = PackageDescription(package.size, build_date, os.path.basename(path), package.isize) + properties = PackageDescription(package.size, package.builddate, os.path.basename(path), package.isize) return cls(package.base, package.version, aur_url, {package.name: properties}) @classmethod diff --git a/src/ahriman/models/package_desciption.py b/src/ahriman/models/package_desciption.py index 53f4daa1..28ded82a 100644 --- a/src/ahriman/models/package_desciption.py +++ b/src/ahriman/models/package_desciption.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import datetime - from dataclasses import dataclass from typing import Optional @@ -34,6 +32,6 @@ class PackageDescription: ''' archive_size: Optional[int] = None - build_date: Optional[datetime.datetime] = None + build_date: Optional[int] = None filename: Optional[str] = None installed_size: Optional[int] = None diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 66c34f61..69fa0a1e 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -74,22 +74,22 @@ def setup_service(architecture: str, config: Configuration) -> web.Application: :param config: configuration instance :return: web application instance ''' - app = web.Application(logger=logging.getLogger('http')) - app.on_shutdown.append(on_shutdown) - app.on_startup.append(on_startup) + application = web.Application(logger=logging.getLogger('http')) + application.on_shutdown.append(on_shutdown) + application.on_startup.append(on_startup) - app.middlewares.append(web.normalize_path_middleware(append_slash=False, remove_slash=True)) - app.middlewares.append(exception_handler(app.logger)) + application.middlewares.append(web.normalize_path_middleware(append_slash=False, remove_slash=True)) + application.middlewares.append(exception_handler(application.logger)) - app.logger.info('setup routes') - setup_routes(app) - app.logger.info('setup templates') - aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader(config.get('web', 'templates'))) + application.logger.info('setup routes') + setup_routes(application) + application.logger.info('setup templates') + aiohttp_jinja2.setup(application, loader=jinja2.FileSystemLoader(config.get('web', 'templates'))) - app.logger.info('setup configuration') - app['config'] = config + application.logger.info('setup configuration') + application['config'] = config - app.logger.info('setup watcher') - app['watcher'] = Watcher(architecture, config) + application.logger.info('setup watcher') + application['watcher'] = Watcher(architecture, config) - return app + return application