web service improvements

* load and save web service state to cache file
* disable web reporting to self
* restore console handler settings
* allow to redirect logs to stderr
* verbose http error logging
* update package status by group, not by single package
* split Repository class to several traits
* move json generators/readers to dataclasses

package/archlinux/PKGBUILD

@@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
         'ahriman.sysusers'
         'ahriman.tmpfiles')
-sha512sums=('54286cfd1c9b03e7adfa639b976ace233e4e3ea8d2a2cbd11c22fc43eda60906e1d3b795e1505b40e41171948ba95d6591a4f7c328146200f4622a8ed657e8a5'
+sha512sums=('d1a88fc3c5c14258cd0f84c815ebd254749ca8f9ba4dafe4a7385ac2eafc27cc552812a1951ccf1741ddc7ac7d4b2d2ebbe15960b0796e4f37653184adb86e30'
             '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
             '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
 backup=('etc/ahriman.ini'

package/etc/ahriman.ini.d/logging.ini

@@ -2,11 +2,17 @@
 keys = root,builder,build_details,http
 
 [handlers]
-keys = build_file_handler,file_handler,http_handler
+keys = console_handler,build_file_handler,file_handler,http_handler
 
 [formatters]
 keys = generic_format
 
+[handler_console_handler]
+class = StreamHandler
+level = DEBUG
+formatter = generic_format
+args = (sys.stderr,)
+
 [handler_file_handler]
 class = logging.handlers.RotatingFileHandler
 level = DEBUG
@@ -26,7 +32,7 @@ formatter = generic_format
 args = ('/var/log/ahriman/http.log', 'a', 20971520, 20)
 
 [formatter_generic_format]
-format = %(asctime)s : %(levelname)s : %(funcName)s : %(message)s
+format = [%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s
 datefmt =
 
 [logger_root]

src/ahriman/application/ahriman.py

@@ -39,7 +39,7 @@ def _call(args: argparse.Namespace, architecture: str, config: Configuration) ->
     :return: True on success, False otherwise
     '''
     try:
-        with Lock(args.lock, architecture, config):
+        with Lock(args, architecture, config):
             args.fn(args, architecture, config)
         return True
     except Exception:
@@ -169,6 +169,8 @@ if __name__ == '__main__':
     parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini')
     parser.add_argument('--force', help='force run, remove file lock', action='store_true')
     parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
+    parser.add_argument('--no-log', help='redirect all log messages to stderr', action='store_true')
+    parser.add_argument('--no-report', help='force disable reporting to web service', action='store_true')
     parser.add_argument('--unsafe', help='allow to run ahriman as non-ahriman user', action='store_true')
     parser.add_argument('-v', '--version', action='version', version=version.__version__)
     subparsers = parser.add_subparsers(title='command')
@@ -178,7 +180,7 @@ if __name__ == '__main__':
     add_parser.add_argument('--without-dependencies', help='do not add dependencies', action='store_true')
     add_parser.set_defaults(fn=add)
 
-    check_parser = subparsers.add_parser('check', description='check for updates')
+    check_parser = subparsers.add_parser('check', description='check for updates. Same as update --dry-run --no-manual')
     check_parser.add_argument('package', help='filter check by packages', nargs='*')
     check_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
     check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=True)
@@ -216,20 +218,20 @@ if __name__ == '__main__':
     update_parser.add_argument('package', help='filter check by packages', nargs='*')
     update_parser.add_argument(
         '--dry-run', help='just perform check for updates, same as check command', action='store_true')
-    update_parser.add_argument('--no-aur', help='do not check for AUR updates', action='store_true')
+    update_parser.add_argument('--no-aur', help='do not check for AUR updates. Implies --no-vcs', action='store_true')
     update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true')
     update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
     update_parser.set_defaults(fn=update)
 
     web_parser = subparsers.add_parser('web', description='start web server')
-    web_parser.set_defaults(fn=web, lock=None)
+    web_parser.set_defaults(fn=web, lock=None, no_report=True)
 
     cmd_args = parser.parse_args()
     if 'fn' not in cmd_args:
         parser.print_help()
         sys.exit(1)
 
-    configuration = Configuration.from_path(cmd_args.config)
+    configuration = Configuration.from_path(cmd_args.config, not cmd_args.no_log)
     with Pool(len(cmd_args.architecture)) as pool:
         result = pool.starmap(
             _call, [(cmd_args, architecture, configuration) for architecture in cmd_args.architecture])

src/ahriman/application/application.py

@@ -25,7 +25,7 @@ from typing import Callable, Iterable, List, Optional, Set
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
-from ahriman.core.repository import Repository
+from ahriman.repository.repository import Repository
 from ahriman.core.tree import Tree
 from ahriman.models.package import Package
 

src/ahriman/application/lock.py

@@ -53,7 +53,7 @@ class Lock:
         self.unsafe = args.unsafe
 
         self.root = config.get('repository', 'root')
-        self.reporter = Client.load(architecture, config)
+        self.reporter = Client() if args.no_report else Client.load(architecture, config)
 
     def __enter__(self) -> Lock:
         '''

src/ahriman/core/configuration.py

@@ -37,7 +37,7 @@ class Configuration(configparser.RawConfigParser):
     :cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump)
     '''
 
-    DEFAULT_LOG_FORMAT = '%(asctime)s : %(levelname)s : %(funcName)s : %(message)s'
+    DEFAULT_LOG_FORMAT = '[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s'
     DEFAULT_LOG_LEVEL = logging.DEBUG
 
     STATIC_SECTIONS = ['alpm', 'report', 'repository', 'settings', 'upload']
@@ -45,7 +45,7 @@ class Configuration(configparser.RawConfigParser):
 
     def __init__(self) -> None:
         '''
-        default constructor
+        default constructor. In the most cases must not be called directly
         '''
         configparser.RawConfigParser.__init__(self, allow_no_value=True)
         self.path: Optional[str] = None
@@ -58,15 +58,16 @@ class Configuration(configparser.RawConfigParser):
         return self.get('settings', 'include')
 
     @classmethod
-    def from_path(cls: Type[Configuration], path: str) -> Configuration:
+    def from_path(cls: Type[Configuration], path: str, logfile: bool) -> Configuration:
         '''
         constructor with full object initialization
         :param path: path to root configuration file
+        :param logfile: use log file to output messages
         :return: configuration instance
         '''
         config = cls()
         config.load(path)
-        config.load_logging()
+        config.load_logging(logfile)
         return config
 
     def dump(self, architecture: str) -> Dict[str, Dict[str, str]]:
@@ -129,13 +130,23 @@ class Configuration(configparser.RawConfigParser):
         except (FileNotFoundError, configparser.NoOptionError):
             pass
 
-    def load_logging(self) -> None:
+    def load_logging(self, logfile: bool) -> None:
         '''
         setup logging settings from configuration
+        :param logfile: use log file to output messages
         '''
-        try:
+        def file_logger() -> None:
-            fileConfig(self.get('settings', 'logging'))
+            try:
-        except PermissionError:
+                fileConfig(self.get('settings', 'logging'))
+            except PermissionError:
+                console_logger()
+                logging.error('could not create logfile, fallback to stderr', exc_info=True)
+
+        def console_logger() -> None:
             logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT,
                                 level=Configuration.DEFAULT_LOG_LEVEL)
-            logging.error('could not create logfile, fallback to stderr', exc_info=True)
+
+        if logfile:
+            file_logger()
+        else:
+            console_logger()

src/ahriman/core/repository.py

@@ -1,297 +0,0 @@
-#
-# Copyright (c) 2021 Evgenii Alekseev.
-#
-# This file is part of ahriman
-# (see https://github.com/arcan1s/ahriman).
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import logging
-import os
-import shutil
-
-from typing import Dict, Iterable, List, Optional
-
-from ahriman.core.alpm.pacman import Pacman
-from ahriman.core.alpm.repo import Repo
-from ahriman.core.build_tools.task import Task
-from ahriman.core.configuration import Configuration
-from ahriman.core.report.report import Report
-from ahriman.core.sign.gpg import GPG
-from ahriman.core.upload.uploader import Uploader
-from ahriman.core.util import package_like
-from ahriman.core.watcher.client import Client
-from ahriman.models.package import Package
-from ahriman.models.repository_paths import RepositoryPaths
-
-
-class Repository:
-    '''
-    base repository control class
-    :ivar architecture: repository architecture
-    :ivar aur_url: base AUR url
-    :ivar config: configuration instance
-    :ivar logger: class logger
-    :ivar name: repository name
-    :ivar pacman: alpm wrapper instance
-    :ivar paths: repository paths instance
-    :ivar repo: repo commands wrapper instance
-    :ivar reporter: build status reporter instance
-    :ivar sign: GPG wrapper instance
-    '''
-
-    def __init__(self, architecture: str, config: Configuration) -> None:
-        '''
-        default constructor
-        :param architecture: repository architecture
-        :param config: configuration instance
-        '''
-        self.logger = logging.getLogger('builder')
-        self.architecture = architecture
-        self.config = config
-
-        self.aur_url = config.get('alpm', 'aur_url')
-        self.name = config.get('repository', 'name')
-
-        self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
-        self.paths.create_tree()
-
-        self.pacman = Pacman(config)
-        self.sign = GPG(architecture, config)
-        self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
-        self.reporter = Client.load(architecture, config)
-
-    def clear_build(self) -> None:
-        '''
-        clear sources directory
-        '''
-        for package in os.listdir(self.paths.sources):
-            shutil.rmtree(os.path.join(self.paths.sources, package))
-
-    def clear_cache(self) -> None:
-        '''
-        clear cache directory
-        '''
-        for package in os.listdir(self.paths.cache):
-            shutil.rmtree(os.path.join(self.paths.cache, package))
-
-    def clear_chroot(self) -> None:
-        '''
-        clear cache directory. Warning: this method is architecture independent and will clear every chroot
-        '''
-        for chroot in os.listdir(self.paths.chroot):
-            shutil.rmtree(os.path.join(self.paths.chroot, chroot))
-
-    def clear_manual(self) -> None:
-        '''
-        clear directory with manual package updates
-        '''
-        for package in os.listdir(self.paths.manual):
-            shutil.rmtree(os.path.join(self.paths.manual, package))
-
-    def clear_packages(self) -> None:
-        '''
-        clear directory with built packages (NOT repository itself)
-        '''
-        for package in self.packages_built():
-            os.remove(package)
-
-    def packages(self) -> List[Package]:
-        '''
-        generate list of repository packages
-        :return: list of packages properties
-        '''
-        result: Dict[str, Package] = {}
-        for fn in os.listdir(self.paths.repository):
-            if not package_like(fn):
-                continue
-            full_path = os.path.join(self.paths.repository, fn)
-            try:
-                local = Package.load(full_path, self.pacman, self.aur_url)
-                result.setdefault(local.base, local).packages.update(local.packages)
-            except Exception:
-                self.logger.exception(f'could not load package from {fn}', exc_info=True)
-                continue
-        return list(result.values())
-
-    def packages_built(self) -> List[str]:
-        '''
-        get list of files in built packages directory
-        :return: list of filenames from the directory
-        '''
-        return [
-            os.path.join(self.paths.packages, fn)
-            for fn in os.listdir(self.paths.packages)
-        ]
-
-    def process_build(self, updates: Iterable[Package]) -> List[str]:
-        '''
-        build packages
-        :param updates: list of packages properties to build
-        :return: `packages_built`
-        '''
-        def build_single(package: Package) -> None:
-            self.reporter.set_building(package.base)
-            task = Task(package, self.architecture, self.config, self.paths)
-            task.init()
-            built = task.build()
-            for src in built:
-                dst = os.path.join(self.paths.packages, os.path.basename(src))
-                shutil.move(src, dst)
-
-        for package in updates:
-            try:
-                build_single(package)
-            except Exception:
-                self.reporter.set_failed(package.base)
-                self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
-                continue
-        self.clear_build()
-
-        return self.packages_built()
-
-    def process_remove(self, packages: Iterable[str]) -> str:
-        '''
-        remove packages from list
-        :param packages: list of package names or bases to rmeove
-        :return: path to repository database
-        '''
-        def remove_single(package: str) -> None:
-            try:
-                self.repo.remove(package)
-            except Exception:
-                self.logger.exception(f'could not remove {package}', exc_info=True)
-
-        requested = set(packages)
-        for local in self.packages():
-            if local.base in packages:
-                to_remove = set(local.packages.keys())
-                self.reporter.remove(local.base)  # we only update status page in case of base removal
-            elif requested.intersection(local.packages.keys()):
-                to_remove = requested.intersection(local.packages.keys())
-            else:
-                to_remove = set()
-            for package in to_remove:
-                remove_single(package)
-
-        return self.repo.repo_path
-
-    def process_report(self, targets: Optional[Iterable[str]]) -> None:
-        '''
-        generate reports
-        :param targets: list of targets to generate reports. Configuration option will be used if it is not set
-        '''
-        if targets is None:
-            targets = self.config.getlist('report', 'target')
-        for target in targets:
-            Report.run(self.architecture, self.config, target, self.packages())
-
-    def process_sync(self, targets: Optional[Iterable[str]]) -> None:
-        '''
-        process synchronization to remote servers
-        :param targets: list of targets to sync. Configuration option will be used if it is not set
-        '''
-        if targets is None:
-            targets = self.config.getlist('upload', 'target')
-        for target in targets:
-            Uploader.run(self.architecture, self.config, target, self.paths.repository)
-
-    def process_update(self, packages: Iterable[str]) -> str:
-        '''
-        sign packages, add them to repository and update repository database
-        :param packages: list of filenames to run
-        :return: path to repository database
-        '''
-        def update_single(fn: Optional[str], base: str) -> None:
-            if fn is None:
-                self.logger.warning(f'received empty package name for base {base}')
-                return  # suppress type checking, it never can be none actually
-            files = self.sign.sign_package(fn, base)
-            for src in files:
-                dst = os.path.join(self.paths.repository, os.path.basename(src))
-                shutil.move(src, dst)
-            package_fn = os.path.join(self.paths.repository, os.path.basename(fn))
-            self.repo.add(package_fn)
-
-        # we are iterating over bases, not single packages
-        updates: Dict[str, Package] = {}
-        for fn in packages:
-            local = Package.load(fn, self.pacman, self.aur_url)
-            updates.setdefault(local.base, local).packages.update(local.packages)
-
-        for local in updates.values():
-            try:
-                for description in local.packages.values():
-                    update_single(description.filename, local.base)
-                self.reporter.set_success(local)
-            except Exception:
-                self.reporter.set_failed(local.base)
-                self.logger.exception(f'could not process {local.base}', exc_info=True)
-        self.clear_packages()
-
-        return self.repo.repo_path
-
-    def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
-        '''
-        check AUR for updates
-        :param filter_packages: do not check every package just specified in the list
-        :param no_vcs: do not check VCS packages
-        :return: list of packages which are out-of-dated
-        '''
-        result: List[Package] = []
-
-        build_section = self.config.get_section_name('build', self.architecture)
-        ignore_list = self.config.getlist(build_section, 'ignore_packages')
-
-        for local in self.packages():
-            if local.base in ignore_list:
-                continue
-            if local.is_vcs and no_vcs:
-                continue
-            if filter_packages and local.base not in filter_packages:
-                continue
-
-            try:
-                remote = Package.load(local.base, self.pacman, self.aur_url)
-                if local.is_outdated(remote, self.paths):
-                    self.reporter.set_pending(local.base)
-                    result.append(remote)
-            except Exception:
-                self.reporter.set_failed(local.base)
-                self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
-                continue
-
-        return result
-
-    def updates_manual(self) -> List[Package]:
-        '''
-        check for packages for which manual update has been requested
-        :return: list of packages which are out-of-dated
-        '''
-        result: List[Package] = []
-        known_bases = {package.base for package in self.packages()}
-
-        for fn in os.listdir(self.paths.manual):
-            try:
-                local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
-                result.append(local)
-                if local.base not in known_bases:
-                    self.reporter.set_unknown(local)
-                else:
-                    self.reporter.set_pending(local.base)
-            except Exception:
-                self.logger.exception(f'could not add package from {fn}', exc_info=True)
-        self.clear_manual()
-
-        return result

src/ahriman/core/watcher/watcher.py

@@ -24,7 +24,7 @@ import os
 from typing import Any, Dict, List, Optional, Tuple
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.repository import Repository
+from ahriman.repository.repository import Repository
 from ahriman.models.build_status import BuildStatus, BuildStatusEnum
 from ahriman.models.package import Package
 
@@ -58,7 +58,7 @@ class Watcher:
         '''
         :return: path to dump with json cache
         '''
-        return os.path.join(self.repository.paths.root, 'cache.json')
+        return os.path.join(self.repository.paths.root, 'status_cache.json')
 
     @property
     def packages(self) -> List[Tuple[Package, BuildStatus]]:
@@ -99,8 +99,11 @@ class Watcher:
                 } for package, status in self.packages
             ]
         }
-        with open(self.cache_path, 'w') as cache:
+        try:
-            json.dump(dump, cache)
+            with open(self.cache_path, 'w') as cache:
+                json.dump(dump, cache)
+        except Exception:
+            self.logger.exception('cannot dump cache', exc_info=True)
 
     def get(self, base: str) -> Tuple[Package, BuildStatus]:
         '''

src/ahriman/core/watcher/web_client.py

@@ -20,8 +20,6 @@
 import logging
 import requests
 
-from dataclasses import asdict
-
 from ahriman.core.watcher.client import Client
 from ahriman.models.build_status import BuildStatusEnum
 from ahriman.models.package import Package
@@ -68,12 +66,14 @@ class WebClient(Client):
         '''
         payload = {
             'status': status.value,
-            'package': asdict(package)
+            'package': package.view()
         }
 
         try:
             response = requests.post(self._package_url(package.base), json=payload)
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not add {package.base}: {e.response.text}', exc_info=True)
         except Exception:
             self.logger.exception(f'could not add {package.base}', exc_info=True)
 
@@ -85,6 +85,8 @@ class WebClient(Client):
         try:
             response = requests.delete(self._package_url(base))
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not delete {base}: {e.response.text}', exc_info=True)
         except Exception:
             self.logger.exception(f'could not delete {base}', exc_info=True)
 
@@ -99,6 +101,8 @@ class WebClient(Client):
         try:
             response = requests.post(self._package_url(base), json=payload)
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not update {base}: {e.response.text}', exc_info=True)
         except Exception:
             self.logger.exception(f'could not update {base}', exc_info=True)
 
@@ -112,5 +116,7 @@ class WebClient(Client):
         try:
             response = requests.post(self._ahriman_url(), json=payload)
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not update service status: {e.response.text}', exc_info=True)
         except Exception:
             self.logger.exception('could not update service status', exc_info=True)

src/ahriman/models/package.py

@@ -126,7 +126,7 @@ class Package:
         '''
         packages = {
             key: PackageDescription(**value)
-            for key, value in dump.get('packages', {})
+            for key, value in dump.get('packages', {}).items()
         }
         return Package(
             base=dump['base'],

src/ahriman/repository/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

src/ahriman/repository/cleaner.py

@@ -0,0 +1,78 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+import shutil
+
+from typing import List
+
+from ahriman.repository.properties import Properties
+
+
+class Cleaner(Properties):
+    '''
+    trait to clean common repository objects
+    '''
+
+    def packages_built(self) -> List[str]:
+        '''
+        get list of files in built packages directory
+        :return: list of filenames from the directory
+        '''
+        raise NotImplementedError
+
+    def clear_build(self) -> None:
+        '''
+        clear sources directory
+        '''
+        self.logger.info('clear package sources directory')
+        for package in os.listdir(self.paths.sources):
+            shutil.rmtree(os.path.join(self.paths.sources, package))
+
+    def clear_cache(self) -> None:
+        '''
+        clear cache directory
+        '''
+        self.logger.info('clear packages sources cache directory')
+        for package in os.listdir(self.paths.cache):
+            shutil.rmtree(os.path.join(self.paths.cache, package))
+
+    def clear_chroot(self) -> None:
+        '''
+        clear cache directory. Warning: this method is architecture independent and will clear every chroot
+        '''
+        self.logger.info('clear build chroot directory')
+        for chroot in os.listdir(self.paths.chroot):
+            shutil.rmtree(os.path.join(self.paths.chroot, chroot))
+
+    def clear_manual(self) -> None:
+        '''
+        clear directory with manual package updates
+        '''
+        self.logger.info('clear manual packages')
+        for package in os.listdir(self.paths.manual):
+            shutil.rmtree(os.path.join(self.paths.manual, package))
+
+    def clear_packages(self) -> None:
+        '''
+        clear directory with built packages (NOT repository itself)
+        '''
+        self.logger.info('clear built packages directory')
+        for package in self.packages_built():
+            os.remove(package)

src/ahriman/repository/executor.py

@@ -0,0 +1,151 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+import shutil
+
+from typing import Dict, Iterable, List, Optional
+
+from ahriman.core.build_tools.task import Task
+from ahriman.core.report.report import Report
+from ahriman.core.upload.uploader import Uploader
+from ahriman.models.package import Package
+from ahriman.repository.cleaner import Cleaner
+
+
+class Executor(Cleaner):
+    '''
+    trait for common repository update processes
+    '''
+
+    def packages(self) -> List[Package]:
+        '''
+        generate list of repository packages
+        :return: list of packages properties
+        '''
+        raise NotImplementedError
+
+    def process_build(self, updates: Iterable[Package]) -> List[str]:
+        '''
+        build packages
+        :param updates: list of packages properties to build
+        :return: `packages_built`
+        '''
+        def build_single(package: Package) -> None:
+            self.reporter.set_building(package.base)
+            task = Task(package, self.architecture, self.config, self.paths)
+            task.init()
+            built = task.build()
+            for src in built:
+                dst = os.path.join(self.paths.packages, os.path.basename(src))
+                shutil.move(src, dst)
+
+        for package in updates:
+            try:
+                build_single(package)
+            except Exception:
+                self.reporter.set_failed(package.base)
+                self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
+                continue
+        self.clear_build()
+
+        return self.packages_built()
+
+    def process_remove(self, packages: Iterable[str]) -> str:
+        '''
+        remove packages from list
+        :param packages: list of package names or bases to rmeove
+        :return: path to repository database
+        '''
+        def remove_single(package: str) -> None:
+            try:
+                self.repo.remove(package)
+            except Exception:
+                self.logger.exception(f'could not remove {package}', exc_info=True)
+
+        requested = set(packages)
+        for local in self.packages():
+            if local.base in packages:
+                to_remove = set(local.packages.keys())
+                self.reporter.remove(local.base)  # we only update status page in case of base removal
+            elif requested.intersection(local.packages.keys()):
+                to_remove = requested.intersection(local.packages.keys())
+            else:
+                to_remove = set()
+            for package in to_remove:
+                remove_single(package)
+
+        return self.repo.repo_path
+
+    def process_report(self, targets: Optional[Iterable[str]]) -> None:
+        '''
+        generate reports
+        :param targets: list of targets to generate reports. Configuration option will be used if it is not set
+        '''
+        if targets is None:
+            targets = self.config.getlist('report', 'target')
+        for target in targets:
+            Report.run(self.architecture, self.config, target, self.packages())
+
+    def process_sync(self, targets: Optional[Iterable[str]]) -> None:
+        '''
+        process synchronization to remote servers
+        :param targets: list of targets to sync. Configuration option will be used if it is not set
+        '''
+        if targets is None:
+            targets = self.config.getlist('upload', 'target')
+        for target in targets:
+            Uploader.run(self.architecture, self.config, target, self.paths.repository)
+
+    def process_update(self, packages: Iterable[str]) -> str:
+        '''
+        sign packages, add them to repository and update repository database
+        :param packages: list of filenames to run
+        :return: path to repository database
+        '''
+        def update_single(fn: Optional[str], base: str) -> None:
+            if fn is None:
+                self.logger.warning(f'received empty package name for base {base}')
+                return  # suppress type checking, it never can be none actually
+            # in theory it might be NOT packages directory, but we suppose it is
+            full_path = os.path.join(self.paths.packages, fn)
+            files = self.sign.sign_package(full_path, base)
+            for src in files:
+                dst = os.path.join(self.paths.repository, os.path.basename(src))
+                shutil.move(src, dst)
+            package_path = os.path.join(self.paths.repository, fn)
+            self.repo.add(package_path)
+
+        # we are iterating over bases, not single packages
+        updates: Dict[str, Package] = {}
+        for fn in packages:
+            local = Package.load(fn, self.pacman, self.aur_url)
+            updates.setdefault(local.base, local).packages.update(local.packages)
+
+        for local in updates.values():
+            try:
+                for description in local.packages.values():
+                    update_single(description.filename, local.base)
+                self.reporter.set_success(local)
+            except Exception:
+                self.reporter.set_failed(local.base)
+                self.logger.exception(f'could not process {local.base}', exc_info=True)
+        self.clear_packages()
+
+        return self.repo.repo_path

src/ahriman/repository/properties.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+
+from ahriman.core.alpm.pacman import Pacman
+from ahriman.core.alpm.repo import Repo
+from ahriman.core.configuration import Configuration
+from ahriman.core.sign.gpg import GPG
+from ahriman.core.watcher.client import Client
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+class Properties:
+    '''
+    repository internal objects holder
+    :ivar architecture: repository architecture
+    :ivar aur_url: base AUR url
+    :ivar config: configuration instance
+    :ivar logger: class logger
+    :ivar name: repository name
+    :ivar pacman: alpm wrapper instance
+    :ivar paths: repository paths instance
+    :ivar repo: repo commands wrapper instance
+    :ivar reporter: build status reporter instance
+    :ivar sign: GPG wrapper instance
+    '''
+
+    def __init__(self, architecture: str, config: Configuration) -> None:
+        self.logger = logging.getLogger('builder')
+        self.architecture = architecture
+        self.config = config
+
+        self.aur_url = config.get('alpm', 'aur_url')
+        self.name = config.get('repository', 'name')
+
+        self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
+        self.paths.create_tree()
+
+        self.pacman = Pacman(config)
+        self.sign = GPG(architecture, config)
+        self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
+        self.reporter = Client.load(architecture, config)

src/ahriman/repository/repository.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+
+from typing import Dict, List
+
+from ahriman.core.util import package_like
+from ahriman.models.package import Package
+from ahriman.repository.executor import Executor
+from ahriman.repository.update_handler import UpdateHandler
+
+
+class Repository(Executor, UpdateHandler):
+    '''
+    base repository control class
+    '''
+
+    def packages(self) -> List[Package]:
+        '''
+        generate list of repository packages
+        :return: list of packages properties
+        '''
+        result: Dict[str, Package] = {}
+        for fn in os.listdir(self.paths.repository):
+            if not package_like(fn):
+                continue
+            full_path = os.path.join(self.paths.repository, fn)
+            try:
+                local = Package.load(full_path, self.pacman, self.aur_url)
+                result.setdefault(local.base, local).packages.update(local.packages)
+            except Exception:
+                self.logger.exception(f'could not load package from {fn}', exc_info=True)
+                continue
+        return list(result.values())
+
+    def packages_built(self) -> List[str]:
+        '''
+        get list of files in built packages directory
+        :return: list of filenames from the directory
+        '''
+        return [
+            os.path.join(self.paths.packages, fn)
+            for fn in os.listdir(self.paths.packages)
+        ]

src/ahriman/repository/update_handler.py

@@ -0,0 +1,92 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+
+from typing import Iterable, List
+
+from ahriman.models.package import Package
+from ahriman.repository.cleaner import Cleaner
+
+
+class UpdateHandler(Cleaner):
+    '''
+    trait to get package update list
+    '''
+
+    def packages(self) -> List[Package]:
+        '''
+        generate list of repository packages
+        :return: list of packages properties
+        '''
+        raise NotImplementedError
+
+    def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
+        '''
+        check AUR for updates
+        :param filter_packages: do not check every package just specified in the list
+        :param no_vcs: do not check VCS packages
+        :return: list of packages which are out-of-dated
+        '''
+        result: List[Package] = []
+
+        build_section = self.config.get_section_name('build', self.architecture)
+        ignore_list = self.config.getlist(build_section, 'ignore_packages')
+
+        for local in self.packages():
+            if local.base in ignore_list:
+                continue
+            if local.is_vcs and no_vcs:
+                continue
+            if filter_packages and local.base not in filter_packages:
+                continue
+
+            try:
+                remote = Package.load(local.base, self.pacman, self.aur_url)
+                if local.is_outdated(remote, self.paths):
+                    self.reporter.set_pending(local.base)
+                    result.append(remote)
+            except Exception:
+                self.reporter.set_failed(local.base)
+                self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
+                continue
+
+        return result
+
+    def updates_manual(self) -> List[Package]:
+        '''
+        check for packages for which manual update has been requested
+        :return: list of packages which are out-of-dated
+        '''
+        result: List[Package] = []
+        known_bases = {package.base for package in self.packages()}
+
+        for fn in os.listdir(self.paths.manual):
+            try:
+                local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
+                result.append(local)
+                if local.base not in known_bases:
+                    self.reporter.set_unknown(local)
+                else:
+                    self.reporter.set_pending(local.base)
+            except Exception:
+                self.logger.exception(f'could not add package from {fn}', exc_info=True)
+        self.clear_manual()
+
+        return result