report support, readme

CONFIGURING.md

@@ -0,0 +1,67 @@
+# ahriman configuration
+
+## `settings` group
+
+Base configuration settings:
+
+* `include` - path to directory with configuration files overrides, string, required.
+* `logging` - path to logging configuration, string, required. Check `logging.ini` for reference.
+
+## `aur` group
+
+AUR related configuration:
+
+* `url` - base url for AUR, string, required.
+
+## `build` group
+
+Build related configuration:
+
+* `archbuild_flags` - additional flags passed to `archbuild` command, space separated list of strings, optional.
+* `build_command` - default build command, string, required.
+* `makepkg_flags` - additional flags passed to `makepkg` command, space separated list of strings, optional.
+* `makechrootpkg_flags` - additional flags passed to `makechrootpkg` command, space separated list of strings, optional.
+
+## `repository` group
+
+Base repository settings:
+
+* `name` - repository name, string, required.
+* `root` - root path for application, string, required.
+
+## `sign` group
+
+Settings for signing packages or repository:
+
+* `enabled` - configuration flag to enable signing, string, required. Allowed values are `disabled`, `package` (sign each package separately), `repository` (sign repository database file).
+* `key` - PGP key, string, optional.
+
+## `report` group
+
+Report generation settings:
+
+* `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`.
+
+### `html` group
+
+* `path` - path to html report file, string, required.
+* `css_path` - path to CSS to include in HTML, string, optional.
+* `link_path` - prefix for HTML links, string, required.
+
+## `upload` group
+
+Remote synchronization settings:
+
+* `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`.
+
+### `s3`
+
+Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`.
+
+* `bucket` - bucket name (e.g. `s3://bucket/path`), string, required.
+
+### `rsync`
+
+Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`.
+
+* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required.

README.md

@@ -0,0 +1,20 @@
+# ArcHlinux ReposItory MANager
+
+Wrapper for managing custom repository inspired by [repo-scripts](https://github.com/arcan1s/repo-scripts).
+
+## Installation and run
+
+* Install package as usual.
+* Change settings if required, see `CONFIGURING.md` for more details.
+* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`).
+* Configure build tools (it might be required if your package will use any custom repositories):
+    * create build command if required, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command);
+    * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,custom}.conf`;
+    * change configuration file: add your own repository, add multilib repository;
+    * set `build.build_command` to point to your command.
+* Start and enable `ahriman.timer` via `systemctl`.
+* Add packages by using `ahriman add {package}` command.
+
+## Limitations
+
+* It does not manage dependencies, so you have to add them before main package.

make_release.sh

@@ -4,7 +4,7 @@ set -e
 
 VERSION="$1"
 ARCHIVE="ahriman"
-FILES="COPYING README.md package src setup.py"
+FILES="COPYING CONFIGURING.md README.md package src setup.py"
 IGNORELIST="build .idea package/archlinux package/*src.tar.xz"
 
 # set version

package/etc/ahriman.ini

@@ -6,10 +6,10 @@ logging = /etc/ahriman.ini.d/logging.ini
 url = https://aur.archlinux.org
 
 [build]
-archbuild_flags = -c
-extra_build = extra-x86_64-build
+archbuild_flags =
+build_command = extra-x86_64-build
+makechrootpkg_flags =
 makepkg_flags = --skippgpcheck
-multilib_build = multilib-build
 
 [repository]
 name = aur-clone
@@ -19,11 +19,19 @@ root = /var/lib/ahriman
 enabled = disabled
 key =
 
+[report]
+target =
+
+[html]
+path =
+css_path =
+link_path =
+
 [upload]
-enabled = disabled
+target =
 
 [s3]
 bucket =
 
 [rsync]
-remote =
+remote =

src/ahriman/application/ahriman.py

@@ -51,12 +51,17 @@ def remove(args: argparse.Namespace) -> None:
     _get_app(args).remove(args.package)
 
 
+def report(args: argparse.Namespace) -> None:
+    _get_app(args).report(args.target)
+
+
 def sync(args: argparse.Namespace) -> None:
-    _get_app(args).sync()
+    _get_app(args).sync(args.target)
 
 
 def update(args: argparse.Namespace) -> None:
-    _get_app(args).update(args.sync)
+    check_only = (args.command == 'check')
+    _get_app(args).update(check_only)
 
 
 if __name__ == '__main__':
@@ -65,21 +70,28 @@ if __name__ == '__main__':
     parser.add_argument('--force', help='force run, remove file lock', action='store_true')
     parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
     parser.add_argument('-v', '--version', action='version', version=version.__version__)
-    subparsers = parser.add_subparsers(title='commands')
+    subparsers = parser.add_subparsers(title='command')
 
     add_parser = subparsers.add_parser('add', description='add package')
     add_parser.add_argument('package', help='package name', nargs='+')
     add_parser.set_defaults(fn=add)
 
+    check_parser = subparsers.add_parser('check', description='check for updates')
+    check_parser.set_defaults(fn=update)
+
     remove_parser = subparsers.add_parser('remove', description='remove package')
     remove_parser.add_argument('package', help='package name', nargs='+')
     remove_parser.set_defaults(fn=remove)
 
+    report_parser = subparsers.add_parser('report', description='generate report')
+    report_parser.add_argument('target', help='target to generate report', nargs='*')
+    report_parser.set_defaults(fn=report)
+
     sync_parser = subparsers.add_parser('sync', description='sync packages to remote server')
+    sync_parser.add_argument('target', help='target to sync', nargs='*')
     sync_parser.set_defaults(fn=sync)
 
     update_parser = subparsers.add_parser('update', description='run updates')
-    update_parser.add_argument('-s', '--sync', help='sync packages to remote server', action='store_true')
     update_parser.set_defaults(fn=update)
 
     args = parser.parse_args()

src/ahriman/application/application.py

@@ -17,9 +17,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import logging
 import os
 
-from typing import List
+from typing import List, Optional
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
@@ -30,6 +31,7 @@ from ahriman.models.package import Package
 class Application:
 
     def __init__(self, config: Configuration) -> None:
+        self.logger = logging.getLogger('root')
         self.config = config
         self.repository = Repository(config)
 
@@ -42,12 +44,25 @@ class Application:
     def remove(self, names: List[str]) -> None:
         self.repository.process_remove(names)
 
-    def sync(self) -> None:
-        self.repository.process_sync()
+    def report(self, target: Optional[List[str]] = None) -> None:
+        targets = target or None
+        self.repository.process_report(targets)
 
-    def update(self, sync: bool) -> None:
+    def sync(self, target: Optional[List[str]] = None) -> None:
+        targets = target or None
+        self.repository.process_sync(targets)
+
+    def update(self, dry_run: bool) -> None:
         updates = self.repository.updates()
+        log_fn = print if dry_run else self.logger.info
+        for package in updates:
+            log_fn(f'{package.name} = {package.version}')  # type: ignore
+
+        if dry_run:
+            return
+
         packages = self.repository.process_build(updates)
         self.repository.process_update(packages)
-        if sync:
-            self.sync()
+
+        self.report()
+        self.sync()

src/ahriman/core/build_tools/task.py

@@ -25,7 +25,7 @@ from typing import List, Optional
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.exceptions import BuildFailed
-from ahriman.core.util import check_output
+from ahriman.core.util import check_output, options_list
 from ahriman.models.package import Package
 from ahriman.models.repository_paths import RepositoryPaths
 
@@ -38,22 +38,20 @@ class Task:
         self.package = package
         self.paths = paths
 
-        self.archbuild_flags = config.get('build_tools', 'archbuild_flags').split()
-        self.extra_build = config.get('build_tools', 'extra_build')
-        self.makepkg_flags = config.get('build_tools', 'makepkg_flags').split()
-        self.multilib_build = config.get('build_tools', 'multilib_build')
+        self.archbuild_flags = options_list(config, 'build', 'archbuild_flags')
+        self.build_command = config.get('build', 'build_command')
+        self.makepkg_flags = options_list(config, 'build', 'makepkg_flags')
+        self.makechrootpkg_flags = options_list(config, 'build', 'makechrootpkg_flags')
 
     @property
     def git_path(self) -> str:
         return os.path.join(self.paths.sources, self.package.name)
 
     def build(self) -> List[str]:
-        build_tool = self.multilib_build if self.package.is_multilib else self.extra_build
-
-        cmd = [build_tool, '-r', self.paths.chroot]
+        cmd = [self.build_command, '-r', self.paths.chroot]
         cmd.extend(self.archbuild_flags)
-        if self.makepkg_flags:
-            cmd.extend(['--', '--'] + self.makepkg_flags)
+        cmd.extend(['--'] + self.makechrootpkg_flags)
+        cmd.extend(['--'] + self.makepkg_flags)
         self.logger.info(f'using {cmd} for {self.package.name}')
 
         check_output(

src/ahriman/core/exceptions.py

@@ -40,6 +40,11 @@ class MissingConfiguration(Exception):
         Exception.__init__(self, f'No section `{name}` found')
 
 
+class ReportFailed(Exception):
+    def __init__(self, cause: Exception) -> None:
+        Exception.__init__(self, f'Report failed with reason {cause}')
+
+
 class SyncFailed(Exception):
     def __init__(self, cause: Exception) -> None:
         Exception.__init__(self, f'Sync failed with reason {cause}')

src/ahriman/core/report/dummy.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from ahriman.core.report.report import Report
+
+
+class Dummy(Report):
+
+    def generate(self, path: str) -> None:
+        pass

src/ahriman/core/report/html.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.report.report import Report
+
+
+class HTML(Report):
+
+    def __init__(self, config: Configuration) -> None:
+        Report.__init__(self, config)
+        self.report_path = config.get('html', 'path')
+        self.css_path = config.get('html', 'css_path')
+        self.link_path = config.get('html', 'link_path')
+        self.title = config.get('repository', 'name')
+
+    def generate(self, path: str) -> None:
+        # lets not use libraries here
+        html = f'''<html lang="en"><head><title>{self.title}</title>'''
+        if self.css_path:
+            html += f'''<link rel="stylesheet" type="text/css" href="{self.css_path}">'''
+        html += '''</head><body>'''
+
+        html += '''<ul>'''
+        for package in sorted(os.listdir(path)):
+            if '.pkg.' not in package:
+                continue
+            html += f'''<li><a href="{self.link_path}/{package}">{package}</a></li>'''
+        html += '''</ul>'''
+
+        html += '''</body></html>'''
+
+        with open(self.report_path, 'w') as out:
+            out.write(html)

src/ahriman/core/report/report.py

@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import ReportFailed
+from ahriman.models.report_settings import ReportSettings
+
+
+class Report:
+
+    def __init__(self, config: Configuration) -> None:
+        self.config = config
+        self.logger = logging.getLogger('builder')
+
+    @staticmethod
+    def run(config: Configuration, target: str, path: str) -> None:
+        provider = ReportSettings.from_option(target)
+        if provider == ReportSettings.HTML:
+            from ahriman.core.report.html import HTML
+            report: Report = HTML(config)
+        else:
+            from ahriman.core.report.dummy import Dummy
+            report = Dummy(config)
+
+        try:
+            report.generate(path)
+        except Exception as e:
+            raise ReportFailed(e) from e
+
+    def generate(self, path: str) -> None:
+        raise NotImplementedError

src/ahriman/core/repository.py

@@ -21,13 +21,15 @@ import logging
 import os
 import shutil
 
-from typing import List
+from typing import List, Optional
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
 from ahriman.core.repo.repo_wrapper import RepoWrapper
-from ahriman.core.sign.sign import Sign
+from ahriman.core.report.report import Report
+from ahriman.core.sign.gpg_wrapper import GPGWrapper
 from ahriman.core.upload.uploader import Uploader
+from ahriman.core.util import options_list
 from ahriman.models.package import Package
 from ahriman.models.repository_paths import RepositoryPaths
 
@@ -44,7 +46,7 @@ class Repository:
         self.paths = RepositoryPaths(config.get('repository', 'root'))
         self.paths.create_tree()
 
-        self.sign = Sign(config)
+        self.sign = GPGWrapper(config)
         self.wrapper = RepoWrapper(self.name, self.paths)
 
     def _clear_build(self) -> None:
@@ -96,8 +98,17 @@ class Repository:
         self.sign.sign_repository(self.wrapper.repo_path)
         return self.wrapper.repo_path
 
-    def process_sync(self) -> None:
-        return Uploader.run(self.config, self.paths.repository)
+    def process_report(self, targets: Optional[List[str]]) -> None:
+        if targets is None:
+            targets = options_list(self.config, 'report', 'target')
+        for target in targets:
+            Report.run(self.config, target, self.paths.repository)
+
+    def process_sync(self, targets: Optional[List[str]]) -> None:
+        if targets is None:
+            targets = options_list(self.config, 'upload', 'target')
+        for target in targets:
+            Uploader.run(self.config, target, self.paths.repository)
 
     def process_update(self, packages: List[str]) -> str:
         for package in packages:

src/ahriman/core/sign/gpg_wrapper.py

@@ -28,7 +28,7 @@ from ahriman.core.util import check_output
 from ahriman.models.sign_settings import SignSettings
 
 
-class Sign:
+class GPGWrapper:
 
     def __init__(self, config: Configuration) -> None:
         self.logger = logging.getLogger('build_details')

src/ahriman/core/upload/s3.py

@@ -29,7 +29,7 @@ class S3(Uploader):
         self.bucket = self.config.get('s3', 'bucket')
 
     def sync(self, path: str) -> None:
-        # TODO rewrite to boto, but it is bs
+        # TODO rewrite to boto, but it is bullshit
         check_output('aws', 's3', 'sync', path, self.bucket,
                      exception=None,
                      logger=self.logger)

src/ahriman/core/upload/uploader.py

@@ -31,8 +31,8 @@ class Uploader:
         self.logger = logging.getLogger('builder')
 
     @staticmethod
-    def run(config: Configuration, path: str) -> None:
-        provider = UploadSettings.from_option(config.get('upload', 'enabled'))
+    def run(config: Configuration, target: str, path: str) -> None:
+        provider = UploadSettings.from_option(target)
         if provider == UploadSettings.Rsync:
             from ahriman.core.upload.rsync import Rsync
             uploader: Uploader = Rsync(config)
@@ -46,7 +46,7 @@ class Uploader:
         try:
             uploader.sync(path)
         except Exception as e:
-            raise SyncFailed from e
+            raise SyncFailed(e) from e
 
     def sync(self, path: str) -> None:
         raise NotImplementedError

src/ahriman/core/util.py

@@ -20,7 +20,9 @@
 import subprocess
 
 from logging import Logger
-from typing import Optional
+from typing import List, Optional
+
+from ahriman.core.configuration import Configuration
 
 
 def check_output(*args: str, exception: Optional[Exception],
@@ -36,4 +38,11 @@ def check_output(*args: str, exception: Optional[Exception],
             for line in e.output.decode('utf8').splitlines():
                 logger.debug(line)
         raise exception or e
-    return result
+    return result
+
+
+def options_list(config: Configuration, section: str, key: str) -> List[str]:
+    raw = config.get(section, key, fallback=None)
+    if not raw:  # empty string or none
+        return []
+    return raw.split()

src/ahriman/models/package.py

@@ -37,10 +37,6 @@ class Package:
     version: str
     url: str
 
-    @property
-    def is_multilib(self) -> bool:
-        return self.name.startswith('lib32-')
-
     @classmethod
     def from_archive(cls: Type[Package], path: str, aur_url: str) -> Package:
         name, version = check_output('expac', '-p', '%n %v', path, exception=None).split()

src/ahriman/models/report_settings.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman 
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from enum import Enum, auto
+from typing import Type
+
+from ahriman.core.exceptions import InvalidOptionException
+
+
+class ReportSettings(Enum):
+    HTML = auto()
+
+    @classmethod
+    def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings:
+        if value.lower() in ('html',):
+            return cls.HTML
+        raise InvalidOptionException(value)

src/ahriman/models/upload_settings.py

@@ -26,15 +26,12 @@ from ahriman.core.exceptions import InvalidOptionException
 
 
 class UploadSettings(Enum):
-    Disabled = auto()
     Rsync = auto()
     S3 = auto()
 
     @classmethod
     def from_option(cls: Type[UploadSettings], value: str) -> UploadSettings:
-        if value.lower() in ('no', 'disabled'):
-            return cls.Disabled
-        elif value.lower() in ('rsync',):
+        if value.lower() in ('rsync',):
             return cls.Rsync
         elif value.lower() in ('s3',):
             return cls.S3