diff --git a/Makefile b/Makefile index f082e6c5..7b15133c 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PROJECT := ahriman -FILES := AUTHORS COPYING CONFIGURING.md README.md package src setup.cfg setup.py +FILES := AUTHORS COPYING CONFIGURING.md README.md docs package src setup.cfg setup.py TARGET_FILES := $(addprefix $(PROJECT)/, $(FILES)) IGNORE_FILES := package/archlinux src/.mypy_cache diff --git a/README.md b/README.md index e62edcd7..1b896549 100644 --- a/README.md +++ b/README.md @@ -13,61 +13,60 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github * Sign support with gpg (repository, package, per package settings). * Synchronization to remote services (rsync, s3) and report generation (html). * Dependency manager. -* Repository status interface with optional authorization. +* Repository status interface with optional authorization and control options: + + ![web interface](web.png) ## Installation and run -* Install package as usual. -* Change settings if required, see [CONFIGURING](CONFIGURING.md) for more details. -* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`): +For installation details please refer to the [documentation](docs/setup.md). For command help, `--help` subcommand must be used, e.g.: - ```shell - echo 'PACKAGER="John Doe "' | sudo -u ahriman tee -a /var/lib/ahriman/.makepkg.conf - ``` +```shell +$ ahriman --help +usage: ahriman [-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--no-log] [--no-report] [--unsafe] [-v] + {add,check,clean,config,create-user,init,key-import,rebuild,remove,remove-unknown,report,search,setup,sign,status,status-update,sync,update,web} ... -* Configure build tools (it is required for correct dependency management system): +ArcHlinux ReposItory MANager - * create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build` (you can choose any name for command, basically it should be `{name}-{arch}-build`); - * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,ahriman}.conf` (same as previous `pacman-{name}.conf`); - * change configuration file, add your own repository, add multilib repository etc; - * set `build_command` option to point to your command; - * configure `/etc/sudoers.d/ahriman` to allow running command without a password. +optional arguments: + -h, --help show this help message and exit + -a ARCHITECTURE, --architecture ARCHITECTURE + target architectures (can be used multiple times) (default: None) + -c CONFIGURATION, --configuration CONFIGURATION + configuration path (default: /etc/ahriman.ini) + --force force run, remove file lock (default: False) + -l LOCK, --lock LOCK lock file (default: /tmp/ahriman.lock) + --no-log redirect all log messages to stderr (default: False) + --no-report force disable reporting to web service (default: False) + --unsafe allow to run ahriman as non-ahriman user (default: False) + -v, --version show program's version number and exit - ```shell - ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build - cp /usr/share/devtools/pacman-{extra,ahriman}.conf +command: + {add,check,clean,config,create-user,init,key-import,rebuild,remove,remove-unknown,report,search,setup,sign,status,status-update,sync,update,web} + command to run + add add package + check check for updates + clean clean local caches + config dump configuration + create-user create user for web services + init create repository tree + key-import import PGP key + rebuild rebuild repository + remove remove package + remove-unknown remove unknown packages + report generate report + search search for package + setup initial service configuration + sign sign packages + status get package status + status-update update package status + sync sync repository + update update packages + web start web server +``` - echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf - echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman-ahriman.conf +Subcommands have own help message as well. - echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf - echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman-ahriman.conf - echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman-ahriman.conf +## Configuration - echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini - echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini - - echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman - echo 'ahriman ALL=(ALL) NOPASSWD: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman - chmod 400 /etc/sudoers.d/ahriman - ``` - -* Start and enable `ahriman@.timer` via `systemctl`: - - ```shell - systemctl enable --now ahriman@x86_64.timer - ``` - -* Start and enable status page: - - ```shell - systemctl enable --now ahriman-web@x86_64 - ``` - -* Add packages by using `ahriman add {package}` command: - - ```shell - sudo -u ahriman ahriman -a x86_64 add yay --now - ``` - -Note that initial service configuration can be done by running `ahriman setup` with specific arguments. +Every available option is described in the [documentation](docs/configuration.md). diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..cba25b8d --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,157 @@ +# Package structure + +Packages have strict rules of importing: + +* `ahriman.application` package must not be used anywhere except for itself. +* `ahriman.core` and `ahriman.models` packages don't have any import restriction. Actually we would like to totally restrict importing of `core` package from `models`, but it is impossible at the moment. +* `ahriman.web` package is allowed to be imported from `ahriman.application` (web handler only, only `ahriman.web.web` methods). It also must not be imported globally, only local import is allowed. + +Full dependency diagram: + +![architecture](ahriman-architecture.svg) + +## `ahriman.application` package + +This package contains application (aka executable) related classes and everything for that. It also contains package called `ahriman.application.handlers` in which all available subcommands are described as separated classes derived from base `ahriman.application.handlers.handler.Handler` class. `ahriman.application.ahriman` contains only command line parses and executes specified `Handler` on success, `ahriman.application.application.Application` is a god class which provides interfaces for all repository related actions. `ahriman.application.lock.Lock` is additional class which provides file-based lock and also performs some common checks. + +## `ahriman.core` package + +This package contains everything which is required for any time of application run and separated to several packages: + +* `ahriman.core.alpm` package controls pacman related functions. It provides wrappers for `pyalpm` library and safe calls for repository tools (`repo-add` and `repo-remove`). +* `ahriman.core.auth` package provides classes for authorization methods used by web mostly. Base class is `ahriman.core.auth.auth.Auth` which must be called by `load` method. +* `ahriman.core.build_tools` is a package which provides wrapper for `devtools` commands. +* `ahriman.core.report` is a package with reporting classes. Usually it must be called by `ahriman.core.report.report.Report.load` method. +* `ahriman.core.repository` contains several traits and base repository (`ahriman.core.repository.repository.Repository` class) implementation. +* `ahriman.core.sign` package provides sign feature (only gpg calls are available). +* `ahriman.core.status` contains helpers and watcher class which are required for web application. Reporter must be initialized by using `ahriman.core.status.client.Client.load` method. +* `ahriman.core.upload` package provides sync feature, must be called by `ahriman.core.upload.upload.Upload.load` method. + +This package also provides some generic functions and classes which may be used by other packages: + +* `ahriman.core.configuration.Configuration` is an extension for standard `configparser` library. +* `ahriman.core.exceptions` provides custom exceptions. +* `ahriman.core.spawn.Spawn` is a tool which can spawn another `ahriman` process. This feature is used by web application. +* `ahriman.core.tree` is a dependency tree implementation. + +## `ahriman.models` package + +It provides models for any other part of application. Unlike `ahriman.core` package classes from here provides only conversion methods (e.g. create class from another or convert to). Mostly case classes and enumerations. + +## `ahriman.web` package + +Web application. It is important that this package is isolated from any other to allow it to be optional feature (i.e. dependencies which are required by the package are optional). + +* `ahriman.web.middlewares` provides middlewares for request handlers. +* `ahriman.web.views` contains web views derived from aiohttp view class. +* `ahriman.web.routes` creates routes for web application. +* `ahriman.web.web` provides main web application functions (e.g. start, initialization). + +# Application run + +* Parse command line arguments, find command and related handler which is set by parser. +* Call `Handler.execute` method. +* Define list of architectures to run. In case if there is more than one architecture specified run several subprocesses or process in current process otherwise. Class attribute `ALLOW_MULTI_ARCHITECTURE_RUN` controls whether application can be run in multiple processes or not - this feature is required for some handlers (e.g. `Web`) which should be able to spawn child process in daemon mode (it is impossible to do for daemonic processes). +* In each child process call lock functions. +* After success checks pass control to `Handler.run` method defined by specific handler class. +* Return result (success or failure) of each subprocess and exit from application. + +In most cases handlers spawn god class `ahriman.application.application.Application` class and call required methods. + +Application is designed to run from `systemd` services and provides parametrized by architecture timer and service file for that. + +# Basic flows + +## Add new packages or rebuild existing + +Idea is to copy package to the directory from which it will be handled at the next update run. Different variants are supported: + +* If supplied argument is file then application moves the file to the directory with built packages. Same rule applies for directory, but in this case it copies every package-like file from the specified directory. +* If supplied argument iis not file then application tries to lookup for the specified name in AUR and clones it into the directory with manual updates. This scenario can also handle package dependencies which are missing in repositories. + +## Rebuild packages + +Same as add function for every package in repository. Optional filter by reverse dependency can be supplied. + +## Remove packages + +This flow removes package from filesystem, updates repository database and also runs synchronization and reporting methods. + +## Update packages + +This feature is divided into to stages: check AUR for updates and run rebuild for required packages. Whereas check does not do anything except for check itself, update flow is the following: + +1. Process every built package first. Those packages are usually added manually. +2. Run sync and report methods. +3. Generate dependency tree for packages to be built. +4. For each level of tree it does: + 1. Download package data from AUR. + 2. Build every package in clean chroot. + 3. Sign packages if required. + 4. Add packages to database and sign database if required. + 5. Process sync and report methods. + +After any step any package data is being removed. + +# Core functions reference + +## Configuration + +`ahriman.core.configuration.Configuration` class provides some additional methods (e.g. `getpath` and `getlist`) and also combines multiple files into single configuration dictionary using architecture overrides. It is recommended to read class related settings from the class, not outside. + +## Utils + +For every external command run (which is actually not recommended if possible) custom wrapper for `subprocess` is used. Additional functions `ahriman.core.auth.helpers` provide safe calls for `aiohttp_security` methods and are required to make this dependency optional. + +## Submodules + +Some packages provide different behaviour depending on configuration settings. In this cases inheritance is used and recommended way to deal with them is to call class method `load` from base classes. + +## Additional features + +Some features require optional dependencies to be installed: + +* Version control executables (e.g. `git`, `svn`) for VCS packages. +* `gnupg` application for package and repository sign feature. +* `rsync` application for rsync based repository sync. +* `boto3` python package for `S3` sync. +* `Jinja2` python package for HTML report generation (it is also used by web application). + +# Web application + +Web application requires the following python packages to be installed: + +* Core part requires `aiohttp` (application itself), `aiohttp_jinja2` and `Jinja2` (HTML generation from templates). +* In addition authorization feature requires `aiohttp_security`, `aiohttp_session` and `cryptography`. + +## Middlewares + +Service provides some custom middlewares, e.g. logging every exception (except for user ones) and user authorization. + +## Web views + +All web views are defined in separated package and derived from `ahriman.web.views.base.Base` class which provides typed interfaces for web application. + +REST API supports both form and JSON data, but the last one is recommended. + +Different APIs are separated into different packages: + +* `ahriman.web.views.service` provides views for application controls. +* `ahriman.web.views.status` package provides REST API for application reporting. +* `ahriman.web.views.user` package provides login and logout methods which can be called without authorization. + +## Templating + +Package provides base jinja templates which can be overridden by settings. Vanilla templates are actively using bootstrap library. + +## Requests and scopes + +Service provides optional authorization which can be turned on in settings. In order to control user access there are two levels of authorization - read-only (only GET-like requests) and write (anything). + +If this feature is configured any request except for whitelisted will be prohibited without authentication. In addition, configuration flag `auth.allow_read_only` can be used in order to allow seeing main page without authorization (this page is in default white list). + +For authorized users it uses encrypted session cookies to store tokens; encryption key is generated each time at the start of the application. + +## External calls + +Web application provides external calls to control main service. It spawns child process with specific arguments and waits for its termination. This feature must be used either with authorization or in safe (i.e. when status page is not available world-wide) environment. diff --git a/CONFIGURING.md b/docs/configuration.md similarity index 99% rename from CONFIGURING.md rename to docs/configuration.md index 45f311cf..4d113643 100644 --- a/CONFIGURING.md +++ b/docs/configuration.md @@ -23,7 +23,7 @@ libalpm and AUR related configuration. Base authorization settings. * `target` - specifies authorization provider, string, optional, default `disabled`. Allowed values are `disabled`, `configuration`. -* `allow_read_only` - allow to request read only pages without authorization, boolean, required. +* `allow_read_only` - allow requesting read only pages without authorization, boolean, required. * `allowed_paths` - URI paths (exact match) which can be accessed without authorization, space separated list of strings, optional. * `allowed_paths_groups` - URI paths prefixes which can be accessed without authorization, space separated list of strings, optional. * `salt` - password hash salt, string, required in case if authorization enabled (automatically generated by `create-user` subcommand). diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 00000000..b302d977 --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,60 @@ +# Setup instructions + +1. Install package as usual. +2. Change settings if required, see [configuration reference](configuration.md) for more details. +3. Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`): + + ```shell + echo 'PACKAGER="John Doe "' | sudo -u ahriman tee -a /var/lib/ahriman/.makepkg.conf + ``` + +4. Configure build tools (it is required for correct dependency management system): + + 1. Create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build` (you can choose any name for command, basically it should be `{name}-{arch}-build`). + 2. Create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,ahriman}.conf` (same as previous `pacman-{name}.conf`). + 3. Change configuration file, add your own repository, add multilib repository etc; + 4. Set `build_command` option to point to your command. + 5. Configure `/etc/sudoers.d/ahriman` to allow running command without a password. + + ```shell + ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build + cp /usr/share/devtools/pacman-{extra,ahriman}.conf + + echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf + echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman-ahriman.conf + + echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf + echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman-ahriman.conf + echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman-ahriman.conf + + echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini + echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini + + echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman + echo 'ahriman ALL=(ALL) NOPASSWD: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman + chmod 400 /etc/sudoers.d/ahriman + ``` + +5. Start and enable `ahriman@.timer` via `systemctl`: + + ```shell + systemctl enable --now ahriman@x86_64.timer + ``` + +6. Start and enable status page: + + ```shell + systemctl enable --now ahriman-web@x86_64 + ``` + +7. Add packages by using `ahriman add {package}` command: + + ```shell + sudo -u ahriman ahriman -a x86_64 add yay --now + ``` + +Note that initial service configuration can be done by running `ahriman setup` with specific arguments. + +## User creation + +`create-user` subcommand is recommended for new user creation. \ No newline at end of file diff --git a/setup.py b/setup.py index a1ec1fef..4104b16f 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,6 @@ setup( "aiohttp_session", "aiohttp_security", "cryptography", - "passlib", ], }, ) diff --git a/web.png b/web.png new file mode 100644 index 00000000..8c2a5201 Binary files /dev/null and b/web.png differ