arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 15:27:17 +00:00
Code Issues Projects Releases Wiki Activity

Add tests (#1) (#5)

Browse Source 
* add models tests (#1)

also replace single quote to double one to confort PEP docstring
+ move _check_output to class properties to make it available for
mocking

* alpm tests implementation

* try to replace os with pathlib

* update tests for pathlib

* fix includes glob and trim version from dependencies

* build_tools package tests

* repository component tests

* add sign tests

* complete status tests

* handle exceptions in actual_version calls

* complete core tests

* move configuration to root conftest

* application tests

* complete application tests

* change copyright to more generic one

* base web tests

* complete web tests

* complete testkit

also add argument parsers test
This commit is contained in:
Evgenii Alekseev 2021-03-28 15:30:51 +03:00 committed by GitHub
parent 69499b2d0a
commit 74a244f06c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
139 changed files with 4606 additions and 1124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Makefile
View File

@ -1,4 +1,4 @@
.PHONY: archive archive_directory archlinux check clean directory push version
.PHONY: archive archive_directory archlinux check clean directory push tests version
.DEFAULT_GOAL := archlinux
PROJECT := ahriman
@ -16,21 +16,21 @@ archive: archive_directory
archive_directory: $(TARGET_FILES)
rm -fr $(addprefix $(PROJECT)/, $(IGNORE_FILES))
find $(PROJECT) -type f -name '*.pyc' -delete
find $(PROJECT) -depth -type d -name '__pycache__' -execdir rm -rf {} +
find $(PROJECT) -depth -type d -name '*.egg-info' -execdir rm -rf {} +
find "$(PROJECT)" -type f -name "*.pyc" -delete
find "$(PROJECT)" -depth -type d -name "__pycache__" -execdir rm -rf {} +
find "$(PROJECT)" -depth -type d -name "*.egg-info" -execdir rm -rf {} +
archlinux: archive
sed -i "/sha512sums=('[0-9A-Fa-f]*/s/[^'][^)]*/sha512sums=('$$(sha512sum $(PROJECT)-$(VERSION)-src.tar.xz | awk '{print $$1}')'/" package/archlinux/PKGBUILD
sed -i "s/pkgver=[0-9.]*/pkgver=$(VERSION)/" package/archlinux/PKGBUILD
check:
cd src && mypy --implicit-reexport --strict -p $(PROJECT)
cd src && find $(PROJECT) -name '*.py' -execdir autopep8 --max-line-length 120 -aa -i {} +
cd src && pylint --rcfile=../.pylintrc $(PROJECT)
cd src && mypy --implicit-reexport --strict -p "$(PROJECT)"
find "src/$(PROJECT)" tests -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} +
cd src && pylint --rcfile=../.pylintrc "$(PROJECT)"
clean:
find . -type f -name '$(PROJECT)-*-src.tar.xz' -delete
find . -type f -name "$(PROJECT)-*-src.tar.xz" -delete
rm -rf "$(PROJECT)"
directory: clean
@ -43,8 +43,11 @@ push: archlinux
git tag "$(VERSION)"
git push --tags
tests:
python setup.py test
version:
ifndef VERSION
$(error VERSION is required, but not set)
endif
sed -i "/__version__ = '[0-9.]*/s/[^'][^)]*/__version__ = '$(VERSION)'/" src/ahriman/version.py
sed -i '/__version__ = "[0-9.]*/s/[^"][^)]*/__version__ = "$(VERSION)"/' src/ahriman/version.py

2
package/archlinux/PKGBUILD
View File

@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
'ahriman.sysusers'
'ahriman.tmpfiles')
sha512sums=('a1db44390ce1785da3d535e3cfd2242d8d56070228eb9b3c1d5629163b65941d60753c481c0fdc69e475e534a828ceea39568dc6711abeee092616dac08e31a9'
sha512sums=('ed1ef5ee9a2fb25ee1220acb4e7ac30eec0783375766f7ca8c812e1aa84e28d8426e382c1ec3d5357f1141ff683f9dd346970fe1f8bb0c7e29373ee55e478ef4'
'13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
'55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
backup=('etc/ahriman.ini'

2
package/etc/ahriman.ini
View File

@ -42,6 +42,4 @@ remote =
bucket =
[web]
host =
port =
templates = /usr/share/ahriman

6
package/etc/ahriman.ini.d/logging.ini
View File

@ -17,19 +17,19 @@ args = (sys.stderr,)
class = logging.handlers.RotatingFileHandler
level = DEBUG
formatter = generic_format
args = ('/var/log/ahriman/ahriman.log', 'a', 20971520, 20)
args = ("/var/log/ahriman/ahriman.log", "a", 20971520, 20)
[handler_build_file_handler]
class = logging.handlers.RotatingFileHandler
level = DEBUG
formatter = generic_format
args = ('/var/log/ahriman/build.log', 'a', 20971520, 20)
args = ("/var/log/ahriman/build.log", "a", 20971520, 20)
[handler_http_handler]
class = logging.handlers.RotatingFileHandler
level = DEBUG
formatter = generic_format
args = ('/var/log/ahriman/http.log', 'a', 20971520, 20)
args = ("/var/log/ahriman/http.log", "a", 20971520, 20)
[formatter_generic_format]
format = [%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s

5
setup.cfg Normal file
View File

@ -0,0 +1,5 @@
[aliases]
test = pytest
[tool:pytest]
addopts = --cov=ahriman --pspec

74
setup.py
View File

@ -4,69 +4,75 @@ from os import path
here = path.abspath(path.dirname(__file__))
metadata = dict()
with open(convert_path('src/ahriman/version.py')) as metadata_file:
with open(convert_path("src/ahriman/version.py")) as metadata_file:
exec(metadata_file.read(), metadata)
setup(
name='ahriman',
name="ahriman",
version=metadata['__version__'],
version=metadata["__version__"],
zip_safe=False,
description='ArcHlinux ReposItory MANager',
description="ArcHlinux ReposItory MANager",
author='arcanis',
author_email='',
url='https://github.com/arcan1s/ahriman',
author="arcanis",
author_email="",
url="https://github.com/arcan1s/ahriman",
license='GPL3',
license="GPL3",
packages=find_packages('src'),
package_dir={'': 'src'},
packages=find_packages("src"),
package_dir={"": "src"},
dependency_links=[
],
install_requires=[
'aur',
'pyalpm',
'srcinfo',
"aur",
"pyalpm",
"srcinfo",
],
setup_requires=[
'pytest-runner',
"pytest-runner",
],
tests_require=[
'pytest',
"pytest",
"pytest-aiohttp",
"pytest-cov",
"pytest-helpers-namespace",
"pytest-mock",
"pytest-pspec",
"pytest-resource-path",
],
include_package_data=True,
scripts=[
'package/bin/ahriman',
"package/bin/ahriman",
],
data_files=[
('/etc', [
'package/etc/ahriman.ini',
("/etc", [
"package/etc/ahriman.ini",
]),
('/etc/ahriman.ini.d', [
'package/etc/ahriman.ini.d/logging.ini',
("/etc/ahriman.ini.d", [
"package/etc/ahriman.ini.d/logging.ini",
]),
('lib/systemd/system', [
'package/lib/systemd/system/ahriman@.service',
'package/lib/systemd/system/ahriman@.timer',
'package/lib/systemd/system/ahriman-web@.service',
("lib/systemd/system", [
"package/lib/systemd/system/ahriman@.service",
"package/lib/systemd/system/ahriman@.timer",
"package/lib/systemd/system/ahriman-web@.service",
]),
('share/ahriman', [
'package/share/ahriman/build-status.jinja2',
'package/share/ahriman/repo-index.jinja2',
'package/share/ahriman/search.jinja2',
'package/share/ahriman/search-line.jinja2',
'package/share/ahriman/sorttable.jinja2',
'package/share/ahriman/style.jinja2',
("share/ahriman", [
"package/share/ahriman/build-status.jinja2",
"package/share/ahriman/repo-index.jinja2",
"package/share/ahriman/search.jinja2",
"package/share/ahriman/search-line.jinja2",
"package/share/ahriman/sorttable.jinja2",
"package/share/ahriman/style.jinja2",
]),
],
extras_require={
'html-templates': ['Jinja2'],
'test': ['coverage', 'pytest'],
'web': ['Jinja2', 'aiohttp', 'aiohttp_jinja2', 'requests'],
"html-templates": ["Jinja2"],
"test": ["pytest", "pytest-cov", "pytest-helpers-namespace", "pytest-mock", "pytest-pspec", "pytest-resource-path"],
"web": ["Jinja2", "aiohttp", "aiohttp_jinja2", "requests"],
},
)

2
src/ahriman/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

2
src/ahriman/application/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

110
src/ahriman/application/ahriman.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -24,82 +24,90 @@ import ahriman.application.handlers as handlers
import ahriman.version as version
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='ahriman', description='ArcHlinux ReposItory MANager')
# pylint: disable=too-many-statements
def _parser() -> argparse.ArgumentParser:
"""
command line parser generator
:return: command line parser for the application
"""
parser = argparse.ArgumentParser(prog="ahriman", description="ArcHlinux ReposItory MANager")
parser.add_argument(
'-a',
'--architecture',
help='target architectures (can be used multiple times)',
action='append')
parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini')
parser.add_argument('--force', help='force run, remove file lock', action='store_true')
parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
parser.add_argument('--no-log', help='redirect all log messages to stderr', action='store_true')
parser.add_argument('--no-report', help='force disable reporting to web service', action='store_true')
parser.add_argument('--unsafe', help='allow to run ahriman as non-ahriman user', action='store_true')
parser.add_argument('-v', '--version', action='version', version=version.__version__)
subparsers = parser.add_subparsers(title='command')
"-a",
"--architecture",
help="target architectures (can be used multiple times)",
action="append",
required=True)
parser.add_argument("-c", "--config", help="configuration path", default="/etc/ahriman.ini")
parser.add_argument("--force", help="force run, remove file lock", action="store_true")
parser.add_argument("--lock", help="lock file", default="/tmp/ahriman.lock")
parser.add_argument("--no-log", help="redirect all log messages to stderr", action="store_true")
parser.add_argument("--no-report", help="force disable reporting to web service", action="store_true")
parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user", action="store_true")
parser.add_argument("-v", "--version", action="version", version=version.__version__)
subparsers = parser.add_subparsers(title="command", help="command to run", dest="command", required=True)
add_parser = subparsers.add_parser('add', description='add package')
add_parser.add_argument('package', help='package base/name or archive path', nargs='+')
add_parser.add_argument('--without-dependencies', help='do not add dependencies', action='store_true')
add_parser = subparsers.add_parser("add", description="add package")
add_parser.add_argument("package", help="package base/name or archive path", nargs="+")
add_parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
add_parser.set_defaults(handler=handlers.Add)
check_parser = subparsers.add_parser('check', description='check for updates. Same as update --dry-run --no-manual')
check_parser.add_argument('package', help='filter check by package base', nargs='*')
check_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
check_parser = subparsers.add_parser("check", description="check for updates. Same as update --dry-run --no-manual")
check_parser.add_argument("package", help="filter check by package base", nargs="*")
check_parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
check_parser.set_defaults(handler=handlers.Update, no_aur=False, no_manual=True, dry_run=True)
clean_parser = subparsers.add_parser('clean', description='clear all local caches')
clean_parser.add_argument('--no-build', help='do not clear directory with package sources', action='store_true')
clean_parser.add_argument('--no-cache', help='do not clear directory with package caches', action='store_true')
clean_parser.add_argument('--no-chroot', help='do not clear build chroot', action='store_true')
clean_parser = subparsers.add_parser("clean", description="clear all local caches")
clean_parser.add_argument("--no-build", help="do not clear directory with package sources", action="store_true")
clean_parser.add_argument("--no-cache", help="do not clear directory with package caches", action="store_true")
clean_parser.add_argument("--no-chroot", help="do not clear build chroot", action="store_true")
clean_parser.add_argument(
'--no-manual',
help='do not clear directory with manually added packages',
action='store_true')
clean_parser.add_argument('--no-packages', help='do not clear directory with built packages', action='store_true')
"--no-manual",
help="do not clear directory with manually added packages",
action="store_true")
clean_parser.add_argument("--no-packages", help="do not clear directory with built packages", action="store_true")
clean_parser.set_defaults(handler=handlers.Clean)
config_parser = subparsers.add_parser('config', description='dump configuration for specified architecture')
config_parser = subparsers.add_parser("config", description="dump configuration for specified architecture")
config_parser.set_defaults(handler=handlers.Dump, lock=None, no_report=True, unsafe=True)
rebuild_parser = subparsers.add_parser('rebuild', description='rebuild whole repository')
rebuild_parser = subparsers.add_parser("rebuild", description="rebuild whole repository")
rebuild_parser.set_defaults(handler=handlers.Rebuild)
remove_parser = subparsers.add_parser('remove', description='remove package')
remove_parser.add_argument('package', help='package name or base', nargs='+')
remove_parser = subparsers.add_parser("remove", description="remove package")
remove_parser.add_argument("package", help="package name or base", nargs="+")
remove_parser.set_defaults(handler=handlers.Remove)
report_parser = subparsers.add_parser('report', description='generate report')
report_parser.add_argument('target', help='target to generate report', nargs='*')
report_parser = subparsers.add_parser("report", description="generate report")
report_parser.add_argument("target", help="target to generate report", nargs="*")
report_parser.set_defaults(handler=handlers.Report)
status_parser = subparsers.add_parser('status', description='request status of the package')
status_parser.add_argument('--ahriman', help='get service status itself', action='store_true')
status_parser.add_argument('package', help='filter status by package base', nargs='*')
status_parser = subparsers.add_parser("status", description="request status of the package")
status_parser.add_argument("--ahriman", help="get service status itself", action="store_true")
status_parser.add_argument("package", help="filter status by package base", nargs="*")
status_parser.set_defaults(handler=handlers.Status, lock=None, no_report=True, unsafe=True)
sync_parser = subparsers.add_parser('sync', description='sync packages to remote server')
sync_parser.add_argument('target', help='target to sync', nargs='*')
sync_parser = subparsers.add_parser("sync", description="sync packages to remote server")
sync_parser.add_argument("target", help="target to sync", nargs="*")
sync_parser.set_defaults(handler=handlers.Sync)
update_parser = subparsers.add_parser('update', description='run updates')
update_parser.add_argument('package', help='filter check by package base', nargs='*')
update_parser = subparsers.add_parser("update", description="run updates")
update_parser.add_argument("package", help="filter check by package base", nargs="*")
update_parser.add_argument(
'--dry-run', help='just perform check for updates, same as check command', action='store_true')
update_parser.add_argument('--no-aur', help='do not check for AUR updates. Implies --no-vcs', action='store_true')
update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true')
update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
"--dry-run", help="just perform check for updates, same as check command", action="store_true")
update_parser.add_argument("--no-aur", help="do not check for AUR updates. Implies --no-vcs", action="store_true")
update_parser.add_argument("--no-manual", help="do not include manual updates", action="store_true")
update_parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
update_parser.set_defaults(handler=handlers.Update)
web_parser = subparsers.add_parser('web', description='start web server')
web_parser = subparsers.add_parser("web", description="start web server")
web_parser.set_defaults(handler=handlers.Web, lock=None, no_report=True)
args = parser.parse_args()
if 'handler' not in args:
parser.print_help()
sys.exit(1)
return parser
if __name__ == "__main__":
arg_parser = _parser()
args = arg_parser.parse_args()
handler: handlers.Handler = args.handler
status = handler.execute(args)

91
src/ahriman/application/application.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -18,9 +18,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import os
import shutil
from pathlib import Path
from typing import Callable, Iterable, List, Optional, Set
from ahriman.core.build_tools.task import Task
@ -32,30 +32,30 @@ from ahriman.models.package import Package
class Application:
'''
"""
base application class
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar logger: application logger
:ivar repository: repository instance
'''
"""
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('root')
"""
self.logger = logging.getLogger("root")
self.config = config
self.architecture = architecture
self.repository = Repository(architecture, config)
def _known_packages(self) -> Set[str]:
'''
"""
load packages from repository and pacman repositories
:return: list of known packages
'''
"""
known_packages: Set[str] = set()
# local set
for package in self.repository.packages():
@ -64,15 +64,15 @@ class Application:
return known_packages
def _finalize(self) -> None:
'''
"""
generate report and sync to remote server
'''
"""
self.report()
self.sync()
def get_updates(self, filter_packages: List[str], no_aur: bool, no_manual: bool, no_vcs: bool,
log_fn: Callable[[str], None]) -> List[Package]:
'''
"""
get list of packages to run update process
:param filter_packages: do not check every package just specified in the list
:param no_aur: do not check for aur updates
@ -80,7 +80,7 @@ class Application:
:param no_vcs: do not check VCS packages
:param log_fn: logger function to log updates
:return: list of out-of-dated packages
'''
"""
updates = []
if not no_aur:
@ -89,60 +89,60 @@ class Application:
updates.extend(self.repository.updates_manual())
for package in updates:
log_fn(f'{package.base} = {package.version}')
log_fn(f"{package.base} = {package.version}")
return updates
def add(self, names: Iterable[str], without_dependencies: bool) -> None:
'''
"""
add packages for the next build
:param names: list of package bases to add
:param without_dependencies: if set, dependency check will be disabled
'''
"""
known_packages = self._known_packages()
def add_directory(path: str) -> None:
for package in filter(package_like, os.listdir(path)):
full_path = os.path.join(path, package)
add_manual(full_path)
def add_directory(path: Path) -> None:
for full_path in filter(package_like, path.iterdir()):
add_archive(full_path)
def add_manual(name: str) -> str:
package = Package.load(name, self.repository.pacman, self.config.get('alpm', 'aur_url'))
path = os.path.join(self.repository.paths.manual, package.base)
def add_manual(src: str) -> Path:
package = Package.load(src, self.repository.pacman, self.config.get("alpm", "aur_url"))
path = self.repository.paths.manual / package.base
Task.fetch(path, package.git_url)
return path
def add_archive(src: str) -> None:
dst = os.path.join(self.repository.paths.packages, os.path.basename(src))
def add_archive(src: Path) -> None:
dst = self.repository.paths.packages / src.name
shutil.move(src, dst)
def process_dependencies(path: str) -> None:
def process_dependencies(path: Path) -> None:
if without_dependencies:
return
dependencies = Package.dependencies(path)
self.add(dependencies.difference(known_packages), without_dependencies)
def process_single(name: str) -> None:
if os.path.isdir(name):
add_directory(name)
elif os.path.isfile(name):
add_archive(name)
def process_single(src: str) -> None:
maybe_path = Path(src)
if maybe_path.is_dir():
add_directory(maybe_path)
elif maybe_path.is_file():
add_archive(maybe_path)
else:
path = add_manual(name)
path = add_manual(src)
process_dependencies(path)
for name in names:
process_single(name)
def clean(self, no_build: bool, no_cache: bool, no_chroot: bool, no_manual: bool, no_packages: bool) -> None:
'''
"""
run all clean methods. Warning: some functions might not be available under non-root
:param no_build: do not clear directory with package sources
:param no_cache: do not clear directory with package caches
:param no_chroot: do not clear build chroot
:param no_manual: do not clear directory with manually added packages
:param no_packages: do not clear directory with built packages
'''
"""
if not no_build:
self.repository.clear_build()
if not no_cache:
@ -155,35 +155,35 @@ class Application:
self.repository.clear_packages()
def remove(self, names: Iterable[str]) -> None:
'''
"""
remove packages from repository
:param names: list of packages (either base or name) to remove
'''
"""
self.repository.process_remove(names)
self._finalize()
def report(self, target: Optional[Iterable[str]] = None) -> None:
'''
"""
generate report
:param target: list of targets to run (e.g. html)
'''
"""
targets = target or None
self.repository.process_report(targets)
def sync(self, target: Optional[Iterable[str]] = None) -> None:
'''
"""
sync to remote server
:param target: list of targets to run (e.g. s3)
'''
"""
targets = target or None
self.repository.process_sync(targets)
def update(self, updates: Iterable[Package]) -> None:
'''
"""
run package updates
:param updates: list of packages to update
'''
def process_update(paths: Iterable[str]) -> None:
"""
def process_update(paths: Iterable[Path]) -> None:
self.repository.process_update(paths)
self._finalize()
@ -192,9 +192,8 @@ class Application:
process_update(packages)
# process manual packages
tree = Tree()
tree.load(updates)
tree = Tree.load(updates)
for num, level in enumerate(tree.levels()):
self.logger.info(f'processing level #{num} {[package.base for package in level]}')
self.logger.info(f"processing level #{num} {[package.base for package in level]}")
packages = self.repository.process_build(level)
process_update(packages)

2
src/ahriman/application/handlers/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

10
src/ahriman/application/handlers/add.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,16 +27,16 @@ from ahriman.core.configuration import Configuration
class Add(Handler):
'''
"""
add packages handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Application(architecture, config).add(args.package, args.without_dependencies)

10
src/ahriman/application/handlers/clean.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,17 +27,17 @@ from ahriman.core.configuration import Configuration
class Clean(Handler):
'''
"""
clean caches handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Application(architecture, config).clean(args.no_build, args.no_cache, args.no_chroot,
args.no_manual, args.no_packages)

14
src/ahriman/application/handlers/dump.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -26,21 +26,21 @@ from ahriman.core.configuration import Configuration
class Dump(Handler):
'''
"""
dump config handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
config_dump = config.dump(architecture)
for section, values in sorted(config_dump.items()):
print(f'[{section}]')
print(f"[{section}]")
for key, value in sorted(values.items()):
print(f'{key} = {value}')
print(f"{key} = {value}")
print()

22
src/ahriman/application/handlers/handler.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -21,8 +21,8 @@ from __future__ import annotations
import argparse
import logging
from multiprocessing import Pool
from multiprocessing import Pool
from typing import Type
from ahriman.application.lock import Lock
@ -30,34 +30,34 @@ from ahriman.core.configuration import Configuration
class Handler:
'''
"""
base handler class for command callbacks
'''
"""
@classmethod
def _call(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> bool:
'''
"""
additional function to wrap all calls for multiprocessing library
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
:return: True on success, False otherwise
'''
"""
try:
with Lock(args, architecture, config):
cls.run(args, architecture, config)
return True
except Exception:
logging.getLogger('root').exception('process exception', exc_info=True)
logging.getLogger("root").exception("process exception")
return False
@classmethod
def execute(cls: Type[Handler], args: argparse.Namespace) -> int:
'''
"""
execute function for all aru
:param args: command line args
:return: 0 on success, 1 otherwise
'''
"""
configuration = Configuration.from_path(args.config, not args.no_log)
with Pool(len(args.architecture)) as pool:
result = pool.starmap(
@ -66,10 +66,10 @@ class Handler:
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
raise NotImplementedError

10
src/ahriman/application/handlers/rebuild.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,18 +27,18 @@ from ahriman.core.configuration import Configuration
class Rebuild(Handler):
'''
"""
make world handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
application = Application(architecture, config)
packages = application.repository.packages()
application.update(packages)

10
src/ahriman/application/handlers/remove.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,16 +27,16 @@ from ahriman.core.configuration import Configuration
class Remove(Handler):
'''
"""
remove packages handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Application(architecture, config).remove(args.package)

10
src/ahriman/application/handlers/report.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,16 +27,16 @@ from ahriman.core.configuration import Configuration
class Report(Handler):
'''
"""
generate report handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Application(architecture, config).report(args.target)

14
src/ahriman/application/handlers/status.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -29,18 +29,18 @@ from ahriman.models.package import Package
class Status(Handler):
'''
"""
package status handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
application = Application(architecture, config)
if args.ahriman:
ahriman = application.repository.reporter.get_self()
@ -54,5 +54,5 @@ class Status(Handler):
packages = application.repository.reporter.get(None)
for package, package_status in sorted(packages, key=lambda item: item[0].base):
print(package.pretty_print())
print(f'\t{package.version}')
print(f'\t{package_status.pretty_print()}')
print(f"\t{package.version}")
print(f"\t{package_status.pretty_print()}")

10
src/ahriman/application/handlers/sync.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,16 +27,16 @@ from ahriman.core.configuration import Configuration
class Sync(Handler):
'''
"""
remove sync handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Application(architecture, config).sync(args.target)

10
src/ahriman/application/handlers/update.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,18 +27,18 @@ from ahriman.core.configuration import Configuration
class Update(Handler):
'''
"""
package update handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
# typing workaround
def log_fn(line: str) -> None:
return print(line) if args.dry_run else application.logger.info(line)

12
src/ahriman/application/handlers/web.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -26,18 +26,18 @@ from ahriman.core.configuration import Configuration
class Web(Handler):
'''
"""
web server handler
'''
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
"""
from ahriman.web.web import run_server, setup_service
application = setup_service(architecture, config)
run_server(application, architecture)
run_server(application)

59
src/ahriman/application/lock.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -22,41 +22,42 @@ from __future__ import annotations
import argparse
import os
from pathlib import Path
from types import TracebackType
from typing import Literal, Optional, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import DuplicateRun, UnsafeRun
from ahriman.core.watcher.client import Client
from ahriman.core.status.client import Client
from ahriman.models.build_status import BuildStatusEnum
class Lock:
'''
"""
wrapper for application lock file
:ivar force: remove lock file on start if any
:ivar path: path to lock file if any
:ivar reporter: build status reporter instance
:ivar root: repository root (i.e. ahriman home)
:ivar unsafe: skip user check
'''
"""
def __init__(self, args: argparse.Namespace, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param args: command line args
:param architecture: repository architecture
:param config: configuration instance
'''
self.path = f'{args.lock}_{architecture}' if args.lock is not None else None
"""
self.path = Path(f"{args.lock}_{architecture}") if args.lock is not None else None
self.force = args.force
self.unsafe = args.unsafe
self.root = config.get('repository', 'root')
self.root = Path(config.get("repository", "root"))
self.reporter = Client() if args.no_report else Client.load(architecture, config)
def __enter__(self) -> Lock:
'''
"""
default workflow is the following:
check user UID
@ -64,62 +65,52 @@ class Lock:
check if there is lock file
create lock file
report to web if enabled
'''
"""
self.check_user()
if self.force:
self.remove()
self.check()
self.create()
self.reporter.update_self(BuildStatusEnum.Building)
return self
def __exit__(self, exc_type: Optional[Type[Exception]], exc_val: Optional[Exception],
exc_tb: TracebackType) -> Literal[False]:
'''
"""
remove lock file when done
:param exc_type: exception type name if any
:param exc_val: exception raised if any
:param exc_tb: exception traceback if any
:return: always False (do not suppress any exception)
'''
"""
self.remove()
status = BuildStatusEnum.Success if exc_val is None else BuildStatusEnum.Failed
self.reporter.update_self(status)
return False
def check(self) -> None:
'''
check if lock file exists, raise exception if it does
'''
if self.path is None:
return
if os.path.exists(self.path):
raise DuplicateRun()
def check_user(self) -> None:
'''
"""
check if current user is actually owner of ahriman root
'''
"""
if self.unsafe:
return
current_uid = os.getuid()
root_uid = os.stat(self.root).st_uid
root_uid = self.root.stat().st_uid
if current_uid != root_uid:
raise UnsafeRun(current_uid, root_uid)
def create(self) -> None:
'''
"""
create lock file
'''
"""
if self.path is None:
return
open(self.path, 'w').close()
try:
self.path.touch(exist_ok=self.force)
except FileExistsError:
raise DuplicateRun()
def remove(self) -> None:
'''
"""
remove lock file
'''
"""
if self.path is None:
return
if os.path.exists(self.path):
os.remove(self.path)
self.path.unlink(missing_ok=True)

2
src/ahriman/core/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

2
src/ahriman/core/alpm/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

22
src/ahriman/core/alpm/pacman.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -24,27 +24,27 @@ from ahriman.core.configuration import Configuration
class Pacman:
'''
"""
alpm wrapper
:ivar handle: pyalpm root `Handle`
'''
"""
def __init__(self, config: Configuration) -> None:
'''
"""
default constructor
:param config: configuration instance
'''
root = config.get('alpm', 'root')
pacman_root = config.get('alpm', 'database')
self.handle = Handle(root, pacman_root)
for repository in config.getlist('alpm', 'repositories'):
"""
root = config.get("alpm", "root")
pacman_root = config.getpath("alpm", "database")
self.handle = Handle(root, str(pacman_root))
for repository in config.getlist("alpm", "repositories"):
self.handle.register_syncdb(repository, 0) # 0 is pgp_level
def all_packages(self) -> List[str]:
'''
"""
get list of packages known for alpm
:return: list of package names
'''
"""
result: Set[str] = set()
for database in self.handle.get_syncdbs():
result.update({package.name for package in database.pkgcache})

53
src/ahriman/core/alpm/repo.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -18,8 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import os
from pathlib import Path
from typing import List
from ahriman.core.exceptions import BuildFailed
@ -28,56 +28,59 @@ from ahriman.models.repository_paths import RepositoryPaths
class Repo:
'''
"""
repo-add and repo-remove wrapper
:ivar logger: class logger
:ivar name: repository name
:ivar paths: repository paths instance
:ivar sign_args: additional args which have to be used to sign repository archive
'''
"""
_check_output = check_output
def __init__(self, name: str, paths: RepositoryPaths, sign_args: List[str]) -> None:
'''
"""
default constructor
:param name: repository name
:param paths: repository paths instance
:param sign_args: additional args which have to be used to sign repository archive
'''
self.logger = logging.getLogger('build_details')
"""
self.logger = logging.getLogger("build_details")
self.name = name
self.paths = paths
self.sign_args = sign_args
@property
def repo_path(self) -> str:
'''
def repo_path(self) -> Path:
"""
:return: path to repository database
'''
return os.path.join(self.paths.repository, f'{self.name}.db.tar.gz')
"""
return self.paths.repository / f"{self.name}.db.tar.gz"
def add(self, path: str) -> None:
'''
def add(self, path: Path) -> None:
"""
add new package to repository
:param path: path to archive to add
'''
check_output(
'repo-add', *self.sign_args, '-R', self.repo_path, path,
exception=BuildFailed(path),
"""
Repo._check_output(
"repo-add", *self.sign_args, "-R", str(self.repo_path), str(path),
exception=BuildFailed(path.name),
cwd=self.paths.repository,
logger=self.logger)
def remove(self, package: str) -> None:
'''
def remove(self, package: str, filename: Path) -> None:
"""
remove package from repository
:param package: package name to remove
'''
:param filename: package filename to remove
"""
# remove package and signature (if any) from filesystem
for fn in filter(lambda f: f.startswith(package), os.listdir(self.paths.repository)):
full_path = os.path.join(self.paths.repository, fn)
os.remove(full_path)
for full_path in self.paths.repository.glob(f"{filename}*"):
full_path.unlink()
# remove package from registry
check_output(
'repo-remove', *self.sign_args, self.repo_path, package,
Repo._check_output(
"repo-remove", *self.sign_args, str(self.repo_path), package,
exception=BuildFailed(package),
cwd=self.paths.repository,
logger=self.logger)

2
src/ahriman/core/build_tools/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

94
src/ahriman/core/build_tools/task.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,10 +17,10 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import logging
import shutil
from pathlib import Path
from typing import List, Optional
from ahriman.core.configuration import Configuration
@ -31,94 +31,98 @@ from ahriman.models.repository_paths import RepositoryPaths
class Task:
'''
"""
base package build task
:ivar build_logger: logger for build process
:ivar logger: class logger
:ivar package: package definitions
:ivar paths: repository paths instance
'''
"""
_check_output = check_output
def __init__(self, package: Package, architecture: str, config: Configuration, paths: RepositoryPaths) -> None:
'''
"""
default constructor
:param package: package definitions
:param architecture: repository architecture
:param config: configuration instance
:param paths: repository paths instance
'''
self.logger = logging.getLogger('builder')
self.build_logger = logging.getLogger('build_details')
"""
self.logger = logging.getLogger("builder")
self.build_logger = logging.getLogger("build_details")
self.package = package
self.paths = paths
section = config.get_section_name('build', architecture)
self.archbuild_flags = config.getlist(section, 'archbuild_flags')
self.build_command = config.get(section, 'build_command')
self.makepkg_flags = config.getlist(section, 'makepkg_flags')
self.makechrootpkg_flags = config.getlist(section, 'makechrootpkg_flags')
section = config.get_section_name("build", architecture)
self.archbuild_flags = config.getlist(section, "archbuild_flags")
self.build_command = config.get(section, "build_command")
self.makepkg_flags = config.getlist(section, "makepkg_flags")
self.makechrootpkg_flags = config.getlist(section, "makechrootpkg_flags")
@property
def cache_path(self) -> str:
'''
def cache_path(self) -> Path:
"""
:return: path to cached packages
'''
return os.path.join(self.paths.cache, self.package.base)
"""
return self.paths.cache / self.package.base
@property
def git_path(self) -> str:
'''
def git_path(self) -> Path:
"""
:return: path to clone package from git
'''
return os.path.join(self.paths.sources, self.package.base)
"""
return self.paths.sources / self.package.base
@staticmethod
def fetch(local: str, remote: str, branch: str = 'master') -> None:
'''
def fetch(local: Path, remote: str, branch: str = "master") -> None:
"""
either clone repository or update it to origin/`branch`
:param local: local path to fetch
:param remote: remote target (from where to fetch)
:param branch: branch name to checkout, master by default
'''
logger = logging.getLogger('build_details')
"""
logger = logging.getLogger("build_details")
# local directory exists and there is .git directory
if os.path.isdir(os.path.join(local, '.git')):
check_output('git', 'fetch', 'origin', branch, exception=None, cwd=local, logger=logger)
if (local / ".git").is_dir():
Task._check_output("git", "fetch", "origin", branch, exception=None, cwd=local, logger=logger)
else:
check_output('git', 'clone', remote, local, exception=None, logger=logger)
Task._check_output("git", "clone", remote, str(local), exception=None, logger=logger)
# and now force reset to our branch
check_output('git', 'reset', '--hard', f'origin/{branch}', exception=None, cwd=local, logger=logger)
Task._check_output("git", "checkout", "--force", branch, exception=None, cwd=local, logger=logger)
Task._check_output("git", "reset", "--hard", f"origin/{branch}", exception=None, cwd=local, logger=logger)
def build(self) -> List[str]:
'''
def build(self) -> List[Path]:
"""
run package build
:return: paths of produced packages
'''
cmd = [self.build_command, '-r', self.paths.chroot]
"""
cmd = [self.build_command, "-r", str(self.paths.chroot)]
cmd.extend(self.archbuild_flags)
cmd.extend(['--'] + self.makechrootpkg_flags)
cmd.extend(['--'] + self.makepkg_flags)
self.logger.info(f'using {cmd} for {self.package.base}')
cmd.extend(["--"] + self.makechrootpkg_flags)
cmd.extend(["--"] + self.makepkg_flags)
self.logger.info(f"using {cmd} for {self.package.base}")
check_output(
Task._check_output(
*cmd,
exception=BuildFailed(self.package.base),
cwd=self.git_path,
logger=self.build_logger)
# well it is not actually correct, but we can deal with it
return check_output('makepkg', '--packagelist',
exception=BuildFailed(self.package.base),
cwd=self.git_path,
logger=self.build_logger).splitlines()
packages = Task._check_output("makepkg", "--packagelist",
exception=BuildFailed(self.package.base),
cwd=self.git_path,
logger=self.build_logger).splitlines()
return [Path(package) for package in packages]
def init(self, path: Optional[str] = None) -> None:
'''
def init(self, path: Optional[Path] = None) -> None:
"""
fetch package from git
:param path: optional local path to fetch. If not set default path will be used
'''
"""
git_path = path or self.git_path
if os.path.isdir(self.cache_path):
if self.cache_path.is_dir():
# no need to clone whole repository, just copy from cache first
shutil.copytree(self.cache_path, git_path)
return Task.fetch(git_path, self.package.git_url)

85
src/ahriman/core/configuration.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -21,61 +21,61 @@ from __future__ import annotations
import configparser
import logging
import os
from logging.config import fileConfig
from pathlib import Path
from typing import Dict, List, Optional, Type
class Configuration(configparser.RawConfigParser):
'''
"""
extension for built-in configuration parser
:ivar path: path to root configuration file
:cvar ARCHITECTURE_SPECIFIC_SECTIONS: known sections which can be architecture specific (required by dump)
:cvar DEFAULT_LOG_FORMAT: default log format (in case of fallback)
:cvar DEFAULT_LOG_LEVEL: default log level (in case of fallback)
:cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump)
'''
"""
DEFAULT_LOG_FORMAT = '[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s'
DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s"
DEFAULT_LOG_LEVEL = logging.DEBUG
STATIC_SECTIONS = ['alpm', 'report', 'repository', 'settings', 'upload']
ARCHITECTURE_SPECIFIC_SECTIONS = ['build', 'html', 'rsync', 's3', 'sign', 'web']
STATIC_SECTIONS = ["alpm", "report", "repository", "settings", "upload"]
ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "html", "rsync", "s3", "sign", "web"]
def __init__(self) -> None:
'''
"""
default constructor. In the most cases must not be called directly
'''
"""
configparser.RawConfigParser.__init__(self, allow_no_value=True)
self.path: Optional[str] = None
self.path: Optional[Path] = None
@property
def include(self) -> str:
'''
def include(self) -> Path:
"""
:return: path to directory with configuration includes
'''
return self.get('settings', 'include')
"""
return self.getpath("settings", "include")
@classmethod
def from_path(cls: Type[Configuration], path: str, logfile: bool) -> Configuration:
'''
def from_path(cls: Type[Configuration], path: Path, logfile: bool) -> Configuration:
"""
constructor with full object initialization
:param path: path to root configuration file
:param logfile: use log file to output messages
:return: configuration instance
'''
"""
config = cls()
config.load(path)
config.load_logging(logfile)
return config
def dump(self, architecture: str) -> Dict[str, Dict[str, str]]:
'''
"""
dump configuration to dictionary
:param architecture: repository architecture
:return: configuration dump for specific architecture
'''
"""
result: Dict[str, Dict[str, str]] = {}
for section in Configuration.STATIC_SECTIONS:
if not self.has_section(section):
@ -90,57 +90,70 @@ class Configuration(configparser.RawConfigParser):
return result
def getlist(self, section: str, key: str) -> List[str]:
'''
"""
get space separated string list option
:param section: section name
:param key: key name
:return: list of string if option is set, empty list otherwise
'''
"""
raw = self.get(section, key, fallback=None)
if not raw: # empty string or none
return []
return raw.split()
def getpath(self, section: str, key: str) -> Path:
"""
helper to generate absolute configuration path for relative settings value
:param section: section name
:param key: key name
:return: absolute path according to current path configuration
"""
value = Path(self.get(section, key))
if self.path is None or value.is_absolute():
return value
return self.path.parent / value
def get_section_name(self, prefix: str, suffix: str) -> str:
'''
"""
check if there is `prefix`_`suffix` section and return it on success. Return `prefix` otherwise
:param prefix: section name prefix
:param suffix: section name suffix (e.g. architecture name)
:return: found section name
'''
probe = f'{prefix}_{suffix}'
"""
probe = f"{prefix}_{suffix}"
return probe if self.has_section(probe) else prefix
def load(self, path: str) -> None:
'''
def load(self, path: Path) -> None:
"""
fully load configuration
:param path: path to root configuration file
'''
"""
self.path = path
self.read(self.path)
self.load_includes()
def load_includes(self) -> None:
'''
"""
load configuration includes
'''
"""
try:
for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))):
self.read(os.path.join(self.include, conf))
for path in sorted(self.include.glob("*.ini")):
self.read(path)
except (FileNotFoundError, configparser.NoOptionError):
pass
def load_logging(self, logfile: bool) -> None:
'''
"""
setup logging settings from configuration
:param logfile: use log file to output messages
'''
"""
def file_logger() -> None:
try:
fileConfig(self.get('settings', 'logging'))
except PermissionError:
config_path = self.getpath("settings", "logging")
fileConfig(config_path)
except (FileNotFoundError, PermissionError):
console_logger()
logging.error('could not create logfile, fallback to stderr', exc_info=True)
logging.exception("could not create logfile, fallback to stderr")
def console_logger() -> None:
logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT,

93
src/ahriman/core/exceptions.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -21,103 +21,112 @@ from typing import Any
class BuildFailed(Exception):
'''
"""
base exception for failed builds
'''
"""
def __init__(self, package: str) -> None:
'''
"""
default constructor
:param package: package base raised exception
'''
Exception.__init__(self, f'Package {package} build failed, check logs for details')
"""
Exception.__init__(self, f"Package {package} build failed, check logs for details")
class DuplicateRun(Exception):
'''
"""
exception which will be raised if there is another application instance
'''
"""
def __init__(self) -> None:
'''
"""
default constructor
'''
Exception.__init__(self, 'Another application instance is run')
"""
Exception.__init__(self, "Another application instance is run")
class InitializeException(Exception):
'''
"""
base service initialization exception
'''
"""
def __init__(self) -> None:
'''
"""
default constructor
'''
Exception.__init__(self, 'Could not load service')
"""
Exception.__init__(self, "Could not load service")
class InvalidOption(Exception):
'''
"""
exception which will be raised on configuration errors
'''
"""
def __init__(self, value: Any) -> None:
'''
"""
default constructor
:param value: option value
'''
Exception.__init__(self, f'Invalid or unknown option value `{value}`')
"""
Exception.__init__(self, f"Invalid or unknown option value `{value}`")
class InvalidPackageInfo(Exception):
'''
"""
exception which will be raised on package load errors
'''
"""
def __init__(self, details: Any) -> None:
'''
"""
default constructor
:param details: error details
'''
Exception.__init__(self, f'There are errors during reading package information: `{details}`')
"""
Exception.__init__(self, f"There are errors during reading package information: `{details}`")
class ReportFailed(Exception):
'''
"""
report generation exception
'''
"""
def __init__(self) -> None:
'''
"""
default constructor
'''
Exception.__init__(self, 'Report failed')
"""
Exception.__init__(self, "Report failed")
class SyncFailed(Exception):
'''
"""
remote synchronization exception
'''
"""
def __init__(self) -> None:
'''
"""
default constructor
'''
Exception.__init__(self, 'Sync failed')
"""
Exception.__init__(self, "Sync failed")
class UnknownPackage(Exception):
"""
exception for status watcher which will be thrown on unknown package
"""
def __init__(self, base: str) -> None:
Exception.__init__(self, f"Package base {base} is unknown")
class UnsafeRun(Exception):
'''
"""
exception which will be raised in case if user is not owner of repository
'''
"""
def __init__(self, current_uid: int, root_uid: int) -> None:
'''
"""
default constructor
'''
"""
Exception.__init__(
self,
f'''Current UID {current_uid} differs from root owner {root_uid}.
f"""Current UID {current_uid} differs from root owner {root_uid}.
Note that for the most actions it is unsafe to run application as different user.
If you are 100% sure that it must be there try --unsafe option''')
If you are 100% sure that it must be there try --unsafe option""")

2
src/ahriman/core/report/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

55
src/ahriman/core/report/html.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -18,7 +18,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import jinja2
import os
from typing import Callable, Dict, Iterable
@ -30,7 +29,7 @@ from ahriman.models.sign_settings import SignSettings
class HTML(Report):
'''
"""
html report generator
It uses jinja2 templates for report generation, the following variables are allowed:
@ -50,50 +49,49 @@ class HTML(Report):
:ivar report_path: output path to html report
:ivar sign_targets: targets to sign enabled in configuration
:ivar tempate_path: path to directory with jinja templates
'''
"""
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Report.__init__(self, architecture, config)
section = config.get_section_name('html', architecture)
self.report_path = config.get(section, 'path')
self.link_path = config.get(section, 'link_path')
self.template_path = config.get(section, 'template_path')
section = config.get_section_name("html", architecture)
self.report_path = config.getpath(section, "path")
self.link_path = config.get(section, "link_path")
self.template_path = config.getpath(section, "template_path")
# base template vars
self.homepage = config.get(section, 'homepage', fallback=None)
self.name = config.get('repository', 'name')
self.homepage = config.get(section, "homepage", fallback=None)
self.name = config.get("repository", "name")
sign_section = config.get_section_name('sign', architecture)
self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist(sign_section, 'target')]
self.pgp_key = config.get(sign_section, 'key') if self.sign_targets else None
sign_section = config.get_section_name("sign", architecture)
self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist(sign_section, "target")]
self.pgp_key = config.get(sign_section, "key") if self.sign_targets else None
def generate(self, packages: Iterable[Package]) -> None:
'''
"""
generate report for the specified packages
:param packages: list of packages to generate report
'''
"""
# idea comes from https://stackoverflow.com/a/38642558
templates_dir, template_name = os.path.split(self.template_path)
loader = jinja2.FileSystemLoader(searchpath=templates_dir)
loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent)
environment = jinja2.Environment(loader=loader)
template = environment.get_template(template_name)
template = environment.get_template(self.template_path.name)
content = [
{
'archive_size': pretty_size(properties.archive_size),
'build_date': pretty_datetime(properties.build_date),
'filename': properties.filename,
'installed_size': pretty_size(properties.installed_size),
'name': package,
'version': base.version
"archive_size": pretty_size(properties.archive_size),
"build_date": pretty_datetime(properties.build_date),
"filename": properties.filename,
"installed_size": pretty_size(properties.installed_size),
"name": package,
"version": base.version
} for base in packages for package, properties in base.packages.items()
]
comparator: Callable[[Dict[str, str]], str] = lambda item: item['filename']
comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
html = template.render(
homepage=self.homepage,
@ -104,5 +102,4 @@ class HTML(Report):
pgp_key=self.pgp_key,
repository=self.name)
with open(self.report_path, 'w') as out:
out.write(html)
self.report_path.write_text(html)

22
src/ahriman/core/report/report.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -28,32 +28,32 @@ from ahriman.models.report_settings import ReportSettings
class Report:
'''
"""
base report generator
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar logger: class logger
'''
"""
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('builder')
"""
self.logger = logging.getLogger("builder")
self.architecture = architecture
self.config = config
@staticmethod
def run(architecture: str, config: Configuration, target: str, packages: Iterable[Package]) -> None:
'''
"""
run report generation
:param architecture: repository architecture
:param config: configuration instance
:param target: target to generate report (e.g. html)
:param packages: list of packages to generate report
'''
"""
provider = ReportSettings.from_option(target)
if provider == ReportSettings.HTML:
from ahriman.core.report.html import HTML
@ -64,11 +64,11 @@ class Report:
try:
report.generate(packages)
except Exception:
report.logger.exception('report generation failed', exc_info=True)
report.logger.exception(f"report generation failed for target {provider.name}")
raise ReportFailed()
def generate(self, packages: Iterable[Package]) -> None:
'''
"""
generate report for the specified packages
:param packages: list of packages to generate report
'''
"""

2
src/ahriman/core/repository/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

62
src/ahriman/core/repository/cleaner.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,62 +17,62 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import shutil
from pathlib import Path
from typing import List
from ahriman.core.repository.properties import Properties
class Cleaner(Properties):
'''
"""
trait to clean common repository objects
'''
"""
def packages_built(self) -> List[str]:
'''
def packages_built(self) -> List[Path]:
"""
get list of files in built packages directory
:return: list of filenames from the directory
'''
"""
raise NotImplementedError
def clear_build(self) -> None:
'''
"""
clear sources directory
'''
self.logger.info('clear package sources directory')
for package in os.listdir(self.paths.sources):
shutil.rmtree(os.path.join(self.paths.sources, package))
"""
self.logger.info("clear package sources directory")
for package in self.paths.sources.iterdir():
shutil.rmtree(package)
def clear_cache(self) -> None:
'''
"""
clear cache directory
'''
self.logger.info('clear packages sources cache directory')
for package in os.listdir(self.paths.cache):
shutil.rmtree(os.path.join(self.paths.cache, package))
"""
self.logger.info("clear packages sources cache directory")
for package in self.paths.cache.iterdir():
shutil.rmtree(package)
def clear_chroot(self) -> None:
'''
"""
clear cache directory. Warning: this method is architecture independent and will clear every chroot
'''
self.logger.info('clear build chroot directory')
for chroot in os.listdir(self.paths.chroot):
shutil.rmtree(os.path.join(self.paths.chroot, chroot))
"""
self.logger.info("clear build chroot directory")
for chroot in self.paths.chroot.iterdir():
shutil.rmtree(chroot)
def clear_manual(self) -> None:
'''
"""
clear directory with manual package updates
'''
self.logger.info('clear manual packages')
for package in os.listdir(self.paths.manual):
shutil.rmtree(os.path.join(self.paths.manual, package))
"""
self.logger.info("clear manual packages")
for package in self.paths.manual.iterdir():
shutil.rmtree(package)
def clear_packages(self) -> None:
'''
"""
clear directory with built packages (NOT repository itself)
'''
self.logger.info('clear built packages directory')
"""
self.logger.info("clear built packages directory")
for package in self.packages_built():
os.remove(package)
package.unlink()

100
src/ahriman/core/repository/executor.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,9 +17,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import shutil
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from ahriman.core.build_tools.task import Task
@ -30,113 +30,123 @@ from ahriman.models.package import Package
class Executor(Cleaner):
'''
"""
trait for common repository update processes
'''
"""
def packages(self) -> List[Package]:
'''
"""
generate list of repository packages
:return: list of packages properties
'''
"""
raise NotImplementedError
def process_build(self, updates: Iterable[Package]) -> List[str]:
'''
def process_build(self, updates: Iterable[Package]) -> List[Path]:
"""
build packages
:param updates: list of packages properties to build
:return: `packages_built`
'''
"""
def build_single(package: Package) -> None:
self.reporter.set_building(package.base)
task = Task(package, self.architecture, self.config, self.paths)
task.init()
built = task.build()
for src in built:
dst = os.path.join(self.paths.packages, os.path.basename(src))
dst = self.paths.packages / src.name
shutil.move(src, dst)
for package in updates:
for single in updates:
try:
build_single(package)
build_single(single)
except Exception:
self.reporter.set_failed(package.base)
self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
continue
self.reporter.set_failed(single.base)
self.logger.exception(f"{single.base} ({self.architecture}) build exception")
self.clear_build()
return self.packages_built()
def process_remove(self, packages: Iterable[str]) -> str:
'''
def process_remove(self, packages: Iterable[str]) -> Path:
"""
remove packages from list
:param packages: list of package names or bases to rmeove
:param packages: list of package names or bases to remove
:return: path to repository database
'''
def remove_single(package: str) -> None:
"""
def remove_single(package: str, fn: Path) -> None:
try:
self.repo.remove(package)
self.repo.remove(package, fn)
except Exception:
self.logger.exception(f'could not remove {package}', exc_info=True)
self.logger.exception(f"could not remove {package}")
requested = set(packages)
for local in self.packages():
if local.base in packages:
to_remove = set(local.packages.keys())
if local.base in packages or all(package in requested for package in local.packages):
to_remove = {
package: Path(properties.filename)
for package, properties in local.packages.items()
if properties.filename is not None
}
self.reporter.remove(local.base) # we only update status page in case of base removal
elif requested.intersection(local.packages.keys()):
to_remove = requested.intersection(local.packages.keys())
to_remove = {
package: Path(properties.filename)
for package, properties in local.packages.items()
if package in requested and properties.filename is not None
}
else:
to_remove = set()
for package in to_remove:
remove_single(package)
to_remove = dict()
for package, filename in to_remove.items():
remove_single(package, filename)
return self.repo.repo_path
def process_report(self, targets: Optional[Iterable[str]]) -> None:
'''
"""
generate reports
:param targets: list of targets to generate reports. Configuration option will be used if it is not set
'''
"""
if targets is None:
targets = self.config.getlist('report', 'target')
targets = self.config.getlist("report", "target")
for target in targets:
Report.run(self.architecture, self.config, target, self.packages())
def process_sync(self, targets: Optional[Iterable[str]]) -> None:
'''
"""
process synchronization to remote servers
:param targets: list of targets to sync. Configuration option will be used if it is not set
'''
"""
if targets is None:
targets = self.config.getlist('upload', 'target')
targets = self.config.getlist("upload", "target")
for target in targets:
Uploader.run(self.architecture, self.config, target, self.paths.repository)
def process_update(self, packages: Iterable[str]) -> str:
'''
def process_update(self, packages: Iterable[Path]) -> Path:
"""
sign packages, add them to repository and update repository database
:param packages: list of filenames to run
:return: path to repository database
'''
"""
def update_single(fn: Optional[str], base: str) -> None:
if fn is None:
self.logger.warning(f'received empty package name for base {base}')
self.logger.warning(f"received empty package name for base {base}")
return # suppress type checking, it never can be none actually
# in theory it might be NOT packages directory, but we suppose it is
full_path = os.path.join(self.paths.packages, fn)
full_path = self.paths.packages / fn
files = self.sign.sign_package(full_path, base)
for src in files:
dst = os.path.join(self.paths.repository, os.path.basename(src))
dst = self.paths.repository / src.name
shutil.move(src, dst)
package_path = os.path.join(self.paths.repository, fn)
package_path = self.paths.repository / fn
self.repo.add(package_path)
# we are iterating over bases, not single packages
updates: Dict[str, Package] = {}
for fn in packages:
local = Package.load(fn, self.pacman, self.aur_url)
updates.setdefault(local.base, local).packages.update(local.packages)
for filename in packages:
try:
local = Package.load(filename, self.pacman, self.aur_url)
updates.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception(f"could not load package from {filename}")
for local in updates.values():
try:
@ -145,7 +155,7 @@ class Executor(Cleaner):
self.reporter.set_success(local)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception(f'could not process {local.base}', exc_info=True)
self.logger.exception(f"could not process {local.base}")
self.clear_packages()
return self.repo.repo_path

16
src/ahriman/core/repository/properties.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -23,12 +23,12 @@ from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.repo import Repo
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.watcher.client import Client
from ahriman.core.status.client import Client
from ahriman.models.repository_paths import RepositoryPaths
class Properties:
'''
"""
repository internal objects holder
:ivar architecture: repository architecture
:ivar aur_url: base AUR url
@ -40,17 +40,17 @@ class Properties:
:ivar repo: repo commands wrapper instance
:ivar reporter: build status reporter instance
:ivar sign: GPG wrapper instance
'''
"""
def __init__(self, architecture: str, config: Configuration) -> None:
self.logger = logging.getLogger('builder')
self.logger = logging.getLogger("builder")
self.architecture = architecture
self.config = config
self.aur_url = config.get('alpm', 'aur_url')
self.name = config.get('repository', 'name')
self.aur_url = config.get("alpm", "aur_url")
self.name = config.get("repository", "name")
self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
self.paths = RepositoryPaths(config.getpath("repository", "root"), architecture)
self.paths.create_tree()
self.pacman = Pacman(config)

31
src/ahriman/core/repository/repository.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,8 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from pathlib import Path
from typing import Dict, List
from ahriman.core.repository.executor import Executor
@ -28,34 +27,30 @@ from ahriman.models.package import Package
class Repository(Executor, UpdateHandler):
'''
"""
base repository control class
'''
"""
def packages(self) -> List[Package]:
'''
"""
generate list of repository packages
:return: list of packages properties
'''
"""
result: Dict[str, Package] = {}
for fn in os.listdir(self.paths.repository):
if not package_like(fn):
for full_path in self.paths.repository.iterdir():
if not package_like(full_path):
continue
full_path = os.path.join(self.paths.repository, fn)
try:
local = Package.load(full_path, self.pacman, self.aur_url)
result.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
self.logger.exception(f"could not load package from {full_path}")
continue
return list(result.values())
def packages_built(self) -> List[str]:
'''
def packages_built(self) -> List[Path]:
"""
get list of files in built packages directory
:return: list of filenames from the directory
'''
return [
os.path.join(self.paths.packages, fn)
for fn in os.listdir(self.paths.packages)
]
"""
return list(self.paths.packages.iterdir())

32
src/ahriman/core/repository/update_handler.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,8 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from typing import Iterable, List
from ahriman.core.repository.cleaner import Cleaner
@ -26,28 +24,28 @@ from ahriman.models.package import Package
class UpdateHandler(Cleaner):
'''
"""
trait to get package update list
'''
"""
def packages(self) -> List[Package]:
'''
"""
generate list of repository packages
:return: list of packages properties
'''
"""
raise NotImplementedError
def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
'''
"""
check AUR for updates
:param filter_packages: do not check every package just specified in the list
:param no_vcs: do not check VCS packages
:return: list of packages which are out-of-dated
'''
"""
result: List[Package] = []
build_section = self.config.get_section_name('build', self.architecture)
ignore_list = self.config.getlist(build_section, 'ignore_packages')
build_section = self.config.get_section_name("build", self.architecture)
ignore_list = self.config.getlist(build_section, "ignore_packages")
for local in self.packages():
if local.base in ignore_list:
@ -64,29 +62,29 @@ class UpdateHandler(Cleaner):
result.append(remote)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
self.logger.exception(f"could not load remote package {local.base}")
continue
return result
def updates_manual(self) -> List[Package]:
'''
"""
check for packages for which manual update has been requested
:return: list of packages which are out-of-dated
'''
"""
result: List[Package] = []
known_bases = {package.base for package in self.packages()}
for fn in os.listdir(self.paths.manual):
for fn in self.paths.manual.iterdir():
try:
local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
local = Package.load(fn, self.pacman, self.aur_url)
result.append(local)
if local.base not in known_bases:
self.reporter.set_unknown(local)
else:
self.reporter.set_pending(local.base)
except Exception:
self.logger.exception(f'could not add package from {fn}', exc_info=True)
self.logger.exception(f"could not add package from {fn}")
self.clear_manual()
return result

2
src/ahriman/core/sign/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

63
src/ahriman/core/sign/gpg.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -18,8 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import os
from pathlib import Path
from typing import List
from ahriman.core.configuration import Configuration
@ -29,79 +29,80 @@ from ahriman.models.sign_settings import SignSettings
class GPG:
'''
"""
gnupg wrapper
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar default_key: default PGP key ID to use
:ivar logger: class logger
:ivar target: list of targets to sign (repository, package etc)
'''
"""
_check_output = check_output
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('build_details')
"""
self.logger = logging.getLogger("build_details")
self.config = config
self.section = config.get_section_name('sign', architecture)
self.target = [SignSettings.from_option(opt) for opt in config.getlist(self.section, 'target')]
self.default_key = config.get(self.section, 'key') if self.target else ''
self.section = config.get_section_name("sign", architecture)
self.target = {SignSettings.from_option(opt) for opt in config.getlist(self.section, "target")}
self.default_key = config.get(self.section, "key") if self.target else ""
@property
def repository_sign_args(self) -> List[str]:
'''
"""
:return: command line arguments for repo-add command to sign database
'''
"""
if SignSettings.SignRepository not in self.target:
return []
return ['--sign', '--key', self.default_key]
return ["--sign", "--key", self.default_key]
@staticmethod
def sign_cmd(path: str, key: str) -> List[str]:
'''
def sign_cmd(path: Path, key: str) -> List[str]:
"""
gpg command to run
:param path: path to file to sign
:param key: PGP key ID
:return: gpg command with all required arguments
'''
return ['gpg', '-u', key, '-b', path]
"""
return ["gpg", "-u", key, "-b", str(path)]
def process(self, path: str, key: str) -> List[str]:
'''
def process(self, path: Path, key: str) -> List[Path]:
"""
gpg command wrapper
:param path: path to file to sign
:param key: PGP key ID
:return: list of generated files including original file
'''
check_output(
"""
GPG._check_output(
*GPG.sign_cmd(path, key),
exception=BuildFailed(path),
cwd=os.path.dirname(path),
exception=BuildFailed(path.name),
logger=self.logger)
return [path, f'{path}.sig']
return [path, path.parent / f"{path.name}.sig"]
def sign_package(self, path: str, base: str) -> List[str]:
'''
def sign_package(self, path: Path, base: str) -> List[Path]:
"""
sign package if required by configuration
:param path: path to file to sign
:param base: package base required to check for key overrides
:return: list of generated files including original file
'''
"""
if SignSettings.SignPackages not in self.target:
return [path]
key = self.config.get(self.section, f'key_{base}', fallback=self.default_key)
key = self.config.get(self.section, f"key_{base}", fallback=self.default_key)
return self.process(path, key)
def sign_repository(self, path: str) -> List[str]:
'''
def sign_repository(self, path: Path) -> List[Path]:
"""
sign repository if required by configuration
:note: more likely you just want to pass `repository_sign_args` to repo wrapper
:param path: path to repository database
:return: list of generated files including original file
'''
"""
if SignSettings.SignRepository not in self.target:
return [path]
return self.process(path, self.default_key)

2
src/ahriman/core/watcher/__init__.py → src/ahriman/core/status/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

66
src/ahriman/core/watcher/client.py → src/ahriman/core/status/client.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -27,102 +27,102 @@ from ahriman.models.package import Package
class Client:
'''
"""
base build status reporter client
'''
"""
def add(self, package: Package, status: BuildStatusEnum) -> None:
'''
"""
add new package with status
:param package: package properties
:param status: current package build status
'''
"""
# pylint: disable=R0201
# pylint: disable=no-self-use
def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
'''
"""
get package status
:param base: package base to get
:return: list of current package description and status if it has been found
'''
"""
del base
return []
# pylint: disable=R0201
# pylint: disable=no-self-use
def get_self(self) -> BuildStatus:
'''
"""
get ahriman status itself
:return: current ahriman status
'''
"""
return BuildStatus()
def remove(self, base: str) -> None:
'''
"""
remove packages from watcher
:param base: package base to remove
'''
"""
def update(self, base: str, status: BuildStatusEnum) -> None:
'''
"""
update package build status. Unlike `add` it does not update package properties
:param base: package base to update
:param status: current package build status
'''
"""
def update_self(self, status: BuildStatusEnum) -> None:
'''
"""
update ahriman status itself
:param status: current ahriman status
'''
"""
def set_building(self, base: str) -> None:
'''
"""
set package status to building
:param base: package base to update
'''
"""
return self.update(base, BuildStatusEnum.Building)
def set_failed(self, base: str) -> None:
'''
"""
set package status to failed
:param base: package base to update
'''
"""
return self.update(base, BuildStatusEnum.Failed)
def set_pending(self, base: str) -> None:
'''
"""
set package status to pending
:param base: package base to update
'''
"""
return self.update(base, BuildStatusEnum.Pending)
def set_success(self, package: Package) -> None:
'''
"""
set package status to success
:param package: current package properties
'''
"""
return self.add(package, BuildStatusEnum.Success)
def set_unknown(self, package: Package) -> None:
'''
"""
set package status to unknown
:param package: current package properties
'''
"""
return self.add(package, BuildStatusEnum.Unknown)
@staticmethod
def load(architecture: str, config: Configuration) -> Client:
'''
"""
load client from settings
:param architecture: repository architecture
:param config: configuration instance
:return: client according to current settings
'''
section = config.get_section_name('web', architecture)
host = config.get(section, 'host', fallback=None)
port = config.getint(section, 'port', fallback=None)
"""
section = config.get_section_name("web", architecture)
host = config.get(section, "host", fallback=None)
port = config.getint(section, "port", fallback=None)
if host is None or port is None:
return Client()
from ahriman.core.watcher.web_client import WebClient
from ahriman.core.status.web_client import WebClient
return WebClient(host, port)

93
src/ahriman/core/watcher/watcher.py → src/ahriman/core/status/watcher.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -19,33 +19,34 @@
#
import json
import logging
import os
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import UnknownPackage
from ahriman.core.repository.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.package import Package
class Watcher:
'''
"""
package status watcher
:ivar architecture: repository architecture
:ivar known: list of known packages. For the most cases `packages` should be used instead
:ivar logger: class logger
:ivar repository: repository object
:ivar status: daemon status
'''
"""
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('http')
"""
self.logger = logging.getLogger("http")
self.architecture = architecture
self.repository = Repository(architecture, config)
@ -54,68 +55,75 @@ class Watcher:
self.status = BuildStatus()
@property
def cache_path(self) -> str:
'''
def cache_path(self) -> Path:
"""
:return: path to dump with json cache
'''
return os.path.join(self.repository.paths.root, 'status_cache.json')
"""
return self.repository.paths.root / "status_cache.json"
@property
def packages(self) -> List[Tuple[Package, BuildStatus]]:
'''
"""
:return: list of packages together with their statuses
'''
"""
return list(self.known.values())
def _cache_load(self) -> None:
'''
"""
update current state from cache
'''
"""
def parse_single(properties: Dict[str, Any]) -> None:
package = Package.from_json(properties['package'])
status = BuildStatus.from_json(properties['status'])
package = Package.from_json(properties["package"])
status = BuildStatus.from_json(properties["status"])
if package.base in self.known:
self.known[package.base] = (package, status)
if not os.path.isfile(self.cache_path):
if not self.cache_path.is_file():
return
with open(self.cache_path) as cache:
dump = json.load(cache)
for item in dump['packages']:
with self.cache_path.open() as cache:
try:
dump = json.load(cache)
except Exception:
self.logger.exception("cannot parse json from file")
dump = {}
for item in dump.get("packages", []):
try:
parse_single(item)
except Exception:
self.logger.exception(f'cannot parse item f{item} to package', exc_info=True)
self.logger.exception(f"cannot parse item f{item} to package")
def _cache_save(self) -> None:
'''
"""
dump current cache to filesystem
'''
"""
dump = {
'packages': [
"packages": [
{
'package': package.view(),
'status': status.view()
"package": package.view(),
"status": status.view()
} for package, status in self.packages
]
}
try:
with open(self.cache_path, 'w') as cache:
with self.cache_path.open("w") as cache:
json.dump(dump, cache)
except Exception:
self.logger.exception('cannot dump cache', exc_info=True)
self.logger.exception("cannot dump cache")
def get(self, base: str) -> Tuple[Package, BuildStatus]:
'''
"""
get current package base build status
:return: package and its status
'''
return self.known[base]
"""
try:
return self.known[base]
except KeyError:
raise UnknownPackage(base)
def load(self) -> None:
'''
"""
load packages from local repository. In case if last status is known, it will use it
'''
"""
for package in self.repository.packages():
# get status of build or assign unknown
current = self.known.get(package.base)
@ -127,29 +135,32 @@ class Watcher:
self._cache_load()
def remove(self, base: str) -> None:
'''
"""
remove package base from known list if any
:param base: package base
'''
"""
self.known.pop(base, None)
self._cache_save()
def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
'''
"""
update package status and description
:param base: package base to update
:param status: new build status
:param package: optional new package description. In case if not set current properties will be used
'''
"""
if package is None:
package, _ = self.known[base]
try:
package, _ = self.known[base]
except KeyError:
raise UnknownPackage(base)
full_status = BuildStatus(status)
self.known[base] = (package, full_status)
self._cache_save()
def update_self(self, status: BuildStatusEnum) -> None:
'''
"""
update service status
:param status: new service status
'''
"""
self.status = BuildStatus(status)

92
src/ahriman/core/watcher/web_client.py → src/ahriman/core/status/web_client.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -18,93 +18,93 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from typing import List, Optional, Tuple
import requests
from ahriman.core.watcher.client import Client
from typing import List, Optional, Tuple
from ahriman.core.status.client import Client
from ahriman.models.build_status import BuildStatusEnum, BuildStatus
from ahriman.models.package import Package
class WebClient(Client):
'''
"""
build status reporter web client
:ivar host: host of web service
:ivar logger: class logger
:ivar port: port of web service
'''
"""
def __init__(self, host: str, port: int) -> None:
'''
"""
default constructor
:param host: host of web service
:param port: port of web service
'''
self.logger = logging.getLogger('http')
"""
self.logger = logging.getLogger("http")
self.host = host
self.port = port
def _ahriman_url(self) -> str:
'''
"""
url generator
:return: full url for web service for ahriman service itself
'''
return f'http://{self.host}:{self.port}/api/v1/ahriman'
"""
return f"http://{self.host}:{self.port}/api/v1/ahriman"
def _package_url(self, base: str = '') -> str:
'''
def _package_url(self, base: str = "") -> str:
"""
url generator
:param base: package base to generate url
:return: full url of web service for specific package base
'''
return f'http://{self.host}:{self.port}/api/v1/packages/{base}'
"""
return f"http://{self.host}:{self.port}/api/v1/packages/{base}"
def add(self, package: Package, status: BuildStatusEnum) -> None:
'''
"""
add new package with status
:param package: package properties
:param status: current package build status
'''
"""
payload = {
'status': status.value,
'package': package.view()
"status": status.value,
"package": package.view()
}
try:
response = requests.post(self._package_url(package.base), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not add {package.base}: {e.response.text}', exc_info=True)
self.logger.exception(f"could not add {package.base}: {e.response.text}")
except Exception:
self.logger.exception(f'could not add {package.base}', exc_info=True)
self.logger.exception(f"could not add {package.base}")
def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
'''
"""
get package status
:param base: package base to get
:return: list of current package description and status if it has been found
'''
"""
try:
response = requests.get(self._package_url(base or ''))
response = requests.get(self._package_url(base or ""))
response.raise_for_status()
status_json = response.json()
return [
(Package.from_json(package['package']), BuildStatus.from_json(package['status']))
(Package.from_json(package["package"]), BuildStatus.from_json(package["status"]))
for package in status_json
]
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not get {base}: {e.response.text}', exc_info=True)
self.logger.exception(f"could not get {base}: {e.response.text}")
except Exception:
self.logger.exception(f'could not get {base}', exc_info=True)
self.logger.exception(f"could not get {base}")
return []
def get_self(self) -> BuildStatus:
'''
"""
get ahriman status itself
:return: current ahriman status
'''
"""
try:
response = requests.get(self._ahriman_url())
response.raise_for_status()
@ -112,51 +112,51 @@ class WebClient(Client):
status_json = response.json()
return BuildStatus.from_json(status_json)
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not get service status: {e.response.text}', exc_info=True)
self.logger.exception(f"could not get service status: {e.response.text}")
except Exception:
self.logger.exception('could not get service status', exc_info=True)
self.logger.exception("could not get service status")
return BuildStatus()
def remove(self, base: str) -> None:
'''
"""
remove packages from watcher
:param base: basename to remove
'''
"""
try:
response = requests.delete(self._package_url(base))
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not delete {base}: {e.response.text}', exc_info=True)
self.logger.exception(f"could not delete {base}: {e.response.text}")
except Exception:
self.logger.exception(f'could not delete {base}', exc_info=True)
self.logger.exception(f"could not delete {base}")
def update(self, base: str, status: BuildStatusEnum) -> None:
'''
"""
update package build status. Unlike `add` it does not update package properties
:param base: package base to update
:param status: current package build status
'''
payload = {'status': status.value}
"""
payload = {"status": status.value}
try:
response = requests.post(self._package_url(base), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not update {base}: {e.response.text}', exc_info=True)
self.logger.exception(f"could not update {base}: {e.response.text}")
except Exception:
self.logger.exception(f'could not update {base}', exc_info=True)
self.logger.exception(f"could not update {base}")
def update_self(self, status: BuildStatusEnum) -> None:
'''
"""
update ahriman status itself
:param status: current ahriman status
'''
payload = {'status': status.value}
"""
payload = {"status": status.value}
try:
response = requests.post(self._ahriman_url(), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not update service status: {e.response.text}', exc_info=True)
self.logger.exception(f"could not update service status: {e.response.text}")
except Exception:
self.logger.exception('could not update service status', exc_info=True)
self.logger.exception("could not update service status")

88
src/ahriman/core/tree.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -22,74 +22,90 @@ from __future__ import annotations
import shutil
import tempfile
from typing import Iterable, List, Set
from pathlib import Path
from typing import Iterable, List, Set, Type
from ahriman.core.build_tools.task import Task
from ahriman.models.package import Package
class Leaf:
'''
"""
tree leaf implementation
:ivar dependencies: list of package dependencies
:ivar package: leaf package properties
'''
"""
def __init__(self, package: Package) -> None:
'''
def __init__(self, package: Package, dependencies: Set[str]) -> None:
"""
default constructor
:param package: package properties
'''
:param dependencies: package dependencies
"""
self.package = package
self.dependencies: Set[str] = set()
self.dependencies = dependencies
@property
def items(self) -> Iterable[str]:
'''
"""
:return: packages containing in this leaf
'''
"""
return self.package.packages.keys()
@classmethod
def load(cls: Type[Leaf], package: Package) -> Leaf:
"""
load leaf from package with dependencies
:param package: package properties
:return: loaded class
"""
clone_dir = Path(tempfile.mkdtemp())
try:
Task.fetch(clone_dir, package.git_url)
dependencies = Package.dependencies(clone_dir)
finally:
shutil.rmtree(clone_dir, ignore_errors=True)
return cls(package, dependencies)
def is_root(self, packages: Iterable[Leaf]) -> bool:
'''
"""
check if package depends on any other package from list of not
:param packages: list of known leaves
:return: True if any of packages is dependency of the leaf, False otherwise
'''
"""
for leaf in packages:
if self.dependencies.intersection(leaf.items):
return False
return True
def load_dependencies(self) -> None:
'''
load dependencies for the leaf
'''
clone_dir = tempfile.mkdtemp()
try:
Task.fetch(clone_dir, self.package.git_url)
self.dependencies = Package.dependencies(clone_dir)
finally:
shutil.rmtree(clone_dir, ignore_errors=True)
class Tree:
'''
"""
dependency tree implementation
:ivar leaves: list of tree leaves
'''
"""
def __init__(self) -> None:
'''
def __init__(self, leaves: List[Leaf]) -> None:
"""
default constructor
'''
self.leaves: List[Leaf] = []
:param leaves: leaves to build the tree
"""
self.leaves = leaves
@classmethod
def load(cls: Type[Tree], packages: Iterable[Package]) -> Tree:
"""
load tree from packages
:param packages: packages list
:return: loaded class
"""
return cls([Leaf.load(package) for package in packages])
def levels(self) -> List[List[Package]]:
'''
"""
get build levels starting from the packages which do not require any other package to build
:return: list of packages lists
'''
"""
result: List[List[Package]] = []
unprocessed = self.leaves[:]
@ -98,13 +114,3 @@ class Tree:
unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)]
return result
def load(self, packages: Iterable[Package]) -> None:
'''
load tree from packages
:param packages: packages list
'''
for package in packages:
leaf = Leaf(package)
leaf.load_dependencies()
self.leaves.append(leaf)

2
src/ahriman/core/upload/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

38
src/ahriman/core/upload/rsync.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,32 +17,44 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.upload.uploader import Uploader
from ahriman.core.util import check_output
class Rsync(Uploader):
'''
"""
rsync wrapper
:ivar remote: remote address to sync
'''
"""
_check_output = check_output
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Uploader.__init__(self, architecture, config)
section = config.get_section_name('rsync', architecture)
self.remote = config.get(section, 'remote')
section = config.get_section_name("rsync", architecture)
self.remote = config.get(section, "remote")
def sync(self, path: str) -> None:
'''
def sync(self, path: Path) -> None:
"""
sync data to remote server
:param path: local path to sync
'''
check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--delete', path, self.remote,
exception=None,
logger=self.logger)
"""
Rsync._check_output(
"rsync",
"--archive",
"--verbose",
"--compress",
"--partial",
"--delete",
str(path),
self.remote,
exception=None,
logger=self.logger)

30
src/ahriman/core/upload/s3.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,33 +17,37 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.upload.uploader import Uploader
from ahriman.core.util import check_output
class S3(Uploader):
'''
"""
aws-cli wrapper
:ivar bucket: full bucket name
'''
"""
_check_output = check_output
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
"""
Uploader.__init__(self, architecture, config)
section = config.get_section_name('s3', architecture)
self.bucket = config.get(section, 'bucket')
section = config.get_section_name("s3", architecture)
self.bucket = config.get(section, "bucket")
def sync(self, path: str) -> None:
'''
def sync(self, path: Path) -> None:
"""
sync data to remote server
:param path: local path to sync
'''
"""
# TODO rewrite to boto, but it is bullshit
check_output('aws', 's3', 'sync', '--quiet', '--delete', path, self.bucket,
exception=None,
logger=self.logger)
S3._check_output("aws", "s3", "sync", "--quiet", "--delete", str(path), self.bucket,
exception=None,
logger=self.logger)

28
src/ahriman/core/upload/uploader.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -19,38 +19,40 @@
#
import logging
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SyncFailed
from ahriman.models.upload_settings import UploadSettings
class Uploader:
'''
"""
base remote sync class
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar logger: application logger
'''
"""
def __init__(self, architecture: str, config: Configuration) -> None:
'''
"""
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('builder')
"""
self.logger = logging.getLogger("builder")
self.architecture = architecture
self.config = config
@staticmethod
def run(architecture: str, config: Configuration, target: str, path: str) -> None:
'''
def run(architecture: str, config: Configuration, target: str, path: Path) -> None:
"""
run remote sync
:param architecture: repository architecture
:param config: configuration instance
:param target: target to run sync (e.g. s3)
:param path: local path to sync
'''
"""
provider = UploadSettings.from_option(target)
if provider == UploadSettings.Rsync:
from ahriman.core.upload.rsync import Rsync
@ -64,11 +66,11 @@ class Uploader:
try:
uploader.sync(path)
except Exception:
uploader.logger.exception('remote sync failed', exc_info=True)
uploader.logger.exception(f"remote sync failed for {provider.name}")
raise SyncFailed()
def sync(self, path: str) -> None:
'''
def sync(self, path: Path) -> None:
"""
sync data to remote server
:param path: local path to sync
'''
"""

50
src/ahriman/core/util.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -21,74 +21,74 @@ import datetime
import subprocess
from logging import Logger
from pathlib import Path
from typing import Optional
from ahriman.core.exceptions import InvalidOption
def check_output(*args: str, exception: Optional[Exception],
cwd: Optional[str] = None, stderr: int = subprocess.STDOUT,
logger: Optional[Logger] = None) -> str:
'''
cwd: Optional[Path] = None, logger: Optional[Logger] = None) -> str:
"""
subprocess wrapper
:param args: command line arguments
:param exception: exception which has to be reraised instead of default subprocess exception
:param cwd: current working directory
:param stderr: standard error output mode
:param logger: logger to log command result if required
:return: command output
'''
"""
try:
result = subprocess.check_output(args, cwd=cwd, stderr=stderr).decode('utf8').strip()
result = subprocess.check_output(args, cwd=cwd, stderr=subprocess.STDOUT).decode("utf8").strip()
if logger is not None:
for line in result.splitlines():
logger.debug(line)
except subprocess.CalledProcessError as e:
if e.output is not None and logger is not None:
for line in e.output.decode('utf8').splitlines():
for line in e.output.decode("utf8").splitlines():
logger.debug(line)
raise exception or e
return result
def package_like(filename: str) -> bool:
'''
def package_like(filename: Path) -> bool:
"""
check if file looks like package
:param filename: name of file to check
:return: True in case if name contains `.pkg.` and not signature, False otherwise
'''
return '.pkg.' in filename and not filename.endswith('.sig')
"""
name = filename.name
return ".pkg." in name and not name.endswith(".sig")
def pretty_datetime(timestamp: Optional[int]) -> str:
'''
"""
convert datetime object to string
:param timestamp: datetime to convert
:return: pretty printable datetime as string
'''
return '' if timestamp is None else datetime.datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
"""
return "" if timestamp is None else datetime.datetime.utcfromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S")
def pretty_size(size: Optional[float], level: int = 0) -> str:
'''
"""
convert size to string
:param size: size to convert
:param level: represents current units, 0 is B, 1 is KiB etc
:return: pretty printable size as string
'''
"""
def str_level() -> str:
if level == 0:
return 'B'
return "B"
if level == 1:
return 'KiB'
return "KiB"
if level == 2:
return 'MiB'
return "MiB"
if level == 3:
return 'GiB'
raise InvalidOption(level) # I hope it will not be more than 1024 GiB
return "GiB"
raise InvalidOption(level) # must never happen actually
if size is None:
return ''
if size < 1024:
return f'{round(size, 2)} {str_level()}'
return ""
if size < 1024 or level == 3:
return f"{size:.1f} {str_level()}"
return pretty_size(size / 1024, level + 1)

2
src/ahriman/models/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

74
src/ahriman/models/build_status.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -28,83 +28,93 @@ from ahriman.core.util import pretty_datetime
class BuildStatusEnum(Enum):
'''
"""
build status enumeration
:cvar Unknown: build status is unknown
:cvar Pending: package is out-of-dated and will be built soon
:cvar Building: package is building right now
:cvar Failed: package build failed
:cvar Success: package has been built without errors
'''
"""
Unknown = 'unknown'
Pending = 'pending'
Building = 'building'
Failed = 'failed'
Success = 'success'
Unknown = "unknown"
Pending = "pending"
Building = "building"
Failed = "failed"
Success = "success"
def badges_color(self) -> str:
'''
"""
convert itself to shield.io badges color
:return: shields.io color
'''
"""
if self == BuildStatusEnum.Pending:
return 'yellow'
return "yellow"
if self == BuildStatusEnum.Building:
return 'yellow'
return "yellow"
if self == BuildStatusEnum.Failed:
return 'critical'
return "critical"
if self == BuildStatusEnum.Success:
return 'success'
return 'inactive'
return "success"
return "inactive"
class BuildStatus:
'''
"""
build status holder
:ivar status: build status
:ivar _timestamp: build status update time
'''
"""
def __init__(self, status: Union[BuildStatusEnum, str, None] = None,
timestamp: Optional[int] = None) -> None:
'''
"""
default constructor
:param status: current build status if known. `BuildStatusEnum.Unknown` will be used if not set
:param timestamp: build status timestamp. Current timestamp will be used if not set
'''
"""
self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown
self.timestamp = timestamp or int(datetime.datetime.utcnow().timestamp())
@classmethod
def from_json(cls: Type[BuildStatus], dump: Dict[str, Any]) -> BuildStatus:
'''
"""
construct status properties from json dump
:param dump: json dump body
:return: status properties
'''
return cls(dump.get('status'), dump.get('timestamp'))
"""
return cls(dump.get("status"), dump.get("timestamp"))
def pretty_print(self) -> str:
'''
"""
generate pretty string representation
:return: print-friendly string
'''
return f'{self.status.value} ({pretty_datetime(self.timestamp)})'
"""
return f"{self.status.value} ({pretty_datetime(self.timestamp)})"
def view(self) -> Dict[str, Any]:
'''
"""
generate json status view
:return: json-friendly dictionary
'''
"""
return {
'status': self.status.value,
'timestamp': self.timestamp
"status": self.status.value,
"timestamp": self.timestamp
}
def __eq__(self, other: Any) -> bool:
"""
compare object to other
:param other: other object to compare
:return: True in case if objects are equal
"""
if not isinstance(other, BuildStatus):
return False
return self.status == other.status and self.timestamp == other.timestamp
def __repr__(self) -> str:
'''
"""
generate string representation of object
:return: unique string representation
'''
return f'BuildStatus(status={self.status.value}, timestamp={self.timestamp})'
"""
return f"BuildStatus(status={self.status.value}, timestamp={self.timestamp})"

190
src/ahriman/models/package.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -21,12 +21,12 @@ from __future__ import annotations
import aur # type: ignore
import logging
import os
from dataclasses import asdict, dataclass
from pathlib import Path
from pyalpm import vercmp # type: ignore
from srcinfo.parse import parse_srcinfo # type: ignore
from typing import Any, Dict, List, Optional, Set, Type
from typing import Any, Dict, List, Optional, Set, Type, Union
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.exceptions import InvalidPackageInfo
@ -37,158 +37,166 @@ from ahriman.models.repository_paths import RepositoryPaths
@dataclass
class Package:
'''
"""
package properties representation
:ivar aurl_url: AUR root url
:ivar aur_url: AUR root url
:ivar base: package base name
:ivar packages: map of package names to their properties. Filled only on load from archive
:ivar version: package full version
'''
"""
base: str
version: str
aur_url: str
packages: Dict[str, PackageDescription]
_check_output = check_output
@property
def git_url(self) -> str:
'''
"""
:return: package git url to clone
'''
return f'{self.aur_url}/{self.base}.git'
"""
return f"{self.aur_url}/{self.base}.git"
@property
def is_single_package(self) -> bool:
'''
"""
:return: true in case if this base has only one package with the same name
'''
"""
return self.base in self.packages and len(self.packages) == 1
@property
def is_vcs(self) -> bool:
'''
"""
:return: True in case if package base looks like VCS package and false otherwise
'''
return self.base.endswith('-bzr') \
or self.base.endswith('-csv')\
or self.base.endswith('-darcs')\
or self.base.endswith('-git')\
or self.base.endswith('-hg')\
or self.base.endswith('-svn')
"""
return self.base.endswith("-bzr") \
or self.base.endswith("-csv")\
or self.base.endswith("-darcs")\
or self.base.endswith("-git")\
or self.base.endswith("-hg")\
or self.base.endswith("-svn")
@property
def web_url(self) -> str:
'''
"""
:return: package AUR url
'''
return f'{self.aur_url}/packages/{self.base}'
"""
return f"{self.aur_url}/packages/{self.base}"
@classmethod
def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
'''
def from_archive(cls: Type[Package], path: Path, pacman: Pacman, aur_url: str) -> Package:
"""
construct package properties from package archive
:param path: path to package archive
:param pacman: alpm wrapper instance
:param aur_url: AUR root url
:return: package properties
'''
package = pacman.handle.load_pkg(path)
properties = PackageDescription(package.size, package.builddate, os.path.basename(path), package.isize)
"""
package = pacman.handle.load_pkg(str(path))
properties = PackageDescription(package.size, package.builddate, path.name, package.isize)
return cls(package.base, package.version, aur_url, {package.name: properties})
@classmethod
def from_aur(cls: Type[Package], name: str, aur_url: str) -> Package:
'''
"""
construct package properties from AUR page
:param name: package name (either base or normal name)
:param aur_url: AUR root url
:return: package properties
'''
"""
package = aur.info(name)
return cls(package.package_base, package.version, aur_url, {package.name: PackageDescription()})
@classmethod
def from_build(cls: Type[Package], path: str, aur_url: str) -> Package:
'''
def from_build(cls: Type[Package], path: Path, aur_url: str) -> Package:
"""
construct package properties from sources directory
:param path: path to package sources directory
:param aur_url: AUR root url
:return: package properties
'''
with open(os.path.join(path, '.SRCINFO')) as srcinfo_file:
srcinfo, errors = parse_srcinfo(srcinfo_file.read())
"""
srcinfo, errors = parse_srcinfo((path / ".SRCINFO").read_text())
if errors:
raise InvalidPackageInfo(errors)
packages = {key: PackageDescription() for key in srcinfo['packages']}
version = cls.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel'])
packages = {key: PackageDescription() for key in srcinfo["packages"]}
version = cls.full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
return cls(srcinfo['pkgbase'], version, aur_url, packages)
return cls(srcinfo["pkgbase"], version, aur_url, packages)
@classmethod
def from_json(cls: Type[Package], dump: Dict[str, Any]) -> Package:
'''
"""
construct package properties from json dump
:param dump: json dump body
:return: package properties
'''
"""
packages = {
key: PackageDescription(**value)
for key, value in dump.get('packages', {}).items()
for key, value in dump.get("packages", {}).items()
}
return Package(
base=dump['base'],
version=dump['version'],
aur_url=dump['aur_url'],
base=dump["base"],
version=dump["version"],
aur_url=dump["aur_url"],
packages=packages)
@staticmethod
def dependencies(path: str) -> Set[str]:
'''
def dependencies(path: Path) -> Set[str]:
"""
load dependencies from package sources
:param path: path to package sources directory
:return: list of package dependencies including makedepends array, but excluding packages from this base
'''
with open(os.path.join(path, '.SRCINFO')) as srcinfo_file:
srcinfo, errors = parse_srcinfo(srcinfo_file.read())
"""
# additional function to remove versions from dependencies
def trim_version(name: str) -> str:
for symbol in ("<", "=", ">"):
name = name.split(symbol)[0]
return name
srcinfo, errors = parse_srcinfo((path / ".SRCINFO").read_text())
if errors:
raise InvalidPackageInfo(errors)
makedepends = srcinfo.get('makedepends', [])
makedepends = srcinfo.get("makedepends", [])
# sum over each package
depends: List[str] = srcinfo.get('depends', [])
for package in srcinfo['packages'].values():
depends.extend(package.get('depends', []))
depends: List[str] = srcinfo.get("depends", [])
for package in srcinfo["packages"].values():
depends.extend(package.get("depends", []))
# we are not interested in dependencies inside pkgbase
packages = set(srcinfo['packages'].keys())
return set(depends + makedepends) - packages
packages = set(srcinfo["packages"].keys())
full_list = set(depends + makedepends) - packages
return {trim_version(package_name) for package_name in full_list}
@staticmethod
def full_version(epoch: Optional[str], pkgver: str, pkgrel: str) -> str:
'''
"""
generate full version from components
:param epoch: package epoch if any
:param pkgver: package version
:param pkgrel: package release version (archlinux specific)
:return: generated version
'''
prefix = f'{epoch}:' if epoch else ''
return f'{prefix}{pkgver}-{pkgrel}'
"""
prefix = f"{epoch}:" if epoch else ""
return f"{prefix}{pkgver}-{pkgrel}"
@staticmethod
def load(path: str, pacman: Pacman, aur_url: str) -> Package:
'''
def load(path: Union[Path, str], pacman: Pacman, aur_url: str) -> Package:
"""
package constructor from available sources
:param path: one of path to sources directory, path to archive or package name/base
:param pacman: alpm wrapper instance (required to load from archive)
:param aur_url: AUR root url
:return: package properties
'''
"""
try:
if os.path.isdir(path):
package: Package = Package.from_build(path, aur_url)
elif os.path.exists(path):
package = Package.from_archive(path, pacman, aur_url)
maybe_path = Path(path)
if maybe_path.is_dir():
package: Package = Package.from_build(maybe_path, aur_url)
elif maybe_path.is_file():
package = Package.from_archive(maybe_path, pacman, aur_url)
else:
package = Package.from_aur(path, aur_url)
package = Package.from_aur(str(path), aur_url)
return package
except InvalidPackageInfo:
raise
@ -196,52 +204,62 @@ class Package:
raise InvalidPackageInfo(str(e))
def actual_version(self, paths: RepositoryPaths) -> str:
'''
"""
additional method to handle VCS package versions
:param paths: repository paths instance
:return: package version if package is not VCS and current version according to VCS otherwise
'''
"""
if not self.is_vcs:
return self.version
from ahriman.core.build_tools.task import Task
clone_dir = os.path.join(paths.cache, self.base)
logger = logging.getLogger('build_details')
clone_dir = paths.cache / self.base
logger = logging.getLogger("build_details")
Task.fetch(clone_dir, self.git_url)
# update pkgver first
check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger)
# generate new .SRCINFO and put it to parser
srcinfo_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger)
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise InvalidPackageInfo(errors)
try:
# update pkgver first
Package._check_output("makepkg", "--nodeps", "--nobuild", exception=None, cwd=clone_dir, logger=logger)
# generate new .SRCINFO and put it to parser
srcinfo_source = Package._check_output(
"makepkg",
"--printsrcinfo",
exception=None,
cwd=clone_dir,
logger=logger)
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise InvalidPackageInfo(errors)
return self.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel'])
return self.full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
except Exception:
logger.exception("cannot determine version of VCS package, make sure that you have VCS tools installed")
return self.version
def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool:
'''
"""
check if package is out-of-dated
:param remote: package properties from remote source
:param paths: repository paths instance. Required for VCS packages cache
:return: True if the package is out-of-dated and False otherwise
'''
"""
remote_version = remote.actual_version(paths) # either normal version or updated VCS
result: int = vercmp(self.version, remote_version)
return result < 0
def pretty_print(self) -> str:
'''
"""
generate pretty string representation
:return: print-friendly string
'''
details = '' if self.is_single_package else f''' ({' '.join(sorted(self.packages.keys()))})'''
return f'{self.base}{details}'
"""
details = "" if self.is_single_package else f""" ({" ".join(sorted(self.packages.keys()))})"""
return f"{self.base}{details}"
def view(self) -> Dict[str, Any]:
'''
"""
generate json package view
:return: json-friendly dictionary
'''
"""
return asdict(self)

14
src/ahriman/models/package_desciption.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -18,20 +18,28 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
@dataclass
class PackageDescription:
'''
"""
package specific properties
:ivar archive_size: package archive size
:ivar build_date: package build date
:ivar filename: package archive name
:ivar installed_size: package installed size
'''
"""
archive_size: Optional[int] = None
build_date: Optional[int] = None
filename: Optional[str] = None
installed_size: Optional[int] = None
@property
def filepath(self) -> Optional[Path]:
"""
:return: path object for current filename
"""
return Path(self.filename) if self.filename is not None else None

12
src/ahriman/models/report_settings.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -25,20 +25,20 @@ from ahriman.core.exceptions import InvalidOption
class ReportSettings(Enum):
'''
"""
report targets enumeration
:cvar HTML: html report generation
'''
"""
HTML = auto()
@staticmethod
def from_option(value: str) -> ReportSettings:
'''
"""
construct value from configuration
:param value: configuration value
:return: parsed value
'''
if value.lower() in ('html',):
"""
if value.lower() in ("html",):
return ReportSettings.HTML
raise InvalidOption(value)

76
src/ahriman/models/repository_paths.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,72 +17,72 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from pathlib import Path
from dataclasses import dataclass
@dataclass
class RepositoryPaths:
'''
"""
repository paths holder. For the most operations with paths you want to use this object
:ivar root: repository root (i.e. ahriman home)
:ivar architecture: repository architecture
'''
"""
root: str
root: Path
architecture: str
@property
def cache(self) -> str:
'''
def cache(self) -> Path:
"""
:return: directory for packages cache (mainly used for VCS packages)
'''
return os.path.join(self.root, 'cache')
"""
return self.root / "cache"
@property
def chroot(self) -> str:
'''
def chroot(self) -> Path:
"""
:return: directory for devtools chroot
'''
# for the chroot directory devtools will create own tree and we don't have to specify architecture here
return os.path.join(self.root, 'chroot')
"""
# for the chroot directory devtools will create own tree and we don"t have to specify architecture here
return self.root / "chroot"
@property
def manual(self) -> str:
'''
def manual(self) -> Path:
"""
:return: directory for manual updates (i.e. from add command)
'''
return os.path.join(self.root, 'manual', self.architecture)
"""
return self.root / "manual" / self.architecture
@property
def packages(self) -> str:
'''
def packages(self) -> Path:
"""
:return: directory for built packages
'''
return os.path.join(self.root, 'packages', self.architecture)
"""
return self.root / "packages" / self.architecture
@property
def repository(self) -> str:
'''
def repository(self) -> Path:
"""
:return: repository directory
'''
return os.path.join(self.root, 'repository', self.architecture)
"""
return self.root / "repository" / self.architecture
@property
def sources(self) -> str:
'''
def sources(self) -> Path:
"""
:return: directory for downloaded PKGBUILDs for current build
'''
return os.path.join(self.root, 'sources', self.architecture)
"""
return self.root / "sources" / self.architecture
def create_tree(self) -> None:
'''
"""
create ahriman working tree
'''
os.makedirs(self.cache, mode=0o755, exist_ok=True)
os.makedirs(self.chroot, mode=0o755, exist_ok=True)
os.makedirs(self.manual, mode=0o755, exist_ok=True)
os.makedirs(self.packages, mode=0o755, exist_ok=True)
os.makedirs(self.repository, mode=0o755, exist_ok=True)
os.makedirs(self.sources, mode=0o755, exist_ok=True)
"""
self.cache.mkdir(mode=0o755, parents=True, exist_ok=True)
self.chroot.mkdir(mode=0o755, parents=True, exist_ok=True)
self.manual.mkdir(mode=0o755, parents=True, exist_ok=True)
self.packages.mkdir(mode=0o755, parents=True, exist_ok=True)
self.repository.mkdir(mode=0o755, parents=True, exist_ok=True)
self.sources.mkdir(mode=0o755, parents=True, exist_ok=True)

14
src/ahriman/models/sign_settings.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -25,24 +25,24 @@ from ahriman.core.exceptions import InvalidOption
class SignSettings(Enum):
'''
"""
sign targets enumeration
:cvar SignPackages: sign each package
:cvar SignRepository: sign repository database file
'''
"""
SignPackages = auto()
SignRepository = auto()
@staticmethod
def from_option(value: str) -> SignSettings:
'''
"""
construct value from configuration
:param value: configuration value
:return: parsed value
'''
if value.lower() in ('package', 'packages', 'sign-package'):
"""
if value.lower() in ("package", "packages", "sign-package"):
return SignSettings.SignPackages
if value.lower() in ('repository', 'sign-repository'):
if value.lower() in ("repository", "sign-repository"):
return SignSettings.SignRepository
raise InvalidOption(value)

14
src/ahriman/models/upload_settings.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -25,24 +25,24 @@ from ahriman.core.exceptions import InvalidOption
class UploadSettings(Enum):
'''
"""
remote synchronization targets enumeration
:cvar Rsync: sync via rsync
:cvar S3: sync to Amazon S3
'''
"""
Rsync = auto()
S3 = auto()
@staticmethod
def from_option(value: str) -> UploadSettings:
'''
"""
construct value from configuration
:param value: configuration value
:return: parsed value
'''
if value.lower() in ('rsync',):
"""
if value.lower() in ("rsync",):
return UploadSettings.Rsync
if value.lower() in ('s3',):
if value.lower() in ("s3",):
return UploadSettings.S3
raise InvalidOption(value)

4
src/ahriman/version.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -17,4 +17,4 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.15.0'
__version__ = "0.15.0"

2
src/ahriman/web/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

2
src/ahriman/web/middlewares/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

8
src/ahriman/web/middlewares/exception_handler.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -28,11 +28,11 @@ HandlerType = Callable[[Request], Awaitable[StreamResponse]]
def exception_handler(logger: Logger) -> Callable[[Request, HandlerType], Awaitable[StreamResponse]]:
'''
"""
exception handler middleware. Just log any exception (except for client ones)
:param logger: class logger
:return: built middleware
'''
"""
@middleware
async def handle(request: Request, handler: HandlerType) -> StreamResponse:
try:
@ -40,7 +40,7 @@ def exception_handler(logger: Logger) -> Callable[[Request, HandlerType], Awaita
except HTTPClientError:
raise
except Exception:
logger.exception(f'exception during performing request to {request.path}', exc_info=True)
logger.exception(f"exception during performing request to {request.path}")
raise
return handle

24
src/ahriman/web/routes.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -26,7 +26,7 @@ from ahriman.web.views.packages import PackagesView
def setup_routes(application: Application) -> None:
'''
"""
setup all defined routes
Available routes are:
@ -45,16 +45,16 @@ def setup_routes(application: Application) -> None:
POST /api/v1/package/:base update package base status
:param application: web application instance
'''
application.router.add_get('/', IndexView)
application.router.add_get('/index.html', IndexView)
"""
application.router.add_get("/", IndexView)
application.router.add_get("/index.html", IndexView)
application.router.add_get('/api/v1/ahriman', AhrimanView)
application.router.add_post('/api/v1/ahriman', AhrimanView)
application.router.add_get("/api/v1/ahriman", AhrimanView)
application.router.add_post("/api/v1/ahriman", AhrimanView)
application.router.add_get('/api/v1/packages', PackagesView)
application.router.add_post('/api/v1/packages', PackagesView)
application.router.add_get("/api/v1/packages", PackagesView)
application.router.add_post("/api/v1/packages", PackagesView)
application.router.add_delete('/api/v1/packages/{package}', PackageView)
application.router.add_get('/api/v1/packages/{package}', PackageView)
application.router.add_post('/api/v1/packages/{package}', PackageView)
application.router.add_delete("/api/v1/packages/{package}", PackageView)
application.router.add_get("/api/v1/packages/{package}", PackageView)
application.router.add_post("/api/v1/packages/{package}", PackageView)

2
src/ahriman/web/views/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).

16
src/ahriman/web/views/ahriman.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -24,19 +24,19 @@ from ahriman.web.views.base import BaseView
class AhrimanView(BaseView):
'''
"""
service status web view
'''
"""
async def get(self) -> Response:
'''
"""
get current service status
:return: 200 with service status object
'''
"""
return json_response(self.service.status.view())
async def post(self) -> Response:
'''
"""
update service status
JSON body must be supplied, the following model is used:
@ -45,11 +45,11 @@ class AhrimanView(BaseView):
}
:return: 204 on success
'''
"""
data = await self.request.json()
try:
status = BuildStatusEnum(data['status'])
status = BuildStatusEnum(data["status"])
except Exception as e:
raise HTTPBadRequest(text=str(e))

14
src/ahriman/web/views/base.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -19,18 +19,18 @@
#
from aiohttp.web import View
from ahriman.core.watcher.watcher import Watcher
from ahriman.core.status.watcher import Watcher
class BaseView(View):
'''
"""
base web view to make things typed
'''
"""
@property
def service(self) -> Watcher:
'''
"""
:return: build status watcher instance
'''
watcher: Watcher = self.request.app['watcher']
"""
watcher: Watcher = self.request.app["watcher"]
return watcher

40
src/ahriman/web/views/index.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -28,7 +28,7 @@ from ahriman.web.views.base import BaseView
class IndexView(BaseView):
'''
"""
root view
It uses jinja2 templates for report generation, the following variables are allowed:
@ -39,35 +39,35 @@ class IndexView(BaseView):
repository - repository name, string, required
service - service status properties: status, status_color, timestamp. Required
version - ahriman version, string, required
'''
"""
@aiohttp_jinja2.template('build-status.jinja2')
@aiohttp_jinja2.template("build-status.jinja2")
async def get(self) -> Dict[str, Any]:
'''
"""
process get request. No parameters supported here
:return: parameters for jinja template
'''
"""
# some magic to make it jinja-friendly
packages = [
{
'base': package.base,
'packages': list(sorted(package.packages)),
'status': status.status.value,
'timestamp': pretty_datetime(status.timestamp),
'version': package.version,
'web_url': package.web_url
"base": package.base,
"packages": list(sorted(package.packages)),
"status": status.status.value,
"timestamp": pretty_datetime(status.timestamp),
"version": package.version,
"web_url": package.web_url
} for package, status in sorted(self.service.packages, key=lambda item: item[0].base)
]
service = {
'status': self.service.status.status.value,
'status_color': self.service.status.status.badges_color(),
'timestamp': pretty_datetime(self.service.status.timestamp)
"status": self.service.status.status.value,
"status_color": self.service.status.status.badges_color(),
"timestamp": pretty_datetime(self.service.status.timestamp)
}
return {
'architecture': self.service.architecture,
'packages': packages,
'repository': self.service.repository.name,
'service': service,
'version': version.__version__,
"architecture": self.service.architecture,
"packages": packages,
"repository": self.service.repository.name,
"service": service,
"version": version.__version__,
}

39
src/ahriman/web/views/package.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -19,48 +19,49 @@
#
from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
from ahriman.core.exceptions import UnknownPackage
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.web.views.base import BaseView
class PackageView(BaseView):
'''
"""
package base specific web view
'''
"""
async def get(self) -> Response:
'''
"""
get current package base status
:return: 200 with package description on success
'''
base = self.request.match_info['package']
"""
base = self.request.match_info["package"]
try:
package, status = self.service.get(base)
except KeyError:
except UnknownPackage:
raise HTTPNotFound()
response = [
{
'package': package.view(),
'status': status.view()
"package": package.view(),
"status": status.view()
}
]
return json_response(response)
async def delete(self) -> Response:
'''
"""
delete package base from status page
:return: 204 on success
'''
base = self.request.match_info['package']
"""
base = self.request.match_info["package"]
self.service.remove(base)
return HTTPNoContent()
async def post(self) -> Response:
'''
"""
update package build status
JSON body must be supplied, the following model is used:
@ -71,19 +72,19 @@ class PackageView(BaseView):
}
:return: 204 on success
'''
base = self.request.match_info['package']
"""
base = self.request.match_info["package"]
data = await self.request.json()
try:
package = Package.from_json(data['package']) if 'package' in data else None
status = BuildStatusEnum(data['status'])
package = Package.from_json(data["package"]) if "package" in data else None
status = BuildStatusEnum(data["status"])
except Exception as e:
raise HTTPBadRequest(text=str(e))
try:
self.service.update(base, status, package)
except KeyError:
raise HTTPBadRequest(text=f'Package {base} is unknown, but no package body set')
except UnknownPackage:
raise HTTPBadRequest(text=f"Package {base} is unknown, but no package body set")
return HTTPNoContent()

18
src/ahriman/web/views/packages.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -23,28 +23,28 @@ from ahriman.web.views.base import BaseView
class PackagesView(BaseView):
'''
"""
global watcher view
'''
"""
async def get(self) -> Response:
'''
"""
get current packages status
:return: 200 with package description on success
'''
"""
response = [
{
'package': package.view(),
'status': status.view()
"package": package.view(),
"status": status.view()
} for package, status in self.service.packages
]
return json_response(response)
async def post(self) -> Response:
'''
"""
reload all packages from repository. No parameters supported here
:return: 204 on success
'''
"""
self.service.load()
return HTTPNoContent()

59
src/ahriman/web/web.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
@ -25,71 +25,72 @@ from aiohttp import web
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import InitializeException
from ahriman.core.watcher.watcher import Watcher
from ahriman.core.status.watcher import Watcher
from ahriman.web.middlewares.exception_handler import exception_handler
from ahriman.web.routes import setup_routes
async def on_shutdown(application: web.Application) -> None:
'''
"""
web application shutdown handler
:param application: web application instance
'''
application.logger.warning('server terminated')
"""
application.logger.warning("server terminated")
async def on_startup(application: web.Application) -> None:
'''
"""
web application start handler
:param application: web application instance
'''
application.logger.info('server started')
"""
application.logger.info("server started")
try:
application['watcher'].load()
application["watcher"].load()
except Exception:
application.logger.exception('could not load packages', exc_info=True)
application.logger.exception("could not load packages")
raise InitializeException()
def run_server(application: web.Application, architecture: str) -> None:
'''
def run_server(application: web.Application) -> None:
"""
run web application
:param application: web application instance
:param architecture: repository architecture
'''
application.logger.info('start server')
"""
application.logger.info("start server")
section = application['config'].get_section_name('web', architecture)
host = application['config'].get(section, 'host')
port = application['config'].getint(section, 'port')
section = application["config"].get_section_name("web", application["architecture"])
host = application["config"].get(section, "host")
port = application["config"].getint(section, "port")
web.run_app(application, host=host, port=port, handle_signals=False,
access_log=logging.getLogger('http'))
access_log=logging.getLogger("http"))
def setup_service(architecture: str, config: Configuration) -> web.Application:
'''
"""
create web application
:param architecture: repository architecture
:param config: configuration instance
:return: web application instance
'''
application = web.Application(logger=logging.getLogger('http'))
"""
application = web.Application(logger=logging.getLogger("http"))
application.on_shutdown.append(on_shutdown)
application.on_startup.append(on_startup)
application.middlewares.append(web.normalize_path_middleware(append_slash=False, remove_slash=True))
application.middlewares.append(exception_handler(application.logger))
application.logger.info('setup routes')
application.logger.info("setup routes")
setup_routes(application)
application.logger.info('setup templates')
aiohttp_jinja2.setup(application, loader=jinja2.FileSystemLoader(config.get('web', 'templates')))
application.logger.info('setup configuration')
application['config'] = config
application.logger.info("setup templates")
aiohttp_jinja2.setup(application, loader=jinja2.FileSystemLoader(config.getpath("web", "templates")))
application.logger.info('setup watcher')
application['watcher'] = Watcher(architecture, config)
application.logger.info("setup configuration")
application["config"] = config
application["architecture"] = architecture
application.logger.info("setup watcher")
application["watcher"] = Watcher(architecture, config)
return application

30
tests/ahriman/application/conftest.py Normal file
View File

@ -0,0 +1,30 @@
import argparse
import pytest
from pytest_mock import MockerFixture
from ahriman.application.ahriman import _parser
from ahriman.application.application import Application
from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
@pytest.fixture
def application(configuration: Configuration, mocker: MockerFixture) -> Application:
mocker.patch("pathlib.Path.mkdir")
return Application("x86_64", configuration)
@pytest.fixture
def args() -> argparse.Namespace:
return argparse.Namespace(lock=None, force=False, unsafe=False, no_report=True)
@pytest.fixture
def lock(args: argparse.Namespace, configuration: Configuration) -> Lock:
return Lock(args, "x86_64", configuration)
@pytest.fixture
def parser() -> argparse.ArgumentParser:
return _parser()

27
tests/ahriman/application/handlers/test_handler.py Normal file
View File

@ -0,0 +1,27 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
def test_call(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must call inside lock
"""
mocker.patch("ahriman.application.handlers.Handler.run")
enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__")
exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__")
assert Handler._call(args, "x86_64", configuration)
enter_mock.assert_called_once()
exit_mock.assert_called_once()
def test_call_exception(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must process exception
"""
mocker.patch("ahriman.application.lock.Lock.__enter__", side_effect=Exception())
assert not Handler._call(args, "x86_64", configuration)

19
tests/ahriman/application/handlers/test_handler_add.py Normal file
View File

@ -0,0 +1,19 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Add
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
args.package = []
args.without_dependencies = False
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.application.application.Application.add")
Add.run(args, "x86_64", configuration)
application_mock.assert_called_once()

22
tests/ahriman/application/handlers/test_handler_clean.py Normal file
View File

@ -0,0 +1,22 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Clean
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
args.no_build = False
args.no_cache = False
args.no_chroot = False
args.no_manual = False
args.no_packages = False
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.application.application.Application.clean")
Clean.run(args, "x86_64", configuration)
application_mock.assert_called_once()

17
tests/ahriman/application/handlers/test_handler_dump.py Normal file
View File

@ -0,0 +1,17 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Dump
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.core.configuration.Configuration.dump")
Dump.run(args, "x86_64", configuration)
application_mock.assert_called_once()

19
tests/ahriman/application/handlers/test_handler_rebuild.py Normal file
View File

@ -0,0 +1,19 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Rebuild
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
mocker.patch("pathlib.Path.mkdir")
application_packages_mock = mocker.patch("ahriman.core.repository.repository.Repository.packages")
application_mock = mocker.patch("ahriman.application.application.Application.update")
Rebuild.run(args, "x86_64", configuration)
application_packages_mock.assert_called_once()
application_mock.assert_called_once()

18
tests/ahriman/application/handlers/test_handler_remove.py Normal file
View File

@ -0,0 +1,18 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Remove
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
args.package = []
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.application.application.Application.remove")
Remove.run(args, "x86_64", configuration)
application_mock.assert_called_once()

18
tests/ahriman/application/handlers/test_handler_report.py Normal file
View File

@ -0,0 +1,18 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Report
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
args.target = []
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.application.application.Application.report")
Report.run(args, "x86_64", configuration)
application_mock.assert_called_once()

22
tests/ahriman/application/handlers/test_handler_status.py Normal file
View File

@ -0,0 +1,22 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Status
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
args.ahriman = True
args.package = []
args.without_dependencies = False
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.core.status.client.Client.get_self")
packages_mock = mocker.patch("ahriman.core.status.client.Client.get")
Status.run(args, "x86_64", configuration)
application_mock.assert_called_once()
packages_mock.assert_called_once()

18
tests/ahriman/application/handlers/test_handler_sync.py Normal file
View File

@ -0,0 +1,18 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Sync
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
args.target = []
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.application.application.Application.sync")
Sync.run(args, "x86_64", configuration)
application_mock.assert_called_once()

40
tests/ahriman/application/handlers/test_handler_update.py Normal file
View File

@ -0,0 +1,40 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Update
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
args.package = []
args.dry_run = False
args.no_aur = False
args.no_manual = False
args.no_vcs = False
mocker.patch("pathlib.Path.mkdir")
application_mock = mocker.patch("ahriman.application.application.Application.update")
updates_mock = mocker.patch("ahriman.application.application.Application.get_updates")
Update.run(args, "x86_64", configuration)
application_mock.assert_called_once()
updates_mock.assert_called_once()
def test_run_dry_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run simplified command
"""
args.package = []
args.dry_run = True
args.no_aur = False
args.no_manual = False
args.no_vcs = False
mocker.patch("pathlib.Path.mkdir")
updates_mock = mocker.patch("ahriman.application.application.Application.get_updates")
Update.run(args, "x86_64", configuration)
updates_mock.assert_called_once()

19
tests/ahriman/application/handlers/test_handler_web.py Normal file
View File

@ -0,0 +1,19 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Web
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
mocker.patch("pathlib.Path.mkdir")
setup_mock = mocker.patch("ahriman.web.web.setup_service")
run_mock = mocker.patch("ahriman.web.web.run_server")
Web.run(args, "x86_64", configuration)
setup_mock.assert_called_once()
run_mock.assert_called_once()

55
tests/ahriman/application/test_ahriman.py Normal file
View File

@ -0,0 +1,55 @@
import argparse
def test_parser(parser: argparse.ArgumentParser) -> None:
"""
must parse valid command line
"""
parser.parse_args(["-a", "x86_64", "config"])
def test_multiple_architectures(parser: argparse.ArgumentParser) -> None:
"""
must accept multiple architectures
"""
args = parser.parse_args(["-a", "x86_64", "-a", "i686", "config"])
assert len(args.architecture) == 2
def test_subparsers_check(parser: argparse.ArgumentParser) -> None:
"""
check command must imply no_aur, no_manual and dry_run
"""
args = parser.parse_args(["-a", "x86_64", "check"])
assert not args.no_aur
assert args.no_manual
assert args.dry_run
def test_subparsers_config(parser: argparse.ArgumentParser) -> None:
"""
config command must imply lock, no_report and unsafe
"""
args = parser.parse_args(["-a", "x86_64", "config"])
assert args.lock is None
assert args.no_report
assert args.unsafe
def test_subparsers_status(parser: argparse.ArgumentParser) -> None:
"""
status command must imply lock, no_report and unsafe
"""
args = parser.parse_args(["-a", "x86_64", "status"])
assert args.lock is None
assert args.no_report
assert args.unsafe
def test_subparsers_web(parser: argparse.ArgumentParser) -> None:
"""
web command must imply lock and no_report
"""
args = parser.parse_args(["-a", "x86_64", "web"])
assert args.lock is None
assert args.no_report

237
tests/ahriman/application/test_application.py Normal file
View File

@ -0,0 +1,237 @@
from pytest_mock import MockerFixture
from unittest import mock
from ahriman.application.application import Application
from ahriman.core.tree import Leaf, Tree
from ahriman.models.package import Package
def test_finalize(application: Application, mocker: MockerFixture) -> None:
"""
must report and sync at the last
"""
report_mock = mocker.patch("ahriman.application.application.Application.report")
sync_mock = mocker.patch("ahriman.application.application.Application.sync")
application._finalize()
report_mock.assert_called_once()
sync_mock.assert_called_once()
def test_get_updates_all(application: Application, mocker: MockerFixture) -> None:
"""
must get updates for all
"""
updates_aur_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_aur")
updates_manual_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_manual")
application.get_updates([], no_aur=False, no_manual=False, no_vcs=False, log_fn=print)
updates_aur_mock.assert_called_with([], False)
updates_manual_mock.assert_called_once()
def test_get_updates_disabled(application: Application, mocker: MockerFixture) -> None:
"""
must get updates without anything
"""
updates_aur_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_aur")
updates_manual_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_manual")
application.get_updates([], no_aur=True, no_manual=True, no_vcs=False, log_fn=print)
updates_aur_mock.assert_not_called()
updates_manual_mock.assert_not_called()
def test_get_updates_no_aur(application: Application, mocker: MockerFixture) -> None:
"""
must get updates without aur
"""
updates_aur_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_aur")
updates_manual_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_manual")
application.get_updates([], no_aur=True, no_manual=False, no_vcs=False, log_fn=print)
updates_aur_mock.assert_not_called()
updates_manual_mock.assert_called_once()
def test_get_updates_no_manual(application: Application, mocker: MockerFixture) -> None:
"""
must get updates without manual
"""
updates_aur_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_aur")
updates_manual_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_manual")
application.get_updates([], no_aur=False, no_manual=True, no_vcs=False, log_fn=print)
updates_aur_mock.assert_called_with([], False)
updates_manual_mock.assert_not_called()
def test_get_updates_no_vcs(application: Application, mocker: MockerFixture) -> None:
"""
must get updates without VCS
"""
updates_aur_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_aur")
updates_manual_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_manual")
application.get_updates([], no_aur=False, no_manual=False, no_vcs=True, log_fn=print)
updates_aur_mock.assert_called_with([], True)
updates_manual_mock.assert_called_once()
def test_get_updates_with_filter(application: Application, mocker: MockerFixture) -> None:
"""
must get updates without VCS
"""
updates_aur_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_aur")
updates_manual_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_manual")
application.get_updates(["filter"], no_aur=False, no_manual=False, no_vcs=False, log_fn=print)
updates_aur_mock.assert_called_with(["filter"], False)
updates_manual_mock.assert_called_once()
def test_add_directory(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must add packages from directory
"""
mocker.patch("ahriman.application.application.Application._known_packages", return_value=set())
mocker.patch("pathlib.Path.is_dir", return_value=True)
iterdir_mock = mocker.patch("pathlib.Path.iterdir",
return_value=[package.filepath for package in package_ahriman.packages.values()])
move_mock = mocker.patch("shutil.move")
application.add([package_ahriman.base], False)
iterdir_mock.assert_called_once()
move_mock.assert_called_once()
def test_add_manual(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must add package from AUR
"""
mocker.patch("ahriman.application.application.Application._known_packages", return_value=set())
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
fetch_mock = mocker.patch("ahriman.core.build_tools.task.Task.fetch")
application.add([package_ahriman.base], True)
fetch_mock.assert_called_once()
def test_add_manual_with_dependencies(application: Application, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must add package from AUR with dependencies
"""
mocker.patch("ahriman.application.application.Application._known_packages", return_value=set())
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
mocker.patch("ahriman.core.build_tools.task.Task.fetch")
dependencies_mock = mocker.patch("ahriman.models.package.Package.dependencies")
application.add([package_ahriman.base], False)
dependencies_mock.assert_called_once()
def test_add_package(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must add package from archive
"""
mocker.patch("ahriman.application.application.Application._known_packages", return_value=set())
mocker.patch("pathlib.Path.is_file", return_value=True)
move_mock = mocker.patch("shutil.move")
application.add([package_ahriman.base], False)
move_mock.assert_called_once()
def test_clean_build(application: Application, mocker: MockerFixture) -> None:
"""
must clean build directory
"""
clear_mock = mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_build")
application.clean(False, True, True, True, True)
clear_mock.assert_called_once()
def test_clean_cache(application: Application, mocker: MockerFixture) -> None:
"""
must clean cache directory
"""
clear_mock = mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_cache")
application.clean(True, False, True, True, True)
clear_mock.assert_called_once()
def test_clean_chroot(application: Application, mocker: MockerFixture) -> None:
"""
must clean chroot directory
"""
clear_mock = mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_chroot")
application.clean(True, True, False, True, True)
clear_mock.assert_called_once()
def test_clean_manual(application: Application, mocker: MockerFixture) -> None:
"""
must clean manual directory
"""
clear_mock = mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_manual")
application.clean(True, True, True, False, True)
clear_mock.assert_called_once()
def test_clean_packages(application: Application, mocker: MockerFixture) -> None:
"""
must clean packages directory
"""
clear_mock = mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_packages")
application.clean(True, True, True, True, False)
clear_mock.assert_called_once()
def test_remove(application: Application, mocker: MockerFixture) -> None:
"""
must remove package
"""
executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove")
finalize_mock = mocker.patch("ahriman.application.application.Application._finalize")
application.remove([])
executor_mock.assert_called_once()
finalize_mock.assert_called_once()
def test_report(application: Application, mocker: MockerFixture) -> None:
"""
must generate report
"""
executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_report")
application.report(None)
executor_mock.assert_called_once()
def test_sync(application: Application, mocker: MockerFixture) -> None:
"""
must sync to remote
"""
executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_sync")
application.sync(None)
executor_mock.assert_called_once()
def test_update(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must process package updates
"""
paths = [package.filepath for package in package_ahriman.packages.values()]
tree = Tree([Leaf(package_ahriman, set())])
mocker.patch("ahriman.core.tree.Tree.load", return_value=tree)
mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[])
build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=paths)
update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update")
finalize_mock = mocker.patch("ahriman.application.application.Application._finalize")
application.update([package_ahriman])
build_mock.assert_called_once()
update_mock.assert_has_calls([mock.call([]), mock.call(paths)])
finalize_mock.assert_has_calls([mock.call(), mock.call()])

151
tests/ahriman/application/test_lock.py Normal file
View File

@ -0,0 +1,151 @@
import pytest
import tempfile
from pathlib import Path
from pytest_mock import MockerFixture
from unittest import mock
from ahriman.application.lock import Lock
from ahriman.core.exceptions import DuplicateRun, UnsafeRun
from ahriman.models.build_status import BuildStatusEnum
def test_enter(lock: Lock, mocker: MockerFixture) -> None:
"""
must process with context manager
"""
check_user_mock = mocker.patch("ahriman.application.lock.Lock.check_user")
remove_mock = mocker.patch("ahriman.application.lock.Lock.remove")
create_mock = mocker.patch("ahriman.application.lock.Lock.create")
update_status_mock = mocker.patch("ahriman.core.status.client.Client.update_self")
with lock:
pass
check_user_mock.assert_called_once()
remove_mock.assert_called_once()
create_mock.assert_called_once()
update_status_mock.assert_has_calls([
mock.call(BuildStatusEnum.Building),
mock.call(BuildStatusEnum.Success)
])
def test_exit_with_exception(lock: Lock, mocker: MockerFixture) -> None:
"""
must process with context manager in case if exception raised
"""
mocker.patch("ahriman.application.lock.Lock.check_user")
mocker.patch("ahriman.application.lock.Lock.remove")
mocker.patch("ahriman.application.lock.Lock.create")
update_status_mock = mocker.patch("ahriman.core.status.client.Client.update_self")
with pytest.raises(Exception):
with lock:
raise Exception()
update_status_mock.assert_has_calls([
mock.call(BuildStatusEnum.Building),
mock.call(BuildStatusEnum.Failed)
])
def test_check_user(lock: Lock, mocker: MockerFixture) -> None:
"""
must check user correctly
"""
stat = Path.cwd().stat()
mocker.patch("pathlib.Path.stat", return_value=stat)
mocker.patch("os.getuid", return_value=stat.st_uid)
lock.check_user()
def test_check_user_exception(lock: Lock, mocker: MockerFixture) -> None:
"""
must raise exception if user differs
"""
stat = Path.cwd().stat()
mocker.patch("pathlib.Path.stat")
mocker.patch("os.getuid", return_value=stat.st_uid + 1)
with pytest.raises(UnsafeRun):
lock.check_user()
def test_check_user_unsafe(lock: Lock) -> None:
"""
must skip user check if unsafe flag set
"""
lock.unsafe = True
lock.check_user()
def test_create(lock: Lock) -> None:
"""
must create lock
"""
lock.path = Path(tempfile.mktemp())
lock.create()
assert lock.path.is_file()
lock.path.unlink()
def test_create_exception(lock: Lock) -> None:
"""
must raise exception if file already exists
"""
lock.path = Path(tempfile.mktemp())
lock.path.touch()
with pytest.raises(DuplicateRun):
lock.create()
lock.path.unlink()
def test_create_skip(lock: Lock, mocker: MockerFixture) -> None:
"""
must skip creating if no file set
"""
touch_mock = mocker.patch("pathlib.Path.touch")
lock.create()
touch_mock.assert_not_called()
def test_create_unsafe(lock: Lock) -> None:
"""
must not raise exception if force flag set
"""
lock.force = True
lock.path = Path(tempfile.mktemp())
lock.path.touch()
lock.create()
lock.path.unlink()
def test_remove(lock: Lock) -> None:
"""
must remove lock file
"""
lock.path = Path(tempfile.mktemp())
lock.path.touch()
lock.remove()
assert not lock.path.is_file()
def test_remove_missing(lock: Lock) -> None:
"""
must not fail on lock removal if file is missing
"""
lock.path = Path(tempfile.mktemp())
lock.remove()
def test_remove_skip(lock: Lock, mocker: MockerFixture) -> None:
"""
must skip removal if no file set
"""
unlink_mock = mocker.patch("pathlib.Path.unlink")
lock.remove()
unlink_mock.assert_not_called()

95
tests/ahriman/conftest.py Normal file
View File

@ -0,0 +1,95 @@
import pytest
from pathlib import Path
from pytest_mock import MockerFixture
from typing import Any, Type, TypeVar
from ahriman.core.configuration import Configuration
from ahriman.core.status.watcher import Watcher
from ahriman.models.package import Package
from ahriman.models.package_desciption import PackageDescription
from ahriman.models.repository_paths import RepositoryPaths
T = TypeVar("T")
# helpers
# https://stackoverflow.com/a/21611963
@pytest.helpers.register
def anyvar(cls: Type[T], strict: bool = False) -> T:
class AnyVar(cls):
def __eq__(self, other: Any) -> bool:
return not strict or isinstance(other, cls)
return AnyVar()
# generic fixtures
@pytest.fixture
def configuration(resource_path_root: Path) -> Configuration:
path = resource_path_root / "core" / "ahriman.ini"
return Configuration.from_path(path=path, logfile=False)
@pytest.fixture
def package_ahriman(package_description_ahriman: PackageDescription) -> Package:
packages = {"ahriman": package_description_ahriman}
return Package(
base="ahriman",
version="0.12.1-1",
aur_url="https://aur.archlinux.org",
packages=packages)
@pytest.fixture
def package_python_schedule(
package_description_python_schedule: PackageDescription,
package_description_python2_schedule: PackageDescription) -> Package:
packages = {
"python-schedule": package_description_python_schedule,
"python2-schedule": package_description_python2_schedule
}
return Package(
base="python-schedule",
version="1.0.0-2",
aur_url="https://aur.archlinux.org",
packages=packages)
@pytest.fixture
def package_description_ahriman() -> PackageDescription:
return PackageDescription(
archive_size=4200,
build_date=42,
filename="ahriman-0.12.1-1-any.pkg.tar.zst",
installed_size=4200000)
@pytest.fixture
def package_description_python_schedule() -> PackageDescription:
return PackageDescription(
archive_size=4201,
build_date=421,
filename="python-schedule-1.0.0-2-any.pkg.tar.zst",
installed_size=4200001)
@pytest.fixture
def package_description_python2_schedule() -> PackageDescription:
return PackageDescription(
archive_size=4202,
build_date=422,
filename="python2-schedule-1.0.0-2-any.pkg.tar.zst",
installed_size=4200002)
@pytest.fixture
def repository_paths(configuration: Configuration) -> RepositoryPaths:
return RepositoryPaths(
architecture="x86_64",
root=configuration.getpath("repository", "root"))
@pytest.fixture
def watcher(configuration: Configuration, mocker: MockerFixture) -> Watcher:
mocker.patch("pathlib.Path.mkdir")
return Watcher("x86_64", configuration)

10
tests/ahriman/core/alpm/test_pacman.py Normal file
View File

@ -0,0 +1,10 @@
from ahriman.core.alpm.pacman import Pacman
def test_all_packages(pacman: Pacman) -> None:
"""
package list must not be empty
"""
packages = pacman.all_packages()
assert packages
assert "pacman" in packages

47
tests/ahriman/core/alpm/test_repo.py Normal file
View File

@ -0,0 +1,47 @@
import pytest
from pathlib import Path
from pytest_mock import MockerFixture
from ahriman.core.alpm.repo import Repo
def test_repo_path(repo: Repo) -> None:
"""
name must be something like archive name
"""
assert repo.repo_path.name.endswith("db.tar.gz")
def test_repo_add(repo: Repo, mocker: MockerFixture) -> None:
"""
must call repo-add on package addition
"""
check_output_mock = mocker.patch("ahriman.core.alpm.repo.Repo._check_output")
repo.add(Path("path"))
check_output_mock.assert_called_once()
assert check_output_mock.call_args[0][0] == "repo-add"
def test_repo_remove(repo: Repo, mocker: MockerFixture) -> None:
"""
must call repo-remove on package addition
"""
mocker.patch("pathlib.Path.glob", return_value=[])
check_output_mock = mocker.patch("ahriman.core.alpm.repo.Repo._check_output")
repo.remove("package", Path("package.pkg.tar.xz"))
check_output_mock.assert_called_once()
assert check_output_mock.call_args[0][0] == "repo-remove"
def test_repo_remove_fail_no_file(repo: Repo, mocker: MockerFixture) -> None:
"""
must fail on missing file
"""
mocker.patch("pathlib.Path.glob", return_value=[Path("package.pkg.tar.xz")])
mocker.patch("pathlib.Path.unlink", side_effect=FileNotFoundError())
with pytest.raises(FileNotFoundError):
repo.remove("package", Path("package.pkg.tar.xz"))

64
tests/ahriman/core/build_tools/test_task.py Normal file
View File

@ -0,0 +1,64 @@
import pytest
import shutil
from pathlib import Path
from pytest_mock import MockerFixture
from unittest import mock
from ahriman.core.build_tools.task import Task
def test_fetch_existing(mocker: MockerFixture) -> None:
"""
must fetch new package via clone command
"""
mocker.patch("pathlib.Path.is_dir", return_value=True)
check_output_mock = mocker.patch("ahriman.core.build_tools.task.Task._check_output")
local = Path("local")
Task.fetch(local, "remote", "master")
check_output_mock.assert_has_calls([
mock.call("git", "fetch", "origin", "master",
exception=pytest.helpers.anyvar(int),
cwd=local, logger=pytest.helpers.anyvar(int)),
mock.call("git", "checkout", "--force", "master",
exception=pytest.helpers.anyvar(int),
cwd=local, logger=pytest.helpers.anyvar(int)),
mock.call("git", "reset", "--hard", "origin/master",
exception=pytest.helpers.anyvar(int),
cwd=local, logger=pytest.helpers.anyvar(int))
])
def test_fetch_new(mocker: MockerFixture) -> None:
"""
must fetch new package via clone command
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
check_output_mock = mocker.patch("ahriman.core.build_tools.task.Task._check_output")
local = Path("local")
Task.fetch(local, "remote", "master")
check_output_mock.assert_has_calls([
mock.call("git", "clone", "remote", str(local),
exception=pytest.helpers.anyvar(int),
logger=pytest.helpers.anyvar(int)),
mock.call("git", "checkout", "--force", "master",
exception=pytest.helpers.anyvar(int),
cwd=local, logger=pytest.helpers.anyvar(int)),
mock.call("git", "reset", "--hard", "origin/master",
exception=pytest.helpers.anyvar(int),
cwd=local, logger=pytest.helpers.anyvar(int))
])
def test_init_with_cache(task_ahriman: Task, mocker: MockerFixture) -> None:
"""
must copy tree instead of fetch
"""
mocker.patch("pathlib.Path.is_dir", return_value=True)
mocker.patch("ahriman.core.build_tools.task.Task.fetch")
copytree_mock = mocker.patch("shutil.copytree")
task_ahriman.init(None)
copytree_mock.assert_called_once()

34
tests/ahriman/core/conftest.py Normal file
View File

@ -0,0 +1,34 @@
import pytest
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.repo import Repo
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
from ahriman.core.tree import Leaf
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
@pytest.fixture
def leaf_ahriman(package_ahriman: Package) -> Leaf:
return Leaf(package_ahriman, set())
@pytest.fixture
def leaf_python_schedule(package_python_schedule: Package) -> Leaf:
return Leaf(package_python_schedule, set())
@pytest.fixture
def pacman(configuration: Configuration) -> Pacman:
return Pacman(configuration)
@pytest.fixture
def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Repo:
return Repo(configuration.get("repository", "name"), repository_paths, [])
@pytest.fixture
def task_ahriman(package_ahriman: Package, configuration: Configuration, repository_paths: RepositoryPaths) -> Task:
return Task(package_ahriman, "x86_64", configuration, repository_paths)

0
tests/ahriman/core/report/test_html.py Normal file
View File

0
tests/ahriman/core/report/test_report.py Normal file
View File

49
tests/ahriman/core/repository/conftest.py Normal file
View File

@ -0,0 +1,49 @@
import pytest
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.repository.executor import Executor
from ahriman.core.repository.properties import Properties
from ahriman.core.repository.repository import Repository
from ahriman.core.repository.update_handler import UpdateHandler
@pytest.fixture
def cleaner(configuration: Configuration, mocker: MockerFixture) -> Cleaner:
mocker.patch("pathlib.Path.mkdir")
return Cleaner("x86_64", configuration)
@pytest.fixture
def executor(configuration: Configuration, mocker: MockerFixture) -> Executor:
mocker.patch("pathlib.Path.mkdir")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_build")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_cache")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_chroot")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_manual")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_packages")
return Executor("x86_64", configuration)
@pytest.fixture
def repository(configuration: Configuration, mocker: MockerFixture) -> Repository:
mocker.patch("pathlib.Path.mkdir")
return Repository("x86_64", configuration)
@pytest.fixture
def properties(configuration: Configuration) -> Properties:
return Properties("x86_64", configuration)
@pytest.fixture
def update_handler(configuration: Configuration, mocker: MockerFixture) -> UpdateHandler:
mocker.patch("pathlib.Path.mkdir")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_build")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_cache")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_chroot")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_manual")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_packages")
return UpdateHandler("x86_64", configuration)

68
tests/ahriman/core/repository/test_cleaner.py Normal file
View File

@ -0,0 +1,68 @@
import shutil
from pathlib import Path
from pytest_mock import MockerFixture
from unittest import mock
from ahriman.core.repository.cleaner import Cleaner
def _mock_clear(mocker: MockerFixture) -> None:
mocker.patch("pathlib.Path.iterdir", return_value=[Path("a"), Path("b"), Path("c")])
mocker.patch("shutil.rmtree")
def _mock_clear_check() -> None:
shutil.rmtree.assert_has_calls([
mock.call(Path("a")),
mock.call(Path("b")),
mock.call(Path("c"))
])
def test_clear_build(cleaner: Cleaner, mocker: MockerFixture) -> None:
"""
must remove directories with sources
"""
_mock_clear(mocker)
cleaner.clear_build()
_mock_clear_check()
def test_clear_cache(cleaner: Cleaner, mocker: MockerFixture) -> None:
"""
must remove every cached sources
"""
_mock_clear(mocker)
cleaner.clear_cache()
_mock_clear_check()
def test_clear_chroot(cleaner: Cleaner, mocker: MockerFixture) -> None:
"""
must clear chroot
"""
_mock_clear(mocker)
cleaner.clear_chroot()
_mock_clear_check()
def test_clear_manual(cleaner: Cleaner, mocker: MockerFixture) -> None:
"""
must clear directory with manual packages
"""
_mock_clear(mocker)
cleaner.clear_manual()
_mock_clear_check()
def test_clear_packages(cleaner: Cleaner, mocker: MockerFixture) -> None:
"""
must delete built packages
"""
mocker.patch("ahriman.core.repository.cleaner.Cleaner.packages_built",
return_value=[Path("a"), Path("b"), Path("c")])
mocker.patch("pathlib.Path.unlink")
cleaner.clear_packages()
Path.unlink.assert_has_calls([mock.call(), mock.call(), mock.call()])

188
tests/ahriman/core/repository/test_executor.py Normal file
View File

@ -0,0 +1,188 @@
from pathlib import Path
from pytest_mock import MockerFixture
from unittest import mock
from ahriman.core.repository.executor import Executor
from ahriman.models.package import Package
def test_process_build(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must run build process
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages_built", return_value=[package_ahriman])
mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)])
mocker.patch("ahriman.core.build_tools.task.Task.init")
move_mock = mocker.patch("shutil.move")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_building")
# must return list of built packages
assert executor.process_build([package_ahriman]) == [package_ahriman]
# must move files (once)
move_mock.assert_called_once()
# must update status
status_client_mock.assert_called_once()
# must clear directory
from ahriman.core.repository.cleaner import Cleaner
Cleaner.clear_build.assert_called_once()
def test_process_build_failure(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must run correct process failed builds
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages_built")
mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)])
mocker.patch("ahriman.core.build_tools.task.Task.init")
mocker.patch("shutil.move", side_effect=Exception())
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_failed")
executor.process_build([package_ahriman])
status_client_mock.assert_called_once()
def test_process_remove_base(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must run remove process for whole base
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.remove")
executor.process_remove([package_ahriman.base])
# must remove via alpm wrapper
repo_remove_mock.assert_called_once()
# must update status
status_client_mock.assert_called_once()
def test_process_remove_base_multiple(executor: Executor, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must run remove process for whole base with multiple packages
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule])
repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.remove")
executor.process_remove([package_python_schedule.base])
# must remove via alpm wrapper
repo_remove_mock.assert_has_calls([
mock.call(package, Path(props.filename))
for package, props in package_python_schedule.packages.items()
], any_order=True)
# must update status
status_client_mock.assert_called_once()
def test_process_remove_base_single(executor: Executor, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must run remove process for single package in base
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule])
repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.remove")
executor.process_remove(["python2-schedule"])
# must remove via alpm wrapper
repo_remove_mock.assert_called_once()
# must not update status
status_client_mock.assert_not_called()
def test_process_remove_nothing(executor: Executor, package_ahriman: Package, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must not remove anything if it was not requested
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove")
executor.process_remove([package_python_schedule.base])
repo_remove_mock.assert_not_called()
def test_process_report_auto(executor: Executor, mocker: MockerFixture) -> None:
"""
must process report in auto mode if no targets supplied
"""
config_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist")
executor.process_report(None)
config_getlist_mock.assert_called_once()
def test_process_sync_auto(executor: Executor, mocker: MockerFixture) -> None:
"""
must process sync in auto mode if no targets supplied
"""
config_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist")
executor.process_sync(None)
config_getlist_mock.assert_called_once()
def test_process_update(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must run update process
"""
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
move_mock = mocker.patch("shutil.move")
repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add")
sign_package_mock = mocker.patch("ahriman.core.sign.gpg.GPG.sign_package", side_effect=lambda fn, _: [fn])
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_success")
# must return complete
assert executor.process_update([Path(package.filename) for package in package_ahriman.packages.values()])
# must move files (once)
move_mock.assert_called_once()
# must sign package
sign_package_mock.assert_called_once()
# must add package
repo_add_mock.assert_called_once()
# must update status
status_client_mock.assert_called_once()
# must clear directory
from ahriman.core.repository.cleaner import Cleaner
Cleaner.clear_packages.assert_called_once()
def test_process_update_group(executor: Executor, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must group single packages under one base
"""
mocker.patch("shutil.move")
mocker.patch("ahriman.models.package.Package.load", return_value=package_python_schedule)
repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_success")
executor.process_update([Path(package.filename) for package in package_python_schedule.packages.values()])
repo_add_mock.assert_has_calls([
mock.call(executor.paths.repository / package.filename)
for package in package_python_schedule.packages.values()
], any_order=True)
status_client_mock.assert_called_with(package_python_schedule)
def test_process_update_failed(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must process update for failed package
"""
mocker.patch("shutil.move", side_effect=Exception())
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_failed")
executor.process_update([Path(package.filename) for package in package_ahriman.packages.values()])
status_client_mock.assert_called_once()
def test_process_update_failed_on_load(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must process update even with failed package load
"""
mocker.patch("shutil.move")
mocker.patch("ahriman.models.package.Package.load", side_effect=Exception())
assert executor.process_update([Path(package.filename) for package in package_ahriman.packages.values()])

14
tests/ahriman/core/repository/test_properties.py Normal file
View File

@ -0,0 +1,14 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.repository.properties import Properties
def test_create_tree_on_load(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must create tree on load
"""
create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
Properties("x86_64", configuration)
create_tree_mock.assert_called_once()

33
tests/ahriman/core/repository/test_repository.py Normal file
View File

@ -0,0 +1,33 @@
from pathlib import Path
from pytest_mock import MockerFixture
from ahriman.core.repository.repository import Repository
from ahriman.models.package import Package
def test_packages(package_ahriman: Package, package_python_schedule: Package,
repository: Repository, mocker: MockerFixture) -> None:
"""
must return all packages grouped by package base
"""
single_packages = [
Package(base=package_python_schedule.base,
version=package_python_schedule.version,
aur_url=package_python_schedule.aur_url,
packages={package: props})
for package, props in package_python_schedule.packages.items()
] + [package_ahriman]
mocker.patch("pathlib.Path.iterdir",
return_value=[Path("a.pkg.tar.xz"), Path("b.pkg.tar.xz"), Path("c.pkg.tar.xz")])
mocker.patch("ahriman.models.package.Package.load", side_effect=single_packages)
packages = repository.packages()
assert len(packages) == 2
assert {package.base for package in packages} == {package_ahriman.base, package_python_schedule.base}
archives = sum([list(package.packages.keys()) for package in packages], start=[])
assert len(archives) == 3
expected = set(package_ahriman.packages.keys())
expected.update(package_python_schedule.packages.keys())
assert set(archives) == expected

124
tests/ahriman/core/repository/test_update_handler.py Normal file
View File

@ -0,0 +1,124 @@
from pytest_mock import MockerFixture
from ahriman.core.repository.update_handler import UpdateHandler
from ahriman.models.package import Package
def test_updates_aur(update_handler: UpdateHandler, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must provide updates with status updates
"""
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
mocker.patch("ahriman.models.package.Package.is_outdated", return_value=True)
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_pending")
assert update_handler.updates_aur([], False) == [package_ahriman]
status_client_mock.assert_called_once()
def test_updates_aur_failed(update_handler: UpdateHandler, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must update status via client for failed load
"""
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
mocker.patch("ahriman.models.package.Package.load", side_effect=Exception())
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_failed")
update_handler.updates_aur([], False)
status_client_mock.assert_called_once()
def test_updates_aur_filter(update_handler: UpdateHandler, package_ahriman: Package, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must provide updates only for filtered packages
"""
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages",
return_value=[package_ahriman, package_python_schedule])
mocker.patch("ahriman.models.package.Package.is_outdated", return_value=True)
package_load_mock = mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
assert update_handler.updates_aur([package_ahriman.base], False) == [package_ahriman]
package_load_mock.assert_called_once()
def test_updates_aur_ignore(update_handler: UpdateHandler, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must skip ignore packages
"""
mocker.patch("ahriman.core.configuration.Configuration.getlist", return_value=[package_ahriman.base])
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
package_load_mock = mocker.patch("ahriman.models.package.Package.load")
update_handler.updates_aur([], False)
package_load_mock.assert_not_called()
def test_updates_aur_ignore_vcs(update_handler: UpdateHandler, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must skip VCS packages check if requested
"""
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
mocker.patch("ahriman.models.package.Package.is_vcs", return_value=True)
package_is_outdated_mock = mocker.patch("ahriman.models.package.Package.is_outdated")
update_handler.updates_aur([], True)
package_is_outdated_mock.assert_not_called()
def test_updates_manual_clear(update_handler: UpdateHandler, mocker: MockerFixture) -> None:
"""
requesting manual updates must clear packages directory
"""
mocker.patch("pathlib.Path.iterdir", return_value=[])
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages")
update_handler.updates_manual()
from ahriman.core.repository.cleaner import Cleaner
Cleaner.clear_manual.assert_called_once()
def test_updates_manual_status_known(update_handler: UpdateHandler, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must create record for known package via reporter
"""
mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.base])
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_pending")
update_handler.updates_manual()
status_client_mock.assert_called_once()
def test_updates_manual_status_unknown(update_handler: UpdateHandler, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must create record for unknown package via reporter
"""
mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.base])
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[])
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_unknown")
update_handler.updates_manual()
status_client_mock.assert_called_once()
def test_updates_manual_with_failures(update_handler: UpdateHandler, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must process through the packages with failure
"""
mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.base])
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[])
mocker.patch("ahriman.models.package.Package.load", side_effect=Exception())
assert update_handler.updates_manual() == []

Some files were not shown because too many files have changed in this diff Show More