arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 07:17:17 +00:00
Code Issues Projects Releases Wiki Activity

web service improvements

Browse Source 
* load and save web service state to cache file
* disable web reporting to self
* restore console handler settings
* allow to redirect logs to stderr
* verbose http error logging
* update package status by group, not by single package
* split Repository class to several traits
* move json generators/readers to dataclasses
This commit is contained in:
Evgenii Alekseev 2021-03-20 18:01:57 +03:00
parent 3e2fb7b4e6
commit 413d3b7509
16 changed files with 515 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package/archlinux/PKGBUILD
View File

@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
'ahriman.sysusers'
'ahriman.tmpfiles')
sha512sums=('54286cfd1c9b03e7adfa639b976ace233e4e3ea8d2a2cbd11c22fc43eda60906e1d3b795e1505b40e41171948ba95d6591a4f7c328146200f4622a8ed657e8a5'
sha512sums=('d1a88fc3c5c14258cd0f84c815ebd254749ca8f9ba4dafe4a7385ac2eafc27cc552812a1951ccf1741ddc7ac7d4b2d2ebbe15960b0796e4f37653184adb86e30'
'13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
'55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
backup=('etc/ahriman.ini'

10
package/etc/ahriman.ini.d/logging.ini
View File

@ -2,11 +2,17 @@
keys = root,builder,build_details,http
[handlers]
keys = build_file_handler,file_handler,http_handler
keys = console_handler,build_file_handler,file_handler,http_handler
[formatters]
keys = generic_format
[handler_console_handler]
class = StreamHandler
level = DEBUG
formatter = generic_format
args = (sys.stderr,)
[handler_file_handler]
class = logging.handlers.RotatingFileHandler
level = DEBUG
@ -26,7 +32,7 @@ formatter = generic_format
args = ('/var/log/ahriman/http.log', 'a', 20971520, 20)
[formatter_generic_format]
format = %(asctime)s : %(levelname)s : %(funcName)s : %(message)s
format = [%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s
datefmt =
[logger_root]

12
src/ahriman/application/ahriman.py
View File

@ -39,7 +39,7 @@ def _call(args: argparse.Namespace, architecture: str, config: Configuration) ->
:return: True on success, False otherwise
'''
try:
with Lock(args.lock, architecture, config):
with Lock(args, architecture, config):
args.fn(args, architecture, config)
return True
except Exception:
@ -169,6 +169,8 @@ if __name__ == '__main__':
parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini')
parser.add_argument('--force', help='force run, remove file lock', action='store_true')
parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
parser.add_argument('--no-log', help='redirect all log messages to stderr', action='store_true')
parser.add_argument('--no-report', help='force disable reporting to web service', action='store_true')
parser.add_argument('--unsafe', help='allow to run ahriman as non-ahriman user', action='store_true')
parser.add_argument('-v', '--version', action='version', version=version.__version__)
subparsers = parser.add_subparsers(title='command')
@ -178,7 +180,7 @@ if __name__ == '__main__':
add_parser.add_argument('--without-dependencies', help='do not add dependencies', action='store_true')
add_parser.set_defaults(fn=add)
check_parser = subparsers.add_parser('check', description='check for updates')
check_parser = subparsers.add_parser('check', description='check for updates. Same as update --dry-run --no-manual')
check_parser.add_argument('package', help='filter check by packages', nargs='*')
check_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=True)
@ -216,20 +218,20 @@ if __name__ == '__main__':
update_parser.add_argument('package', help='filter check by packages', nargs='*')
update_parser.add_argument(
'--dry-run', help='just perform check for updates, same as check command', action='store_true')
update_parser.add_argument('--no-aur', help='do not check for AUR updates', action='store_true')
update_parser.add_argument('--no-aur', help='do not check for AUR updates. Implies --no-vcs', action='store_true')
update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true')
update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
update_parser.set_defaults(fn=update)
web_parser = subparsers.add_parser('web', description='start web server')
web_parser.set_defaults(fn=web, lock=None)
web_parser.set_defaults(fn=web, lock=None, no_report=True)
cmd_args = parser.parse_args()
if 'fn' not in cmd_args:
parser.print_help()
sys.exit(1)
configuration = Configuration.from_path(cmd_args.config)
configuration = Configuration.from_path(cmd_args.config, not cmd_args.no_log)
with Pool(len(cmd_args.architecture)) as pool:
result = pool.starmap(
_call, [(cmd_args, architecture, configuration) for architecture in cmd_args.architecture])

2
src/ahriman/application/application.py
View File

@ -25,7 +25,7 @@ from typing import Callable, Iterable, List, Optional, Set
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Repository
from ahriman.repository.repository import Repository
from ahriman.core.tree import Tree
from ahriman.models.package import Package

2
src/ahriman/application/lock.py
View File

@ -53,7 +53,7 @@ class Lock:
self.unsafe = args.unsafe
self.root = config.get('repository', 'root')
self.reporter = Client.load(architecture, config)
self.reporter = Client() if args.no_report else Client.load(architecture, config)
def __enter__(self) -> Lock:
'''

23
src/ahriman/core/configuration.py
View File

@ -37,7 +37,7 @@ class Configuration(configparser.RawConfigParser):
:cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump)
'''
DEFAULT_LOG_FORMAT = '%(asctime)s : %(levelname)s : %(funcName)s : %(message)s'
DEFAULT_LOG_FORMAT = '[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s'
DEFAULT_LOG_LEVEL = logging.DEBUG
STATIC_SECTIONS = ['alpm', 'report', 'repository', 'settings', 'upload']
@ -45,7 +45,7 @@ class Configuration(configparser.RawConfigParser):
def __init__(self) -> None:
'''
default constructor
default constructor. In the most cases must not be called directly
'''
configparser.RawConfigParser.__init__(self, allow_no_value=True)
self.path: Optional[str] = None
@ -58,15 +58,16 @@ class Configuration(configparser.RawConfigParser):
return self.get('settings', 'include')
@classmethod
def from_path(cls: Type[Configuration], path: str) -> Configuration:
def from_path(cls: Type[Configuration], path: str, logfile: bool) -> Configuration:
'''
constructor with full object initialization
:param path: path to root configuration file
:param logfile: use log file to output messages
:return: configuration instance
'''
config = cls()
config.load(path)
config.load_logging()
config.load_logging(logfile)
return config
def dump(self, architecture: str) -> Dict[str, Dict[str, str]]:
@ -129,13 +130,23 @@ class Configuration(configparser.RawConfigParser):
except (FileNotFoundError, configparser.NoOptionError):
pass
def load_logging(self) -> None:
def load_logging(self, logfile: bool) -> None:
'''
setup logging settings from configuration
:param logfile: use log file to output messages
'''
def file_logger() -> None:
try:
fileConfig(self.get('settings', 'logging'))
except PermissionError:
console_logger()
logging.error('could not create logfile, fallback to stderr', exc_info=True)
def console_logger() -> None:
logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT,
level=Configuration.DEFAULT_LOG_LEVEL)
logging.error('could not create logfile, fallback to stderr', exc_info=True)
if logfile:
file_logger()
else:
console_logger()

297
src/ahriman/core/repository.py
View File

@ -1,297 +0,0 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import os
import shutil
from typing import Dict, Iterable, List, Optional
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.repo import Repo
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
from ahriman.core.report.report import Report
from ahriman.core.sign.gpg import GPG
from ahriman.core.upload.uploader import Uploader
from ahriman.core.util import package_like
from ahriman.core.watcher.client import Client
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
class Repository:
'''
base repository control class
:ivar architecture: repository architecture
:ivar aur_url: base AUR url
:ivar config: configuration instance
:ivar logger: class logger
:ivar name: repository name
:ivar pacman: alpm wrapper instance
:ivar paths: repository paths instance
:ivar repo: repo commands wrapper instance
:ivar reporter: build status reporter instance
:ivar sign: GPG wrapper instance
'''
def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('builder')
self.architecture = architecture
self.config = config
self.aur_url = config.get('alpm', 'aur_url')
self.name = config.get('repository', 'name')
self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
self.paths.create_tree()
self.pacman = Pacman(config)
self.sign = GPG(architecture, config)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client.load(architecture, config)
def clear_build(self) -> None:
'''
clear sources directory
'''
for package in os.listdir(self.paths.sources):
shutil.rmtree(os.path.join(self.paths.sources, package))
def clear_cache(self) -> None:
'''
clear cache directory
'''
for package in os.listdir(self.paths.cache):
shutil.rmtree(os.path.join(self.paths.cache, package))
def clear_chroot(self) -> None:
'''
clear cache directory. Warning: this method is architecture independent and will clear every chroot
'''
for chroot in os.listdir(self.paths.chroot):
shutil.rmtree(os.path.join(self.paths.chroot, chroot))
def clear_manual(self) -> None:
'''
clear directory with manual package updates
'''
for package in os.listdir(self.paths.manual):
shutil.rmtree(os.path.join(self.paths.manual, package))
def clear_packages(self) -> None:
'''
clear directory with built packages (NOT repository itself)
'''
for package in self.packages_built():
os.remove(package)
def packages(self) -> List[Package]:
'''
generate list of repository packages
:return: list of packages properties
'''
result: Dict[str, Package] = {}
for fn in os.listdir(self.paths.repository):
if not package_like(fn):
continue
full_path = os.path.join(self.paths.repository, fn)
try:
local = Package.load(full_path, self.pacman, self.aur_url)
result.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
return list(result.values())
def packages_built(self) -> List[str]:
'''
get list of files in built packages directory
:return: list of filenames from the directory
'''
return [
os.path.join(self.paths.packages, fn)
for fn in os.listdir(self.paths.packages)
]
def process_build(self, updates: Iterable[Package]) -> List[str]:
'''
build packages
:param updates: list of packages properties to build
:return: `packages_built`
'''
def build_single(package: Package) -> None:
self.reporter.set_building(package.base)
task = Task(package, self.architecture, self.config, self.paths)
task.init()
built = task.build()
for src in built:
dst = os.path.join(self.paths.packages, os.path.basename(src))
shutil.move(src, dst)
for package in updates:
try:
build_single(package)
except Exception:
self.reporter.set_failed(package.base)
self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
continue
self.clear_build()
return self.packages_built()
def process_remove(self, packages: Iterable[str]) -> str:
'''
remove packages from list
:param packages: list of package names or bases to rmeove
:return: path to repository database
'''
def remove_single(package: str) -> None:
try:
self.repo.remove(package)
except Exception:
self.logger.exception(f'could not remove {package}', exc_info=True)
requested = set(packages)
for local in self.packages():
if local.base in packages:
to_remove = set(local.packages.keys())
self.reporter.remove(local.base) # we only update status page in case of base removal
elif requested.intersection(local.packages.keys()):
to_remove = requested.intersection(local.packages.keys())
else:
to_remove = set()
for package in to_remove:
remove_single(package)
return self.repo.repo_path
def process_report(self, targets: Optional[Iterable[str]]) -> None:
'''
generate reports
:param targets: list of targets to generate reports. Configuration option will be used if it is not set
'''
if targets is None:
targets = self.config.getlist('report', 'target')
for target in targets:
Report.run(self.architecture, self.config, target, self.packages())
def process_sync(self, targets: Optional[Iterable[str]]) -> None:
'''
process synchronization to remote servers
:param targets: list of targets to sync. Configuration option will be used if it is not set
'''
if targets is None:
targets = self.config.getlist('upload', 'target')
for target in targets:
Uploader.run(self.architecture, self.config, target, self.paths.repository)
def process_update(self, packages: Iterable[str]) -> str:
'''
sign packages, add them to repository and update repository database
:param packages: list of filenames to run
:return: path to repository database
'''
def update_single(fn: Optional[str], base: str) -> None:
if fn is None:
self.logger.warning(f'received empty package name for base {base}')
return # suppress type checking, it never can be none actually
files = self.sign.sign_package(fn, base)
for src in files:
dst = os.path.join(self.paths.repository, os.path.basename(src))
shutil.move(src, dst)
package_fn = os.path.join(self.paths.repository, os.path.basename(fn))
self.repo.add(package_fn)
# we are iterating over bases, not single packages
updates: Dict[str, Package] = {}
for fn in packages:
local = Package.load(fn, self.pacman, self.aur_url)
updates.setdefault(local.base, local).packages.update(local.packages)
for local in updates.values():
try:
for description in local.packages.values():
update_single(description.filename, local.base)
self.reporter.set_success(local)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception(f'could not process {local.base}', exc_info=True)
self.clear_packages()
return self.repo.repo_path
def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
'''
check AUR for updates
:param filter_packages: do not check every package just specified in the list
:param no_vcs: do not check VCS packages
:return: list of packages which are out-of-dated
'''
result: List[Package] = []
build_section = self.config.get_section_name('build', self.architecture)
ignore_list = self.config.getlist(build_section, 'ignore_packages')
for local in self.packages():
if local.base in ignore_list:
continue
if local.is_vcs and no_vcs:
continue
if filter_packages and local.base not in filter_packages:
continue
try:
remote = Package.load(local.base, self.pacman, self.aur_url)
if local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
continue
return result
def updates_manual(self) -> List[Package]:
'''
check for packages for which manual update has been requested
:return: list of packages which are out-of-dated
'''
result: List[Package] = []
known_bases = {package.base for package in self.packages()}
for fn in os.listdir(self.paths.manual):
try:
local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
result.append(local)
if local.base not in known_bases:
self.reporter.set_unknown(local)
else:
self.reporter.set_pending(local.base)
except Exception:
self.logger.exception(f'could not add package from {fn}', exc_info=True)
self.clear_manual()
return result

7
src/ahriman/core/watcher/watcher.py
View File

@ -24,7 +24,7 @@ import os
from typing import Any, Dict, List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Repository
from ahriman.repository.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.package import Package
@ -58,7 +58,7 @@ class Watcher:
'''
:return: path to dump with json cache
'''
return os.path.join(self.repository.paths.root, 'cache.json')
return os.path.join(self.repository.paths.root, 'status_cache.json')
@property
def packages(self) -> List[Tuple[Package, BuildStatus]]:
@ -99,8 +99,11 @@ class Watcher:
} for package, status in self.packages
]
}
try:
with open(self.cache_path, 'w') as cache:
json.dump(dump, cache)
except Exception:
self.logger.exception('cannot dump cache', exc_info=True)
def get(self, base: str) -> Tuple[Package, BuildStatus]:
'''

12
src/ahriman/core/watcher/web_client.py
View File

@ -20,8 +20,6 @@
import logging
import requests
from dataclasses import asdict
from ahriman.core.watcher.client import Client
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
@ -68,12 +66,14 @@ class WebClient(Client):
'''
payload = {
'status': status.value,
'package': asdict(package)
'package': package.view()
}
try:
response = requests.post(self._package_url(package.base), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not add {package.base}: {e.response.text}', exc_info=True)
except Exception:
self.logger.exception(f'could not add {package.base}', exc_info=True)
@ -85,6 +85,8 @@ class WebClient(Client):
try:
response = requests.delete(self._package_url(base))
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not delete {base}: {e.response.text}', exc_info=True)
except Exception:
self.logger.exception(f'could not delete {base}', exc_info=True)
@ -99,6 +101,8 @@ class WebClient(Client):
try:
response = requests.post(self._package_url(base), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not update {base}: {e.response.text}', exc_info=True)
except Exception:
self.logger.exception(f'could not update {base}', exc_info=True)
@ -112,5 +116,7 @@ class WebClient(Client):
try:
response = requests.post(self._ahriman_url(), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception(f'could not update service status: {e.response.text}', exc_info=True)
except Exception:
self.logger.exception('could not update service status', exc_info=True)

2
src/ahriman/models/package.py
View File

@ -126,7 +126,7 @@ class Package:
'''
packages = {
key: PackageDescription(**value)
for key, value in dump.get('packages', {})
for key, value in dump.get('packages', {}).items()
}
return Package(
base=dump['base'],

19
src/ahriman/repository/__init__.py Normal file
View File

@ -0,0 +1,19 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

78
src/ahriman/repository/cleaner.py Normal file
View File

@ -0,0 +1,78 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import shutil
from typing import List
from ahriman.repository.properties import Properties
class Cleaner(Properties):
'''
trait to clean common repository objects
'''
def packages_built(self) -> List[str]:
'''
get list of files in built packages directory
:return: list of filenames from the directory
'''
raise NotImplementedError
def clear_build(self) -> None:
'''
clear sources directory
'''
self.logger.info('clear package sources directory')
for package in os.listdir(self.paths.sources):
shutil.rmtree(os.path.join(self.paths.sources, package))
def clear_cache(self) -> None:
'''
clear cache directory
'''
self.logger.info('clear packages sources cache directory')
for package in os.listdir(self.paths.cache):
shutil.rmtree(os.path.join(self.paths.cache, package))
def clear_chroot(self) -> None:
'''
clear cache directory. Warning: this method is architecture independent and will clear every chroot
'''
self.logger.info('clear build chroot directory')
for chroot in os.listdir(self.paths.chroot):
shutil.rmtree(os.path.join(self.paths.chroot, chroot))
def clear_manual(self) -> None:
'''
clear directory with manual package updates
'''
self.logger.info('clear manual packages')
for package in os.listdir(self.paths.manual):
shutil.rmtree(os.path.join(self.paths.manual, package))
def clear_packages(self) -> None:
'''
clear directory with built packages (NOT repository itself)
'''
self.logger.info('clear built packages directory')
for package in self.packages_built():
os.remove(package)

151
src/ahriman/repository/executor.py Normal file
View File

@ -0,0 +1,151 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import shutil
from typing import Dict, Iterable, List, Optional
from ahriman.core.build_tools.task import Task
from ahriman.core.report.report import Report
from ahriman.core.upload.uploader import Uploader
from ahriman.models.package import Package
from ahriman.repository.cleaner import Cleaner
class Executor(Cleaner):
'''
trait for common repository update processes
'''
def packages(self) -> List[Package]:
'''
generate list of repository packages
:return: list of packages properties
'''
raise NotImplementedError
def process_build(self, updates: Iterable[Package]) -> List[str]:
'''
build packages
:param updates: list of packages properties to build
:return: `packages_built`
'''
def build_single(package: Package) -> None:
self.reporter.set_building(package.base)
task = Task(package, self.architecture, self.config, self.paths)
task.init()
built = task.build()
for src in built:
dst = os.path.join(self.paths.packages, os.path.basename(src))
shutil.move(src, dst)
for package in updates:
try:
build_single(package)
except Exception:
self.reporter.set_failed(package.base)
self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
continue
self.clear_build()
return self.packages_built()
def process_remove(self, packages: Iterable[str]) -> str:
'''
remove packages from list
:param packages: list of package names or bases to rmeove
:return: path to repository database
'''
def remove_single(package: str) -> None:
try:
self.repo.remove(package)
except Exception:
self.logger.exception(f'could not remove {package}', exc_info=True)
requested = set(packages)
for local in self.packages():
if local.base in packages:
to_remove = set(local.packages.keys())
self.reporter.remove(local.base) # we only update status page in case of base removal
elif requested.intersection(local.packages.keys()):
to_remove = requested.intersection(local.packages.keys())
else:
to_remove = set()
for package in to_remove:
remove_single(package)
return self.repo.repo_path
def process_report(self, targets: Optional[Iterable[str]]) -> None:
'''
generate reports
:param targets: list of targets to generate reports. Configuration option will be used if it is not set
'''
if targets is None:
targets = self.config.getlist('report', 'target')
for target in targets:
Report.run(self.architecture, self.config, target, self.packages())
def process_sync(self, targets: Optional[Iterable[str]]) -> None:
'''
process synchronization to remote servers
:param targets: list of targets to sync. Configuration option will be used if it is not set
'''
if targets is None:
targets = self.config.getlist('upload', 'target')
for target in targets:
Uploader.run(self.architecture, self.config, target, self.paths.repository)
def process_update(self, packages: Iterable[str]) -> str:
'''
sign packages, add them to repository and update repository database
:param packages: list of filenames to run
:return: path to repository database
'''
def update_single(fn: Optional[str], base: str) -> None:
if fn is None:
self.logger.warning(f'received empty package name for base {base}')
return # suppress type checking, it never can be none actually
# in theory it might be NOT packages directory, but we suppose it is
full_path = os.path.join(self.paths.packages, fn)
files = self.sign.sign_package(full_path, base)
for src in files:
dst = os.path.join(self.paths.repository, os.path.basename(src))
shutil.move(src, dst)
package_path = os.path.join(self.paths.repository, fn)
self.repo.add(package_path)
# we are iterating over bases, not single packages
updates: Dict[str, Package] = {}
for fn in packages:
local = Package.load(fn, self.pacman, self.aur_url)
updates.setdefault(local.base, local).packages.update(local.packages)
for local in updates.values():
try:
for description in local.packages.values():
update_single(description.filename, local.base)
self.reporter.set_success(local)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception(f'could not process {local.base}', exc_info=True)
self.clear_packages()
return self.repo.repo_path

59
src/ahriman/repository/properties.py Normal file
View File

@ -0,0 +1,59 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.repo import Repo
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.watcher.client import Client
from ahriman.models.repository_paths import RepositoryPaths
class Properties:
'''
repository internal objects holder
:ivar architecture: repository architecture
:ivar aur_url: base AUR url
:ivar config: configuration instance
:ivar logger: class logger
:ivar name: repository name
:ivar pacman: alpm wrapper instance
:ivar paths: repository paths instance
:ivar repo: repo commands wrapper instance
:ivar reporter: build status reporter instance
:ivar sign: GPG wrapper instance
'''
def __init__(self, architecture: str, config: Configuration) -> None:
self.logger = logging.getLogger('builder')
self.architecture = architecture
self.config = config
self.aur_url = config.get('alpm', 'aur_url')
self.name = config.get('repository', 'name')
self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
self.paths.create_tree()
self.pacman = Pacman(config)
self.sign = GPG(architecture, config)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client.load(architecture, config)

61
src/ahriman/repository/repository.py Normal file
View File

@ -0,0 +1,61 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from typing import Dict, List
from ahriman.core.util import package_like
from ahriman.models.package import Package
from ahriman.repository.executor import Executor
from ahriman.repository.update_handler import UpdateHandler
class Repository(Executor, UpdateHandler):
'''
base repository control class
'''
def packages(self) -> List[Package]:
'''
generate list of repository packages
:return: list of packages properties
'''
result: Dict[str, Package] = {}
for fn in os.listdir(self.paths.repository):
if not package_like(fn):
continue
full_path = os.path.join(self.paths.repository, fn)
try:
local = Package.load(full_path, self.pacman, self.aur_url)
result.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
return list(result.values())
def packages_built(self) -> List[str]:
'''
get list of files in built packages directory
:return: list of filenames from the directory
'''
return [
os.path.join(self.paths.packages, fn)
for fn in os.listdir(self.paths.packages)
]

92
src/ahriman/repository/update_handler.py Normal file
View File

@ -0,0 +1,92 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from typing import Iterable, List
from ahriman.models.package import Package
from ahriman.repository.cleaner import Cleaner
class UpdateHandler(Cleaner):
'''
trait to get package update list
'''
def packages(self) -> List[Package]:
'''
generate list of repository packages
:return: list of packages properties
'''
raise NotImplementedError
def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
'''
check AUR for updates
:param filter_packages: do not check every package just specified in the list
:param no_vcs: do not check VCS packages
:return: list of packages which are out-of-dated
'''
result: List[Package] = []
build_section = self.config.get_section_name('build', self.architecture)
ignore_list = self.config.getlist(build_section, 'ignore_packages')
for local in self.packages():
if local.base in ignore_list:
continue
if local.is_vcs and no_vcs:
continue
if filter_packages and local.base not in filter_packages:
continue
try:
remote = Package.load(local.base, self.pacman, self.aur_url)
if local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
continue
return result
def updates_manual(self) -> List[Package]:
'''
check for packages for which manual update has been requested
:return: list of packages which are out-of-dated
'''
result: List[Package] = []
known_bases = {package.base for package in self.packages()}
for fn in os.listdir(self.paths.manual):
try:
local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
result.append(local)
if local.base not in known_bases:
self.reporter.set_unknown(local)
else:
self.reporter.set_pending(local.base)
except Exception:
self.logger.exception(f'could not add package from {fn}', exc_info=True)
self.clear_manual()
return result