triggers implementation (#62)

This commit is contained in:
2022-05-09 17:45:39 +03:00
committed by Evgeniy Alekseev
parent 1905360f8f
commit b9cd98235e
49 changed files with 776 additions and 344 deletions

View File

@ -98,12 +98,11 @@ def _parser() -> argparse.ArgumentParser:
_set_repo_config_parser(subparsers)
_set_repo_rebuild_parser(subparsers)
_set_repo_remove_unknown_parser(subparsers)
_set_repo_report_parser(subparsers)
_set_repo_restore_parser(subparsers)
_set_repo_setup_parser(subparsers)
_set_repo_sign_parser(subparsers)
_set_repo_status_update_parser(subparsers)
_set_repo_sync_parser(subparsers)
_set_repo_triggers_parser(subparsers)
_set_repo_update_parser(subparsers)
_set_user_add_parser(subparsers)
_set_user_list_parser(subparsers)
@ -496,25 +495,6 @@ def _set_repo_remove_unknown_parser(root: SubParserAction) -> argparse.ArgumentP
return parser
def _set_repo_report_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for report subcommand
Args:
root(SubParserAction): subparsers for the commands
Returns:
argparse.ArgumentParser: created argument parser
"""
parser = root.add_parser("repo-report", aliases=["report"], help="generate report",
description="generate repository report according to current settings",
epilog="Create and/or update repository report as configured.",
formatter_class=_formatter)
parser.add_argument("target", help="target to generate report", nargs="*")
parser.set_defaults(handler=handlers.Report)
return parser
def _set_repo_restore_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for repository restore subcommand
@ -600,7 +580,7 @@ def _set_repo_status_update_parser(root: SubParserAction) -> argparse.ArgumentPa
return parser
def _set_repo_sync_parser(root: SubParserAction) -> argparse.ArgumentParser:
def _set_repo_triggers_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for repository sync subcommand
@ -610,12 +590,10 @@ def _set_repo_sync_parser(root: SubParserAction) -> argparse.ArgumentParser:
Returns:
argparse.ArgumentParser: created argument parser
"""
parser = root.add_parser("repo-sync", aliases=["sync"], help="sync repository",
description="sync repository files to remote server according to current settings",
epilog="Synchronize the repository to remote services as configured.",
parser = root.add_parser("repo-triggers", help="run triggers",
description="run triggers on empty build result as configured by settings",
formatter_class=_formatter)
parser.add_argument("target", help="target to sync", nargs="*")
parser.set_defaults(handler=handlers.Sync)
parser.set_defaults(handler=handlers.Triggers)
return parser

View File

@ -56,8 +56,7 @@ class Application(ApplicationPackages, ApplicationRepository):
Args:
result(Result): build result
"""
self.report([], result)
self.sync([], result.success)
self.repository.process_triggers(result)
def _known_packages(self) -> Set[str]:
"""

View File

@ -85,9 +85,9 @@ class ApplicationPackages(ApplicationProperties):
self.database.build_queue_insert(package)
self.database.remote_update(package)
with tmpdir() as local_path:
Sources.load(local_path, package, self.database.patches_get(package.base), self.repository.paths)
self._process_dependencies(local_path, known_packages, without_dependencies)
with tmpdir() as local_dir:
Sources.load(local_dir, package, self.database.patches_get(package.base), self.repository.paths)
self._process_dependencies(local_dir, known_packages, without_dependencies)
def _add_directory(self, source: str, *_: Any) -> None:
"""
@ -96,8 +96,8 @@ class ApplicationPackages(ApplicationProperties):
Args:
source(str): path to local directory
"""
local_path = Path(source)
for full_path in filter(package_like, local_path.iterdir()):
local_dir = Path(source)
for full_path in filter(package_like, local_dir.iterdir()):
self._add_archive(str(full_path))
def _add_local(self, source: str, known_packages: Set[str], without_dependencies: bool) -> None:
@ -146,19 +146,19 @@ class ApplicationPackages(ApplicationProperties):
self.database.remote_update(package)
# repository packages must not depend on unknown packages, thus we are not going to process dependencies
def _process_dependencies(self, local_path: Path, known_packages: Set[str], without_dependencies: bool) -> None:
def _process_dependencies(self, local_dir: Path, known_packages: Set[str], without_dependencies: bool) -> None:
"""
process package dependencies
Args:
local_path(Path): path to local package sources (i.e. cloned AUR repository)
local_dir(Path): path to local package sources (i.e. cloned AUR repository)
known_packages(Set[str]): list of packages which are known by the service
without_dependencies(bool): if set, dependency check will be disabled
"""
if without_dependencies:
return
dependencies = Package.dependencies(local_path)
dependencies = Package.dependencies(local_dir)
self.add(dependencies.difference(known_packages), PackageSource.AUR, without_dependencies)
def add(self, names: Iterable[str], source: PackageSource, without_dependencies: bool) -> None:

View File

@ -66,17 +66,6 @@ class ApplicationRepository(ApplicationProperties):
if packages:
self.repository.clear_packages()
def report(self, target: Iterable[str], result: Result) -> None:
"""
generate report
Args:
target(Iterable[str]): list of targets to run (e.g. html)
result(Result): build result
"""
targets = target or None
self.repository.process_report(targets, result)
def sign(self, packages: Iterable[str]) -> None:
"""
sign packages and repository
@ -102,17 +91,6 @@ class ApplicationRepository(ApplicationProperties):
self.repository.sign.process_sign_repository(self.repository.repo.repo_path)
self._finalize(Result())
def sync(self, target: Iterable[str], built_packages: Iterable[Package]) -> None:
"""
sync to remote server
Args:
target(Iterable[str]): list of targets to run (e.g. s3)
built_packages(Iterable[Package]): list of packages which has just been built
"""
targets = target or None
self.repository.process_sync(targets, built_packages)
def unknown(self) -> List[str]:
"""
get packages which were not found in AUR

View File

@ -29,14 +29,13 @@ from ahriman.application.handlers.patch import Patch
from ahriman.application.handlers.rebuild import Rebuild
from ahriman.application.handlers.remove import Remove
from ahriman.application.handlers.remove_unknown import RemoveUnknown
from ahriman.application.handlers.report import Report
from ahriman.application.handlers.restore import Restore
from ahriman.application.handlers.search import Search
from ahriman.application.handlers.setup import Setup
from ahriman.application.handlers.sign import Sign
from ahriman.application.handlers.status import Status
from ahriman.application.handlers.status_update import StatusUpdate
from ahriman.application.handlers.sync import Sync
from ahriman.application.handlers.triggers import Triggers
from ahriman.application.handlers.unsafe_commands import UnsafeCommands
from ahriman.application.handlers.update import Update
from ahriman.application.handlers.users import Users

View File

@ -71,10 +71,10 @@ class Handler:
if args.architecture: # architecture is specified explicitly
return sorted(set(args.architecture))
config = Configuration()
config.load(args.configuration)
configuration = Configuration()
configuration.load(args.configuration)
# wtf???
root = config.getpath("repository", "root") # pylint: disable=assignment-from-no-return
root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return
architectures = RepositoryPaths.known_architectures(root)
if not architectures: # well we did not find anything

View File

@ -27,9 +27,9 @@ from ahriman.core.configuration import Configuration
from ahriman.models.result import Result
class Report(Handler):
class Triggers(Handler):
"""
generate report handler
triggers handlers
"""
@classmethod
@ -45,4 +45,4 @@ class Report(Handler):
no_report(bool): force disable reporting
unsafe(bool): if set no user check will be performed before path creation
"""
Application(architecture, configuration, no_report, unsafe).report(args.target, Result())
Application(architecture, configuration, no_report, unsafe).repository.process_triggers(Result())

View File

@ -125,11 +125,11 @@ class Configuration(configparser.RawConfigParser):
Returns:
Configuration: configuration instance
"""
config = cls()
config.load(path)
config.merge_sections(architecture)
config.load_logging(quiet)
return config
configuration = cls()
configuration.load(path)
configuration.merge_sections(architecture)
configuration.load_logging(quiet)
return configuration
@staticmethod
def __convert_list(value: str) -> List[str]:

View File

@ -70,6 +70,12 @@ class InitializeException(RuntimeError):
RuntimeError.__init__(self, f"Could not load service: {details}")
class InvalidExtension(RuntimeError):
"""
exception being raised by trigger load in case of errors
"""
class InvalidOption(ValueError):
"""
exception which will be raised on configuration errors

View File

@ -24,3 +24,5 @@ from ahriman.core.report.console import Console
from ahriman.core.report.email import Email
from ahriman.core.report.html import HTML
from ahriman.core.report.telegram import Telegram
from ahriman.core.report.report_trigger import ReportTrigger

View File

@ -109,13 +109,13 @@ class Report:
result(Result): build result
"""
def run(self, packages: Iterable[Package], result: Result) -> None:
def run(self, result: Result, packages: Iterable[Package]) -> None:
"""
run report generation
Args:
packages(Iterable[Package]): list of packages to generate report
result(Result): build result
packages(Iterable[Package]): list of packages to generate report
Raises:
ReportFailed: in case of any report unmatched exception

View File

@ -17,31 +17,42 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
from typing import Iterable
from typing import Type
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.triggers import Trigger
from ahriman.core.report import Report
from ahriman.models.package import Package
from ahriman.models.result import Result
class Sync(Handler):
class ReportTrigger(Trigger):
"""
remote sync handler
report trigger
Attributes:
targets(List[str]): report target list
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
configuration: Configuration, no_report: bool, unsafe: bool) -> None:
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
callback for command line
default constructor
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
configuration(Configuration): configuration instance
no_report(bool): force disable reporting
unsafe(bool): if set no user check will be performed before path creation
"""
Application(architecture, configuration, no_report, unsafe).sync(args.target, [])
Trigger.__init__(self, architecture, configuration)
self.targets = configuration.getlist("report", "target")
def run(self, result: Result, packages: Iterable[Package]) -> None:
"""
run trigger
Args:
result(Result): build result
packages(Iterable[Package]): list of all available packages
"""
for target in self.targets:
runner = Report.load(self.architecture, self.configuration, target)
runner.run(result, packages)

View File

@ -23,9 +23,7 @@ from pathlib import Path
from typing import Iterable, List, Optional, Set
from ahriman.core.build_tools.task import Task
from ahriman.core.report import Report
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.upload import Upload
from ahriman.core.util import tmpdir
from ahriman.models.package import Package
from ahriman.models.result import Result
@ -143,35 +141,14 @@ class Executor(Cleaner):
return self.repo.repo_path
def process_report(self, targets: Optional[Iterable[str]], result: Result) -> None:
def process_triggers(self, result: Result) -> None:
"""
generate reports
process triggers setup by settings
Args:
targets(Optional[Iterable[str]]): list of targets to generate reports. Configuration option will be used
if it is not set
result(Result): build result
"""
if targets is None:
targets = self.configuration.getlist("report", "target")
for target in targets:
runner = Report.load(self.architecture, self.configuration, target)
runner.run(self.packages(), result)
def process_sync(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None:
"""
process synchronization to remote servers
Args:
targets(Optional[Iterable[str]]): list of targets to sync. Configuration option will be used
if it is not set
built_packages(Iterable[Package]): list of packages which has just been built
"""
if targets is None:
targets = self.configuration.getlist("upload", "target")
for target in targets:
runner = Upload.load(self.architecture, self.configuration, target)
runner.run(self.paths.repository, built_packages)
self.triggers.process(result, self.packages())
def process_update(self, packages: Iterable[Path]) -> Result:
"""

View File

@ -26,6 +26,7 @@ from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnsafeRun
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.client import Client
from ahriman.core.triggers import TriggerLoader
from ahriman.core.util import check_user
@ -45,6 +46,7 @@ class RepositoryProperties:
repo(Repo): repo commands wrapper instance
reporter(Client): build status reporter instance
sign(GPG): GPG wrapper instance
triggers(TriggerLoader): triggers holder
"""
def __init__(self, architecture: str, configuration: Configuration, database: SQLite,
@ -78,3 +80,4 @@ class RepositoryProperties:
self.sign = GPG(architecture, configuration)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client() if no_report else Client.load(configuration)
self.triggers = TriggerLoader(architecture, configuration)

View File

@ -0,0 +1,21 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.triggers.trigger import Trigger
from ahriman.core.triggers.trigger_loader import TriggerLoader

View File

@ -0,0 +1,79 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.models.package import Package
from ahriman.models.result import Result
class Trigger:
"""
trigger base class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
logger(logging.Logger): application logger
Examples:
This class must be used in order to create own extension. Basically idea is the following::
>>> class CustomTrigger(Trigger):
>>> def run(self, result: Result, packages: Iterable[Package]) -> None:
>>> perform_some_action()
Having this class you can pass it to ``configuration`` and it will be run on action::
>>> from ahriman.core.triggers import TriggerLoader
>>>
>>> configuration = Configuration()
>>> configuration.set_option("build", "triggers", "my.awesome.package.CustomTrigger")
>>>
>>> loader = TriggerLoader("x86_64", configuration)
>>> loader.process(Result(), [])
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration
def run(self, result: Result, packages: Iterable[Package]) -> None:
"""
run trigger
Args:
result(Result): build result
packages(Iterable[Package]): list of all available packages
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError

View File

@ -0,0 +1,159 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import importlib
import logging
from pathlib import Path
from types import ModuleType
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import InvalidExtension
from ahriman.core.triggers import Trigger
from ahriman.models.package import Package
from ahriman.models.result import Result
class TriggerLoader:
"""
trigger loader class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
logger(logging.Logger): application logger
triggers(List[Trigger]): list of loaded triggers according to the configuration
Examples:
This class more likely must not be used directly, but the usual workflow is the following::
>>> configuration = Configuration() # create configuration
>>> configuration.set_option("build", "triggers", "ahriman.core.report.ReportTrigger") # set class for load
Having such configuration you can create instance of the loader::
>>> loader = TriggerLoader("x86_64", configuration)
>>> print(loader.triggers)
After that you are free to run triggers::
>>> loader.process(Result(), [])
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration
self.triggers = [
self._load_trigger(trigger)
for trigger in configuration.getlist("build", "triggers")
]
def _load_module_from_file(self, module_path: str, implementation: str) -> ModuleType:
"""
load module by given file path
Args:
module_path(str): import package
implementation(str): specific trigger implementation, class name, required by import
Returns:
ModuleType: module loaded from the imported file
"""
self.logger.info("load module %s from path %s", implementation, module_path)
# basically this method is called only if ``module_path`` exists and is file.
# Thus, this method should never throw ``FileNotFoundError`` exception
loader = importlib.machinery.SourceFileLoader(implementation, module_path)
module = ModuleType(loader.name)
loader.exec_module(module)
return module
def _load_module_from_package(self, package: str) -> ModuleType:
"""
load module by given package name
Args:
package(str): package name to import
Returns:
ModuleType: module loaded from the imported module
"""
self.logger.info("load module from package %s", package)
try:
return importlib.import_module(package)
except ModuleNotFoundError:
raise InvalidExtension(f"Module {package} not found")
def _load_trigger(self, module_path: str) -> Trigger:
"""
load trigger by module path
Args:
module_path(str): module import path to load
Returns:
Trigger: loaded trigger based on settings
"""
*package_path_parts, class_name = module_path.split(".")
package_or_path = ".".join(package_path_parts)
if Path(package_or_path).is_file():
module = self._load_module_from_file(package_or_path, class_name)
else:
module = self._load_module_from_package(package_or_path)
trigger_type = getattr(module, class_name, None)
if not isinstance(trigger_type, type):
raise InvalidExtension(f"{class_name} of {package_or_path} is not a type")
self.logger.info("loaded type %s of package %s", class_name, package_or_path)
try:
trigger = trigger_type(self.architecture, self.configuration)
except Exception:
raise InvalidExtension(f"Could not load instance of trigger from {class_name} of {package_or_path}")
if not isinstance(trigger, Trigger):
raise InvalidExtension(f"Class {class_name} of {package_or_path} is not a Trigger")
return trigger
def process(self, result: Result, packages: Iterable[Package]) -> None:
"""
run remote sync
Args:
result(Result): build result
packages(Iterable[Package]): list of all available packages
"""
for trigger in self.triggers:
trigger_name = type(trigger).__name__
try:
self.logger.info("executing extension %s", trigger_name)
trigger.run(result, packages)
except Exception:
self.logger.exception("got exception while run trigger %s", trigger_name)

View File

@ -23,3 +23,5 @@ from ahriman.core.upload.http_upload import HttpUpload
from ahriman.core.upload.github import Github
from ahriman.core.upload.rsync import Rsync
from ahriman.core.upload.s3 import S3
from ahriman.core.upload.upload_trigger import UploadTrigger

View File

@ -68,7 +68,7 @@ class Upload:
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.config = configuration
self.configuration = configuration
@classmethod
def load(cls: Type[Upload], architecture: str, configuration: Configuration, target: str) -> Upload:

View File

@ -0,0 +1,58 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.triggers import Trigger
from ahriman.core.upload import Upload
from ahriman.models.package import Package
from ahriman.models.result import Result
class UploadTrigger(Trigger):
"""
synchronization trigger
Attributes:
targets(List[str]): upload target list
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
self.targets = configuration.getlist("upload", "target")
def run(self, result: Result, packages: Iterable[Package]) -> None:
"""
run trigger
Args:
result(Result): build result
packages(Iterable[Package]): list of all available packages
"""
for target in self.targets:
runner = Upload.load(self.architecture, self.configuration, target)
runner.run(self.configuration.repository_paths.repository, result.success)