arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-16 07:19:57 +00:00
Code Issues Projects Releases Wiki Activity

Extended package status page (#76)

Browse Source 
* implement log storage at backend
* handle process id during removal. During one process we can write logs from different packages in different times (e.g. check and update later) and we would like to store all logs belong to the same process
* set package context in main functions
* implement logs support in interface
* filter out logs posting http logs
* add timestamp to log records
* hide getting logs under reporter permission

List of breaking changes:

* `ahriman.core.lazy_logging.LazyLogging` has been renamed to `ahriman.core.log.LazyLogging`
* `ahriman.core.configuration.Configuration.from_path` does not have `quiet` attribute now
* `ahriman.core.configuration.Configuration` class does not have `load_logging` method now
* `ahriman.core.status.client.Client.load` requires `report` argument now
This commit is contained in:
Evgenii Alekseev
2022-11-22 02:58:22 +03:00
committed by GitHub
parent 2eb93a6090
commit 14cb548c3b
90 changed files with 1650 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/ahriman/application/application/application_properties.py
View File

@ -19,7 +19,7 @@
#
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository
@ -44,7 +44,8 @@ class ApplicationProperties(LazyLogging):
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
unsafe(bool): if set no user check will be performed before path creation
refresh_pacman_database(int): pacman database syncronization level, ``0`` is disabled
refresh_pacman_database(int, optional): pacman database syncronization level, ``0`` is disabled
(Default value = 0)
"""
self.configuration = configuration
self.architecture = architecture

4
src/ahriman/application/handlers/handler.py
View File

@ -28,6 +28,7 @@ from typing import List, Type
from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError
from ahriman.core.log import Log
from ahriman.models.repository_paths import RepositoryPaths
@ -94,7 +95,8 @@ class Handler:
bool: True on success, False otherwise
"""
try:
configuration = Configuration.from_path(args.configuration, architecture, args.quiet)
configuration = Configuration.from_path(args.configuration, architecture)
Log.load(configuration, quiet=args.quiet, report=args.report)
with Lock(args, architecture, configuration):
cls.run(args, architecture, configuration, report=args.report, unsafe=args.unsafe)
return True

4
src/ahriman/application/lock.py
View File

@ -28,7 +28,7 @@ from typing import Literal, Optional, Type
from ahriman import version
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import DuplicateRunError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum
@ -73,7 +73,7 @@ class Lock(LazyLogging):
self.unsafe = args.unsafe
self.paths = configuration.repository_paths
self.reporter = Client.load(configuration) if args.report else Client()
self.reporter = Client.load(configuration, report=args.report)
def __enter__(self) -> Lock:
"""

2
src/ahriman/core/alpm/pacman.py
View File

@ -24,7 +24,7 @@ from pyalpm import DB, Handle, Package, SIG_PACKAGE, error as PyalpmError # typ
from typing import Generator, Set
from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.repository_paths import RepositoryPaths

2
src/ahriman/core/alpm/remote/remote.py
View File

@ -22,7 +22,7 @@ from __future__ import annotations
from typing import Dict, List, Type
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.aur_package import AURPackage

2
src/ahriman/core/alpm/repo.py
View File

@ -21,7 +21,7 @@ from pathlib import Path
from typing import List
from ahriman.core.exceptions import BuildError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output
from ahriman.models.repository_paths import RepositoryPaths

4
src/ahriman/core/auth/auth.py
View File

@ -23,7 +23,7 @@ from typing import Optional, Type
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.auth_settings import AuthSettings
from ahriman.models.user_access import UserAccess
@ -62,7 +62,7 @@ class Auth(LazyLogging):
Returns:
str: login control as html code to insert
"""
return """<button type="button" class="btn btn-link" data-bs-toggle="modal" data-bs-target="#loginForm" style="text-decoration: none">login</button>"""
return """<button type="button" class="btn btn-link" data-bs-toggle="modal" data-bs-target="#loginForm" style="text-decoration: none"><i class="bi bi-box-arrow-in-right"></i> login</button>"""
@classmethod
def load(cls: Type[Auth], configuration: Configuration, database: SQLite) -> Auth:

2
src/ahriman/core/auth/oauth.py
View File

@ -69,7 +69,7 @@ class OAuth(Mapping):
Returns:
str: login control as html code to insert
"""
return """<a class="nav-link" href="/api/v1/login" title="login via OAuth2">login</a>"""
return """<a class="nav-link" href="/api/v1/login" title="login via OAuth2"><i class="bi bi-google"></i> login</a>"""
@staticmethod
def get_provider(name: str) -> Type[aioauth_client.OAuth2Client]:

14
src/ahriman/core/build_tools/sources.py
View File

@ -23,7 +23,7 @@ import shutil
from pathlib import Path
from typing import List, Optional
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, walk
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
@ -174,7 +174,8 @@ class Sources(LazyLogging):
sources_dir(Path): local path to git repository
remote(RemoteSource): remote target, branch and url
*pattern(str): glob patterns
commit_author(Optional[str]): commit author in form of git config (i.e. ``user <user@host>``)
commit_author(Optional[str], optional): commit author in form of git config (i.e. ``user <user@host>``)
(Default value = None)
"""
instance = Sources()
instance.add(sources_dir, *pattern)
@ -188,7 +189,8 @@ class Sources(LazyLogging):
Args:
sources_dir(Path): local path to git repository
*pattern(str): glob patterns
intent_to_add(bool): record only the fact that it will be added later, acts as --intent-to-add git flag
intent_to_add(bool, optional): record only the fact that it will be added later, acts as
--intent-to-add git flag (Default value = False)
"""
# glob directory to find files which match the specified patterns
found_files: List[Path] = []
@ -208,9 +210,9 @@ class Sources(LazyLogging):
Args:
sources_dir(Path): local path to git repository
message(Optional[str]): optional commit message if any. If none set, message will be generated according to
the current timestamp
author(Optional[str]): optional commit author if any
message(Optional[str], optional): optional commit message if any. If none set, message will be generated
according to the current timestamp (Default value = None)
author(Optional[str], optional): optional commit author if any (Default value = None)
"""
if message is None:
message = f"Autogenerated commit at {datetime.datetime.utcnow()}"

12
src/ahriman/core/build_tools/task.py
View File

@ -24,7 +24,7 @@ from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import BuildError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
@ -84,10 +84,12 @@ class Task(LazyLogging):
user=self.uid)
# well it is not actually correct, but we can deal with it
packages = Task._check_output("makepkg", "--packagelist",
exception=BuildError(self.package.base),
cwd=sources_dir,
logger=self.logger).splitlines()
packages = Task._check_output(
"makepkg", "--packagelist",
exception=BuildError(self.package.base),
cwd=sources_dir,
logger=self.logger
).splitlines()
return [Path(package) for package in packages]
def init(self, sources_dir: Path, database: SQLite) -> None:

32
src/ahriman/core/configuration.py
View File

@ -20,10 +20,8 @@
from __future__ import annotations
import configparser
import logging
import sys
from logging.config import fileConfig
from pathlib import Path
from typing import Any, Dict, Generator, List, Optional, Tuple, Type
@ -38,8 +36,6 @@ class Configuration(configparser.RawConfigParser):
Attributes:
ARCHITECTURE_SPECIFIC_SECTIONS(List[str]): (class attribute) known sections which can be architecture specific.
Required by dump and merging functions
DEFAULT_LOG_FORMAT(str): (class attribute) default log format (in case of fallback)
DEFAULT_LOG_LEVEL(int): (class attribute) default log level (in case of fallback)
SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package
architecture(Optional[str]): repository architecture
path(Optional[Path]): path to root configuration file
@ -64,9 +60,6 @@ class Configuration(configparser.RawConfigParser):
>>> path, architecture = configuration.check_loaded()
"""
DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d %(funcName)s]: %(message)s"
DEFAULT_LOG_LEVEL = logging.DEBUG
ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "sign", "web"]
SYSTEM_CONFIGURATION_PATH = Path(sys.prefix) / "share" / "ahriman" / "settings" / "ahriman.ini"
@ -75,8 +68,8 @@ class Configuration(configparser.RawConfigParser):
default constructor. In the most cases must not be called directly
Args:
allow_no_value(bool): copies ``configparser.RawConfigParser`` behaviour. In case if it is set to ``True``,
the keys without values will be allowed
allow_no_value(bool, optional): copies ``configparser.RawConfigParser`` behaviour. In case if it is set
to ``True``, the keys without values will be allowed (Default value = False)
"""
configparser.RawConfigParser.__init__(self, allow_no_value=allow_no_value, converters={
"list": self.__convert_list,
@ -117,14 +110,13 @@ class Configuration(configparser.RawConfigParser):
return RepositoryPaths(self.getpath("repository", "root"), architecture)
@classmethod
def from_path(cls: Type[Configuration], path: Path, architecture: str, quiet: bool) -> Configuration:
def from_path(cls: Type[Configuration], path: Path, architecture: str) -> Configuration:
"""
constructor with full object initialization
Args:
path(Path): path to root configuration file
architecture(str): repository architecture
quiet(bool): force disable any log messages
Returns:
Configuration: configuration instance
@ -132,7 +124,6 @@ class Configuration(configparser.RawConfigParser):
configuration = cls()
configuration.load(path)
configuration.merge_sections(architecture)
configuration.load_logging(quiet)
return configuration
@staticmethod
@ -281,23 +272,6 @@ class Configuration(configparser.RawConfigParser):
except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError):
pass
def load_logging(self, quiet: bool) -> None:
"""
setup logging settings from configuration
Args:
quiet(bool): force disable any log messages
"""
try:
path = self.logging_path
fileConfig(path)
except Exception:
logging.basicConfig(filename=None, format=self.DEFAULT_LOG_FORMAT,
level=self.DEFAULT_LOG_LEVEL)
logging.exception("could not load logging from configuration, fallback to stderr")
if quiet:
logging.disable(logging.WARNING) # only print errors here
def merge_sections(self, architecture: str) -> None:
"""
merge architecture specific sections into main configuration

2
src/ahriman/core/database/migrations/__init__.py
View File

@ -27,7 +27,7 @@ from typing import List, Type
from ahriman.core.configuration import Configuration
from ahriman.core.database.data import migrate_data
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.migration import Migration
from ahriman.models.migration_result import MigrationResult

35
src/ahriman/core/database/migrations/m004_logs.py Normal file
View File

@ -0,0 +1,35 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ["steps"]
steps = [
"""
create table logs (
package_base text not null,
process_id integer not null,
created real not null,
record text
)
""",
"""
create index logs_package_base_process_id on logs (package_base, process_id)
""",
]

1
src/ahriman/core/database/operations/__init__.py
View File

@ -21,5 +21,6 @@ from ahriman.core.database.operations.operations import Operations
from ahriman.core.database.operations.auth_operations import AuthOperations
from ahriman.core.database.operations.build_operations import BuildOperations
from ahriman.core.database.operations.logs_operations import LogsOperations
from ahriman.core.database.operations.package_operations import PackageOperations
from ahriman.core.database.operations.patch_operations import PatchOperations

2
src/ahriman/core/database/operations/build_operations.py
View File

@ -26,7 +26,7 @@ from ahriman.models.package import Package
class BuildOperations(Operations):
"""
operations for main functions
operations for build queue functions
"""
def build_queue_clear(self, package_base: Optional[str]) -> None:

102
src/ahriman/core/database/operations/logs_operations.py Normal file
View File

@ -0,0 +1,102 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from typing import List, Optional
from ahriman.core.database.operations import Operations
from ahriman.core.util import pretty_datetime
from ahriman.models.log_record_id import LogRecordId
class LogsOperations(Operations):
"""
logs operations
"""
def logs_get(self, package_base: str) -> str:
"""
extract logs for specified package base
Args:
package_base(str): package base to extract logs
Return:
str: full package log
"""
def run(connection: Connection) -> List[str]:
return [
f"""[{pretty_datetime(row["created"])}] {row["record"]}"""
for row in connection.execute(
"""
select created, record from logs where package_base = :package_base
order by created asc
""",
{"package_base": package_base})
]
records = self.with_connection(run)
return "\n".join(records)
def logs_insert(self, log_record_id: LogRecordId, created: float, record: str) -> None:
"""
write new log record to database
Args:
log_record_id(LogRecordId): current log record id
created(float): log created timestamp from log record attribute
record(str): log record
"""
def run(connection: Connection) -> None:
connection.execute(
"""
insert into logs
(package_base, process_id, created, record)
values
(:package_base, :process_id, :created, :record)
""",
dict(
package_base=log_record_id.package_base,
process_id=log_record_id.process_id,
created=created,
record=record
)
)
return self.with_connection(run, commit=True)
def logs_remove(self, package_base: str, current_process_id: Optional[int]) -> None:
"""
remove log records for the specified package
Args:
package_base(str): package base to remove logs
current_process_id(Optional[int]): current process id. If set it will remove only logs belonging to another
process
"""
def run(connection: Connection) -> None:
connection.execute(
"""
delete from logs
where package_base = :package_base and (:process_id is null or process_id <> :process_id)
""",
{"package_base": package_base, "process_id": current_process_id}
)
return self.with_connection(run, commit=True)

3
src/ahriman/core/database/operations/operations.py
View File

@ -22,7 +22,8 @@ import sqlite3
from pathlib import Path
from typing import Any, Dict, Tuple, TypeVar, Callable
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
T = TypeVar("T")

5
src/ahriman/core/database/sqlite.py
View File

@ -27,10 +27,11 @@ from typing import Type
from ahriman.core.configuration import Configuration
from ahriman.core.database.migrations import Migrations
from ahriman.core.database.operations import AuthOperations, BuildOperations, PackageOperations, PatchOperations
from ahriman.core.database.operations import AuthOperations, BuildOperations, LogsOperations, PackageOperations, \
PatchOperations
class SQLite(AuthOperations, BuildOperations, PackageOperations, PatchOperations):
class SQLite(AuthOperations, BuildOperations, LogsOperations, PackageOperations, PatchOperations):
"""
wrapper for sqlite3 database

2
src/ahriman/core/gitremote/remote_pull.py
View File

@ -25,7 +25,7 @@ from tempfile import TemporaryDirectory
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import GitRemoteError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.util import walk
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource

2
src/ahriman/core/gitremote/remote_push.py
View File

@ -26,7 +26,7 @@ from typing import Generator
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import GitRemoteError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource

21
src/ahriman/core/log/__init__.py Normal file
View File

@ -0,0 +1,21 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.log.lazy_logging import LazyLogging
from ahriman.core.log.log import Log

61
src/ahriman/core/log/filtered_access_logger.py Normal file
View File

@ -0,0 +1,61 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import re
from aiohttp.abc import BaseRequest, StreamResponse
from aiohttp.web_log import AccessLogger
class FilteredAccessLogger(AccessLogger):
"""
access logger implementation with log filter enabled
Attributes:
LOG_PATH_REGEX(re.Pattern): (class attribute) regex for logs uri
"""
# official packages have only ``[A-Za-z0-9_.+-]`` regex
LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[A-Za-z0-9_.+%-]+/logs$")
@staticmethod
def is_logs_post(request: BaseRequest) -> bool:
"""
check if request looks lie logs posting
Args:
request(BaseRequest): http reqeust descriptor
Returns:
bool: True in case if request looks like logs positing and False otherwise
"""
return request.method == "POST" and FilteredAccessLogger.LOG_PATH_REGEX.match(request.path) is not None
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
"""
access log with enabled filter by request path
Args:
request(BaseRequest): http reqeust descriptor
response(StreamResponse): streaming response object
time(float):
"""
if self.is_logs_post(request):
return
AccessLogger.log(self, request, response, time)

80
src/ahriman/core/log/http_log_handler.py Normal file
View File

@ -0,0 +1,80 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import logging
from ahriman.core.configuration import Configuration
class HttpLogHandler(logging.Handler):
"""
handler for the http logging. Because default ``logging.handlers.HTTPHandler`` does not support cookies
authorization, we have to implement own handler which overrides the ``logging.handlers.HTTPHandler.emit`` method
Attributes:
reporter(Client): build status reporter instance
"""
def __init__(self, configuration: Configuration, *, report: bool) -> None:
"""
default constructor
Args:
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we don't really care about those parameters because they will be handled by the reporter
logging.Handler.__init__(self)
# client has to be importer here because of circular imports
from ahriman.core.status.client import Client
self.reporter = Client.load(configuration, report=report)
@classmethod
def load(cls, configuration: Configuration, *, report: bool) -> HttpLogHandler:
"""
install logger. This function creates handler instance and adds it to the handler list in case if no other
http handler found
Args:
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
root = logging.getLogger()
if (handler := next((handler for handler in root.handlers if isinstance(handler, cls)), None)) is not None:
return handler # there is already registered instance
handler = cls(configuration, report=report)
root.addHandler(handler)
return handler
def emit(self, record: logging.LogRecord) -> None:
"""
emit log records using reporter client
Args:
record(logging.LogRecord): log record to log
"""
try:
self.reporter.logs(record)
except Exception:
self.handleError(record)

47
src/ahriman/core/lazy_logging.py → src/ahriman/core/log/lazy_logging.py
View File

@ -17,9 +17,10 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import contextlib
import logging
from typing import Any
from typing import Any, Generator
class LazyLogging:
@ -62,3 +63,47 @@ class LazyLogging:
clazz = self.__class__
prefix = "" if clazz.__module__ is None else f"{clazz.__module__}."
return f"{prefix}{clazz.__qualname__}"
@staticmethod
def _package_logger_reset() -> None:
"""
reset package logger to empty one
"""
logging.setLogRecordFactory(logging.LogRecord)
@staticmethod
def _package_logger_set(package_base: str) -> None:
"""
set package base as extra info to the logger
Args:
package_base(str): package base
"""
current_factory = logging.getLogRecordFactory()
def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord:
record = current_factory(*args, **kwargs)
record.package_base = package_base
return record
logging.setLogRecordFactory(package_record_factory)
@contextlib.contextmanager
def in_package_context(self, package_base: str) -> Generator[None, None, None]:
"""
execute function while setting package context
Args:
package_base(str): package base to set context in
Examples:
This function is designed to be called as context manager with ``package_base`` argument, e.g.:
>>> with self.in_package_context(package.base):
>>> build_package(package)
"""
try:
self._package_logger_set(package_base)
yield
finally:
self._package_logger_reset()

61
src/ahriman/core/log/log.py Normal file
View File

@ -0,0 +1,61 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from logging.config import fileConfig
from ahriman.core.configuration import Configuration
from ahriman.core.log.http_log_handler import HttpLogHandler
class Log:
"""
simple static method class which setups application loggers
Attributes:
DEFAULT_LOG_FORMAT(str): (class attribute) default log format (in case of fallback)
DEFAULT_LOG_LEVEL(int): (class attribute) default log level (in case of fallback)
"""
DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d %(funcName)s]: %(message)s"
DEFAULT_LOG_LEVEL = logging.DEBUG
@staticmethod
def load(configuration: Configuration, *, quiet: bool, report: bool) -> None:
"""
setup logging settings from configuration
Args:
configuration(Configuration): configuration instance
quiet(bool): force disable any log messages
report(bool): force enable or disable reporting
"""
try:
path = configuration.logging_path
fileConfig(path)
except Exception:
logging.basicConfig(filename=None, format=Log.DEFAULT_LOG_FORMAT,
level=Log.DEFAULT_LOG_LEVEL)
logging.exception("could not load logging from configuration, fallback to stderr")
HttpLogHandler.load(configuration, report=report)
if quiet:
logging.disable(logging.WARNING) # only print errors here

2
src/ahriman/core/report/report.py
View File

@ -23,7 +23,7 @@ from typing import Iterable, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ReportError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.report_settings import ReportSettings
from ahriman.models.result import Result

49
src/ahriman/core/repository/executor.py
View File

@ -84,7 +84,8 @@ class Executor(Cleaner):
result = Result()
for single in updates:
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (build_dir := Path(dir_name)):
with self.in_package_context(single.base), \
TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (build_dir := Path(dir_name)):
try:
build_single(single, build_dir)
result.add_success(single)
@ -110,6 +111,7 @@ class Executor(Cleaner):
self.paths.tree_clear(package_base) # remove all internal files
self.database.build_queue_clear(package_base)
self.database.patches_remove(package_base, [])
self.database.logs_remove(package_base, None)
self.reporter.remove(package_base) # we only update status page in case of base removal
except Exception:
self.logger.exception("could not remove base %s", package_base)
@ -153,21 +155,21 @@ class Executor(Cleaner):
Returns:
Result: path to repository database
"""
def rename(archive: PackageDescription, base: str) -> None:
def rename(archive: PackageDescription, package_base: str) -> None:
if archive.filename is None:
self.logger.warning("received empty package name for base %s", base)
self.logger.warning("received empty package name for base %s", package_base)
return # suppress type checking, it never can be none actually
if (safe := safe_filename(archive.filename)) != archive.filename:
shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe)
archive.filename = safe
def update_single(name: Optional[str], base: str) -> None:
def update_single(name: Optional[str], package_base: str) -> None:
if name is None:
self.logger.warning("received empty package name for base %s", base)
self.logger.warning("received empty package name for base %s", package_base)
return # suppress type checking, it never can be none actually
# in theory, it might be NOT packages directory, but we suppose it is
full_path = self.paths.packages / name
files = self.sign.process_sign_package(full_path, base)
files = self.sign.process_sign_package(full_path, package_base)
for src in files:
dst = self.paths.repository / safe_filename(src.name)
shutil.move(src, dst)
@ -180,24 +182,25 @@ class Executor(Cleaner):
result = Result()
for local in updates:
try:
for description in local.packages.values():
rename(description, local.base)
update_single(description.filename, local.base)
self.reporter.set_success(local)
result.add_success(local)
with self.in_package_context(local.base):
try:
for description in local.packages.values():
rename(description, local.base)
update_single(description.filename, local.base)
self.reporter.set_success(local)
result.add_success(local)
current_package_archives = {
package
for current in current_packages
if current.base == local.base
for package in current.packages
}
removed_packages.extend(current_package_archives.difference(local.packages))
except Exception:
self.reporter.set_failed(local.base)
result.add_failed(local)
self.logger.exception("could not process %s", local.base)
current_package_archives = {
package
for current in current_packages
if current.base == local.base
for package in current.packages
}
removed_packages.extend(current_package_archives.difference(local.packages))
except Exception:
self.reporter.set_failed(local.base)
result.add_failed(local)
self.logger.exception("could not process %s", local.base)
self.clear_packages()
self.process_remove(removed_packages)

7
src/ahriman/core/repository/repository_properties.py
View File

@ -22,7 +22,7 @@ from ahriman.core.alpm.repo import Repo
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnsafeRunError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.client import Client
from ahriman.core.triggers import TriggerLoader
@ -58,7 +58,8 @@ class RepositoryProperties(LazyLogging):
database(SQLite): database instance
report(bool): force enable or disable reporting
unsafe(bool): if set no user check will be performed before path creation
refresh_pacman_database(int): pacman database syncronization level, ``0`` is disabled
refresh_pacman_database(int, optional): pacman database syncronization level, ``0`` is disabled
(Default value = 0)
"""
self.architecture = architecture
self.configuration = configuration
@ -77,5 +78,5 @@ class RepositoryProperties(LazyLogging):
self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database)
self.sign = GPG(architecture, configuration)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client.load(configuration) if report else Client()
self.reporter = Client.load(configuration, report=report)
self.triggers = TriggerLoader(architecture, configuration)

65
src/ahriman/core/repository/update_handler.py
View File

@ -56,26 +56,26 @@ class UpdateHandler(Cleaner):
result: List[Package] = []
for local in self.packages():
if local.base in self.ignore_list:
continue
if local.is_vcs and not vcs:
continue
if filter_packages and local.base not in filter_packages:
continue
source = local.remote.source if local.remote is not None else None
with self.in_package_context(local.base):
if local.base in self.ignore_list:
continue
if local.is_vcs and not vcs:
continue
if filter_packages and local.base not in filter_packages:
continue
source = local.remote.source if local.remote is not None else None
try:
if source == PackageSource.Repository:
remote = Package.from_official(local.base, self.pacman)
else:
remote = Package.from_aur(local.base, self.pacman)
if local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception("could not load remote package %s", local.base)
continue
try:
if source == PackageSource.Repository:
remote = Package.from_official(local.base, self.pacman)
else:
remote = Package.from_aur(local.base, self.pacman)
if local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception("could not load remote package %s", local.base)
return result
@ -89,20 +89,21 @@ class UpdateHandler(Cleaner):
result: List[Package] = []
packages = {local.base: local for local in self.packages()}
for dirname in self.paths.cache.iterdir():
try:
Sources.fetch(dirname, remote=None)
remote = Package.from_build(dirname)
for cache_dir in self.paths.cache.iterdir():
with self.in_package_context(cache_dir.name):
try:
Sources.fetch(cache_dir, remote=None)
remote = Package.from_build(cache_dir)
local = packages.get(remote.base)
if local is None:
self.reporter.set_unknown(remote)
result.append(remote)
elif local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:
self.logger.exception("could not process package at %s", dirname)
local = packages.get(remote.base)
if local is None:
self.reporter.set_unknown(remote)
result.append(remote)
elif local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:
self.logger.exception("could not process package at %s", cache_dir)
return result

8
src/ahriman/core/sign/gpg.py
View File

@ -24,7 +24,7 @@ from typing import List, Optional, Set, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import BuildError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, exception_response_text
from ahriman.models.sign_settings import SignSettings
@ -157,20 +157,20 @@ class GPG(LazyLogging):
logger=self.logger)
return [path, path.parent / f"{path.name}.sig"]
def process_sign_package(self, path: Path, base: str) -> List[Path]:
def process_sign_package(self, path: Path, package_base: str) -> List[Path]:
"""
sign package if required by configuration
Args:
path(Path): path to file to sign
base(str): package base required to check for key overrides
package_base(str): package base required to check for key overrides
Returns:
List[Path]: list of generated files including original file
"""
if SignSettings.Packages not in self.targets:
return [path]
key = self.configuration.get("sign", f"key_{base}", fallback=self.default_key)
key = self.configuration.get("sign", f"key_{package_base}", fallback=self.default_key)
if key is None:
self.logger.error("no default key set, skip package %s sign", path)
return [path]

2
src/ahriman/core/spawn.py
View File

@ -27,7 +27,7 @@ from threading import Lock, Thread
from typing import Callable, Dict, Iterable, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.package_source import PackageSource

49
src/ahriman/core/status/client.py
View File

@ -19,6 +19,8 @@
#
from __future__ import annotations
import logging
from typing import List, Optional, Tuple, Type
from ahriman.core.configuration import Configuration
@ -33,19 +35,24 @@ class Client:
"""
@classmethod
def load(cls: Type[Client], configuration: Configuration) -> Client:
def load(cls: Type[Client], configuration: Configuration, *, report: bool) -> Client:
"""
load client from settings
Args:
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
Returns:
Client: client according to current settings
"""
if not report:
return cls()
address = configuration.get("web", "address", fallback=None)
host = configuration.get("web", "host", fallback=None)
port = configuration.getint("web", "port", fallback=None)
if address or (host and port):
from ahriman.core.status.web_client import WebClient
return WebClient(configuration)
@ -60,17 +67,17 @@ class Client:
status(BuildStatusEnum): current package build status
"""
def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
def get(self, package_base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
"""
get package status
Args:
base(Optional[str]): package base to get
package_base(Optional[str]): package base to get
Returns:
List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found
"""
del base
del package_base
return []
def get_internal(self) -> InternalStatus:
@ -82,20 +89,28 @@ class Client:
"""
return InternalStatus(status=BuildStatus())
def remove(self, base: str) -> None:
def logs(self, record: logging.LogRecord) -> None:
"""
post log record
Args:
record(logging.LogRecord): log record to post to api
"""
def remove(self, package_base: str) -> None:
"""
remove packages from watcher
Args:
base(str): package base to remove
package_base(str): package base to remove
"""
def update(self, base: str, status: BuildStatusEnum) -> None:
def update(self, package_base: str, status: BuildStatusEnum) -> None:
"""
update package build status. Unlike ``add`` it does not update package properties
Args:
base(str): package base to update
package_base(str): package base to update
status(BuildStatusEnum): current package build status
"""
@ -107,32 +122,32 @@ class Client:
status(BuildStatusEnum): current ahriman status
"""
def set_building(self, base: str) -> None:
def set_building(self, package_base: str) -> None:
"""
set package status to building
Args:
base(str): package base to update
package_base(str): package base to update
"""
return self.update(base, BuildStatusEnum.Building)
return self.update(package_base, BuildStatusEnum.Building)
def set_failed(self, base: str) -> None:
def set_failed(self, package_base: str) -> None:
"""
set package status to failed
Args:
base(str): package base to update
package_base(str): package base to update
"""
return self.update(base, BuildStatusEnum.Failed)
return self.update(package_base, BuildStatusEnum.Failed)
def set_pending(self, base: str) -> None:
def set_pending(self, package_base: str) -> None:
"""
set package status to pending
Args:
base(str): package base to update
package_base(str): package base to update
"""
return self.update(base, BuildStatusEnum.Pending)
return self.update(package_base, BuildStatusEnum.Pending)
def set_success(self, package: Package) -> None:
"""

54
src/ahriman/core/status/watcher.py
View File

@ -17,14 +17,17 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from typing import Dict, List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
@ -57,6 +60,9 @@ class Watcher(LazyLogging):
self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
self.status = BuildStatus()
# special variables for updating logs
self._last_log_record_id = LogRecordId("", os.getpid())
@property
def packages(self) -> List[Tuple[Package, BuildStatus]]:
"""
@ -67,12 +73,12 @@ class Watcher(LazyLogging):
"""
return list(self.known.values())
def get(self, base: str) -> Tuple[Package, BuildStatus]:
def get(self, package_base: str) -> Tuple[Package, BuildStatus]:
"""
get current package base build status
Args:
base(str): package base
package_base(str): package base
Returns:
Tuple[Package, BuildStatus]: package and its status
@ -81,9 +87,21 @@ class Watcher(LazyLogging):
UnknownPackage: if no package found
"""
try:
return self.known[base]
return self.known[package_base]
except KeyError:
raise UnknownPackageError(base)
raise UnknownPackageError(package_base)
def get_logs(self, package_base: str) -> str:
"""
extract logs for the package base
Args:
package_base(str): package base
Returns:
str: package logs
"""
return self.database.logs_get(package_base)
def load(self) -> None:
"""
@ -110,6 +128,17 @@ class Watcher(LazyLogging):
"""
self.known.pop(package_base, None)
self.database.package_remove(package_base)
self.remove_logs(package_base, None)
def remove_logs(self, package_base: str, current_process_id: Optional[int]) -> None:
"""
remove package related logs
Args:
package_base(str): package base
current_process_id(int): current process id
"""
self.database.logs_remove(package_base, current_process_id)
def update(self, package_base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
"""
@ -132,6 +161,21 @@ class Watcher(LazyLogging):
self.known[package_base] = (package, full_status)
self.database.package_update(package, full_status)
def update_logs(self, log_record_id: LogRecordId, created: float, record: str) -> None:
"""
make new log record into database
Args:
log_record_id(LogRecordId): log record id
created(float): log created record
record(str): log record
"""
if self._last_log_record_id != log_record_id:
# there is new log record, so we remove old ones
self.remove_logs(log_record_id.package_base, log_record_id.process_id)
self._last_log_record_id = log_record_id
self.database.logs_insert(log_record_id, created, record)
def update_self(self, status: BuildStatusEnum) -> None:
"""
update service status

72
src/ahriman/core/status/web_client.py
View File

@ -17,12 +17,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import requests
from typing import List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.util import exception_response_text
from ahriman.models.build_status import BuildStatusEnum, BuildStatus
@ -114,17 +115,29 @@ class WebClient(Client, LazyLogging):
except Exception:
self.logger.exception("could not login as %s", self.user)
def _package_url(self, base: str = "") -> str:
def _logs_url(self, package_base: str) -> str:
"""
get url for the logs api
Args:
package_base(str): package base
Returns:
str: full url for web service for logs
"""
return f"{self.address}/api/v1/packages/{package_base}/logs"
def _package_url(self, package_base: str = "") -> str:
"""
url generator
Args:
base(str, optional): package base to generate url (Default value = "")
package_base(str, optional): package base to generate url (Default value = "")
Returns:
str: full url of web service for specific package base
"""
return f"{self.address}/api/v1/packages/{base}"
return f"{self.address}/api/v1/packages/{package_base}"
def add(self, package: Package, status: BuildStatusEnum) -> None:
"""
@ -147,18 +160,18 @@ class WebClient(Client, LazyLogging):
except Exception:
self.logger.exception("could not add %s", package.base)
def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
def get(self, package_base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
"""
get package status
Args:
base(Optional[str]): package base to get
package_base(Optional[str]): package base to get
Returns:
List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found
"""
try:
response = self.__session.get(self._package_url(base or ""))
response = self.__session.get(self._package_url(package_base or ""))
response.raise_for_status()
status_json = response.json()
@ -167,9 +180,9 @@ class WebClient(Client, LazyLogging):
for package in status_json
]
except requests.HTTPError as e:
self.logger.exception("could not get %s: %s", base, exception_response_text(e))
self.logger.exception("could not get %s: %s", package_base, exception_response_text(e))
except Exception:
self.logger.exception("could not get %s", base)
self.logger.exception("could not get %s", package_base)
return []
def get_internal(self) -> InternalStatus:
@ -191,38 +204,59 @@ class WebClient(Client, LazyLogging):
self.logger.exception("could not get web service status")
return InternalStatus(status=BuildStatus())
def remove(self, base: str) -> None:
def logs(self, record: logging.LogRecord) -> None:
"""
post log record
Args:
record(logging.LogRecord): log record to post to api
"""
package_base = getattr(record, "package_base", None)
if package_base is None:
return # in case if no package base supplised we need just skip log message
payload = {
"created": record.created,
"message": record.getMessage(),
"process_id": record.process,
}
# in this method exception has to be handled outside in logger handler
response = self.__session.post(self._logs_url(package_base), json=payload)
response.raise_for_status()
def remove(self, package_base: str) -> None:
"""
remove packages from watcher
Args:
base(str): basename to remove
package_base(str): basename to remove
"""
try:
response = self.__session.delete(self._package_url(base))
response = self.__session.delete(self._package_url(package_base))
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not delete %s: %s", base, exception_response_text(e))
self.logger.exception("could not delete %s: %s", package_base, exception_response_text(e))
except Exception:
self.logger.exception("could not delete %s", base)
self.logger.exception("could not delete %s", package_base)
def update(self, base: str, status: BuildStatusEnum) -> None:
def update(self, package_base: str, status: BuildStatusEnum) -> None:
"""
update package build status. Unlike ``add`` it does not update package properties
Args:
base(str): package base to update
package_base(str): package base to update
status(BuildStatusEnum): current package build status
"""
payload = {"status": status.value}
try:
response = self.__session.post(self._package_url(base), json=payload)
response = self.__session.post(self._package_url(package_base), json=payload)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not update %s: %s", base, exception_response_text(e))
self.logger.exception("could not update %s: %s", package_base, exception_response_text(e))
except Exception:
self.logger.exception("could not update %s", base)
self.logger.exception("could not update %s", package_base)
def update_self(self, status: BuildStatusEnum) -> None:
"""

2
src/ahriman/core/triggers/trigger.py
View File

@ -20,7 +20,7 @@
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.result import Result

2
src/ahriman/core/triggers/trigger_loader.py
View File

@ -27,7 +27,7 @@ from typing import Generator, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExtensionError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.triggers import Trigger
from ahriman.models.package import Package
from ahriman.models.result import Result

2
src/ahriman/core/upload/upload.py
View File

@ -24,7 +24,7 @@ from typing import Iterable, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SynchronizationError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.upload_settings import UploadSettings

3
src/ahriman/core/util.py
View File

@ -44,7 +44,8 @@ def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optiona
Args:
*args(str): command line arguments
exception(Optional[Exception]): exception which has to be reraised instead of default subprocess exception
exception(Optional[Exception], optional): exception which has to be reraised instead of default subprocess
exception (Default value = None)
cwd(Optional[Path], optional): current working directory (Default value = None)
input_data(Optional[str], optional): data which will be written to command stdin (Default value = None)
logger(Optional[Logger], optional): logger to log command result if required (Default value = None)

34
src/ahriman/models/log_record_id.py Normal file
View File

@ -0,0 +1,34 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
@dataclass(frozen=True)
class LogRecordId:
"""
log record process identifier
Attributes:
package_base(str): package base for which log record belongs
process_id(int): process id from which log record was emitted
"""
package_base: str
process_id: int

7
src/ahriman/models/package.py
View File

@ -30,7 +30,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Type
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb
from ahriman.core.exceptions import PackageInfoError
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, full_version
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
@ -218,7 +218,7 @@ class Package(LazyLogging):
Args:
name(str): package name (either base or normal name)
pacman(Pacman): alpm wrapper instance
use_syncdb(bool): use pacman databases instead of official repositories RPC (Default value = True)
use_syncdb(bool, optional): use pacman databases instead of official repositories RPC (Default value = True)
Returns:
Package: package properties
@ -365,7 +365,8 @@ class Package(LazyLogging):
Args:
remote(Package): package properties from remote source
paths(RepositoryPaths): repository paths instance. Required for VCS packages cache
calculate_version(bool, optional): expand version to actual value (by calculating git versions) (Default value = True)
calculate_version(bool, optional): expand version to actual value (by calculating git versions)
(Default value = True)
Returns:
bool: True if the package is out-of-dated and False otherwise

9
src/ahriman/web/routes.py
View File

@ -25,6 +25,7 @@ from ahriman.web.views.service.add import AddView
from ahriman.web.views.service.remove import RemoveView
from ahriman.web.views.service.request import RequestView
from ahriman.web.views.service.search import SearchView
from ahriman.web.views.status.logs import LogsView
from ahriman.web.views.status.package import PackageView
from ahriman.web.views.status.packages import PackagesView
from ahriman.web.views.status.status import StatusView
@ -61,6 +62,10 @@ def setup_routes(application: Application, static_path: Path) -> None:
* ``GET /api/v1/package/:base`` get package base status
* ``POST /api/v1/package/:base`` update package base status
* ``DELETE /api/v1/packages/{package}/logs`` delete package related logs
* ``GET /api/v1/packages/{package}/logs`` create log record for the package
* ``POST /api/v1/packages/{package}/logs`` get last package logs
* ``GET /api/v1/status`` get service status itself
* ``POST /api/v1/status`` update service status itself
@ -94,6 +99,10 @@ def setup_routes(application: Application, static_path: Path) -> None:
application.router.add_get("/api/v1/packages/{package}", PackageView, allow_head=True)
application.router.add_post("/api/v1/packages/{package}", PackageView)
application.router.add_delete("/api/v1/packages/{package}/logs", LogsView)
application.router.add_get("/api/v1/packages/{package}/logs", LogsView, allow_head=True)
application.router.add_post("/api/v1/packages/{package}/logs", LogsView)
application.router.add_get("/api/v1/status", StatusView, allow_head=True)
application.router.add_post("/api/v1/status", StatusView)

105
src/ahriman/web/views/status/logs.py Normal file
View File

@ -0,0 +1,105 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import HTTPBadRequest, HTTPNoContent, Response, json_response
from aiohttp.web_exceptions import HTTPNotFound
from ahriman.core.exceptions import UnknownPackageError
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.user_access import UserAccess
from ahriman.web.views.base import BaseView
class LogsView(BaseView):
"""
package logs web view
Attributes:
DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
HEAD_PERMISSION(UserAccess): (class attribute) head permissions of self
POST_PERMISSION(UserAccess): (class attribute) post permissions of self
"""
DELETE_PERMISSION = POST_PERMISSION = UserAccess.Full
GET_PERMISSION = HEAD_PERMISSION = UserAccess.Reporter
async def delete(self) -> None:
"""
delete package logs
Raises:
HTTPNoContent: on success response
"""
package_base = self.request.match_info["package"]
self.service.remove_logs(package_base, None)
raise HTTPNoContent()
async def get(self) -> Response:
"""
get last package logs
Returns:
Response: 200 with package logs on success
"""
package_base = self.request.match_info["package"]
try:
_, status = self.service.get(package_base)
except UnknownPackageError:
raise HTTPNotFound()
logs = self.service.get_logs(package_base)
response = {
"package_base": package_base,
"status": status.view(),
"logs": logs
}
return json_response(response)
async def post(self) -> None:
"""
create new package log record
JSON body must be supplied, the following model is used::
{
"created": 42.001, # log record created timestamp
"message": "log message", # log record
"process_id": 42 # process id from which log record was emitted
}
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
package_base = self.request.match_info["package"]
data = await self.extract_data()
try:
created = data["created"]
record = data["message"]
process_id = data["process_id"]
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.service.update_logs(LogRecordId(package_base, process_id), created, record)
raise HTTPNoContent()

34
src/ahriman/web/views/status/package.py
View File

@ -40,6 +40,18 @@ class PackageView(BaseView):
DELETE_PERMISSION = POST_PERMISSION = UserAccess.Full
GET_PERMISSION = HEAD_PERMISSION = UserAccess.Read
async def delete(self) -> None:
"""
delete package base from status page
Raises:
HTTPNoContent: on success response
"""
package_base = self.request.match_info["package"]
self.service.remove(package_base)
raise HTTPNoContent()
async def get(self) -> Response:
"""
get current package base status
@ -50,10 +62,10 @@ class PackageView(BaseView):
Raises:
HTTPNotFound: if no package was found
"""
base = self.request.match_info["package"]
package_base = self.request.match_info["package"]
try:
package, status = self.service.get(base)
package, status = self.service.get(package_base)
except UnknownPackageError:
raise HTTPNotFound()
@ -65,18 +77,6 @@ class PackageView(BaseView):
]
return json_response(response)
async def delete(self) -> None:
"""
delete package base from status page
Raises:
HTTPNoContent: on success response
"""
base = self.request.match_info["package"]
self.service.remove(base)
raise HTTPNoContent()
async def post(self) -> None:
"""
update package build status
@ -93,7 +93,7 @@ class PackageView(BaseView):
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
base = self.request.match_info["package"]
package_base = self.request.match_info["package"]
data = await self.extract_data()
try:
@ -103,8 +103,8 @@ class PackageView(BaseView):
raise HTTPBadRequest(reason=str(e))
try:
self.service.update(base, status, package)
self.service.update(package_base, status, package)
except UnknownPackageError:
raise HTTPBadRequest(reason=f"Package {base} is unknown, but no package body set")
raise HTTPBadRequest(reason=f"Package {package_base} is unknown, but no package body set")
raise HTTPNoContent()

3
src/ahriman/web/web.py
View File

@ -27,6 +27,7 @@ from ahriman.core.auth import Auth
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import InitializeError
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
from ahriman.core.spawn import Spawn
from ahriman.core.status.watcher import Watcher
from ahriman.web.middlewares.exception_handler import exception_handler
@ -79,7 +80,7 @@ def run_server(application: web.Application) -> None:
port = configuration.getint("web", "port")
web.run_app(application, host=host, port=port, handle_signals=False,
access_log=logging.getLogger("http"))
access_log=logging.getLogger("http"), access_log_class=FilteredAccessLogger)
def setup_service(architecture: str, configuration: Configuration, spawner: Spawn) -> web.Application: