From 71196dc58b201ee6d00f65195a27d508457fc32a Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sat, 20 Mar 2021 05:42:33 +0300 Subject: [PATCH] add watcher cache support --- src/ahriman/core/watcher/watcher.py | 55 +++++++++++++++++++- src/ahriman/models/build_status.py | 12 ++++- src/ahriman/models/package.py | 78 +++++++++++++++++++---------- src/ahriman/web/views/ahriman.py | 2 +- src/ahriman/web/views/base.py | 30 ----------- src/ahriman/web/views/package.py | 7 ++- src/ahriman/web/views/packages.py | 6 ++- 7 files changed, 126 insertions(+), 64 deletions(-) diff --git a/src/ahriman/core/watcher/watcher.py b/src/ahriman/core/watcher/watcher.py index db0d3156..acabb719 100644 --- a/src/ahriman/core/watcher/watcher.py +++ b/src/ahriman/core/watcher/watcher.py @@ -17,7 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from typing import Dict, List, Optional, Tuple +import json +import logging +import os + +from typing import Any, Dict, List, Optional, Tuple from ahriman.core.configuration import Configuration from ahriman.core.repository import Repository @@ -30,7 +34,9 @@ class Watcher: package status watcher :ivar architecture: repository architecture :ivar known: list of known packages. For the most cases `packages` should be used instead + :ivar logger: class logger :ivar repository: repository object + :ivar status: daemon status ''' def __init__(self, architecture: str, config: Configuration) -> None: @@ -39,12 +45,21 @@ class Watcher: :param architecture: repository architecture :param config: configuration instance ''' + self.logger = logging.getLogger('http') + self.architecture = architecture self.repository = Repository(architecture, config) self.known: Dict[str, Tuple[Package, BuildStatus]] = {} self.status = BuildStatus() + @property + def cache_path(self) -> str: + ''' + :return: path to dump with json cache + ''' + return os.path.join(self.repository.paths.root, 'cache.json') + @property def packages(self) -> List[Tuple[Package, BuildStatus]]: ''' @@ -52,6 +67,41 @@ class Watcher: ''' return list(self.known.values()) + def _cache_load(self) -> None: + ''' + update current state from cache + ''' + def parse_single(properties: Dict[str, Any]) -> None: + package = Package.from_json(properties['package']) + status = BuildStatus(**properties['status']) + if package.base in self.known: + self.known[package.base] = (package, status) + + if not os.path.isfile(self.cache_path): + return + with open(self.cache_path) as cache: + dump = json.load(cache) + for item in dump['packages']: + try: + parse_single(item) + except Exception: + self.logger.exception(f'cannot parse item f{item} to package', exc_info=True) + + def _cache_save(self) -> None: + ''' + dump current cache to filesystem + ''' + dump = { + 'packages': [ + { + 'package': package.view(), + 'status': status.view() + } for package, status in self.packages + ] + } + with open(self.cache_path, 'w') as cache: + json.dump(dump, cache) + def get(self, base: str) -> Tuple[Package, BuildStatus]: ''' get current package base build status @@ -71,6 +121,7 @@ class Watcher: else: _, status = current self.known[package.base] = (package, status) + self._cache_load() def remove(self, base: str) -> None: ''' @@ -78,6 +129,7 @@ class Watcher: :param base: package base ''' self.known.pop(base, None) + self._cache_save() def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None: ''' @@ -90,6 +142,7 @@ class Watcher: package, _ = self.known[base] full_status = BuildStatus(status) self.known[base] = (package, full_status) + self._cache_save() def update_self(self, status: BuildStatusEnum) -> None: ''' diff --git a/src/ahriman/models/build_status.py b/src/ahriman/models/build_status.py index 52ca253d..4a6bb028 100644 --- a/src/ahriman/models/build_status.py +++ b/src/ahriman/models/build_status.py @@ -20,7 +20,7 @@ import datetime from enum import Enum -from typing import Optional, Union +from typing import Any, Dict, Optional, Union class BuildStatusEnum(Enum): @@ -71,3 +71,13 @@ class BuildStatus: ''' self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown self.timestamp = timestamp or int(datetime.datetime.utcnow().timestamp()) + + def view(self) -> Dict[str, Any]: + ''' + generate json status view + :return: json-friendly dictionary + ''' + return { + 'status': self.status.value, + 'timestamp': self.timestamp + } diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index a3999594..2f66dea4 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -23,10 +23,10 @@ import aur # type: ignore import logging import os -from dataclasses import dataclass +from dataclasses import asdict, dataclass from pyalpm import vercmp # type: ignore from srcinfo.parse import parse_srcinfo # type: ignore -from typing import Dict, List, Optional, Set, Type +from typing import Any, Dict, List, Optional, Set, Type from ahriman.core.alpm.pacman import Pacman from ahriman.core.exceptions import InvalidPackageInfo @@ -76,31 +76,6 @@ class Package: ''' return f'{self.aur_url}/packages/{self.base}' - def actual_version(self, paths: RepositoryPaths) -> str: - ''' - additional method to handle VCS package versions - :param paths: repository paths instance - :return: package version if package is not VCS and current version according to VCS otherwise - ''' - if not self.is_vcs: - return self.version - - from ahriman.core.build_tools.task import Task - - clone_dir = os.path.join(paths.cache, self.base) - logger = logging.getLogger('build_details') - Task.fetch(clone_dir, self.git_url) - - # update pkgver first - check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger) - # generate new .SRCINFO and put it to parser - srcinfo_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger) - srcinfo, errors = parse_srcinfo(srcinfo_source) - if errors: - raise InvalidPackageInfo(errors) - - return self.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel']) - @classmethod def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package: ''' @@ -142,6 +117,23 @@ class Package: return cls(srcinfo['pkgbase'], version, aur_url, packages) + @classmethod + def from_json(cls: Type[Package], dump: Dict[str, Any]) -> Package: + ''' + construct package properties from json dump + :param dump: json dump body + :return: package properties + ''' + packages = { + key: PackageDescription(**value) + for key, value in dump.get('packages', {}) + } + return Package( + base=dump['base'], + version=dump['version'], + aur_url=dump['aur_url'], + packages=packages) + @staticmethod def dependencies(path: str) -> Set[str]: ''' @@ -196,6 +188,31 @@ class Package: except Exception as e: raise InvalidPackageInfo(str(e)) + def actual_version(self, paths: RepositoryPaths) -> str: + ''' + additional method to handle VCS package versions + :param paths: repository paths instance + :return: package version if package is not VCS and current version according to VCS otherwise + ''' + if not self.is_vcs: + return self.version + + from ahriman.core.build_tools.task import Task + + clone_dir = os.path.join(paths.cache, self.base) + logger = logging.getLogger('build_details') + Task.fetch(clone_dir, self.git_url) + + # update pkgver first + check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger) + # generate new .SRCINFO and put it to parser + srcinfo_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger) + srcinfo, errors = parse_srcinfo(srcinfo_source) + if errors: + raise InvalidPackageInfo(errors) + + return self.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel']) + def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool: ''' check if package is out-of-dated @@ -206,3 +223,10 @@ class Package: remote_version = remote.actual_version(paths) # either normal version or updated VCS result: int = vercmp(self.version, remote_version) return result < 0 + + def view(self) -> Dict[str, Any]: + ''' + generate json package view + :return: json-friendly dictionary + ''' + return asdict(self) diff --git a/src/ahriman/web/views/ahriman.py b/src/ahriman/web/views/ahriman.py index 2f190624..1fb077fb 100644 --- a/src/ahriman/web/views/ahriman.py +++ b/src/ahriman/web/views/ahriman.py @@ -33,7 +33,7 @@ class AhrimanView(BaseView): get current service status :return: 200 with service status object ''' - return json_response(AhrimanView.status_view(self.service.status)) + return json_response(self.service.status.view()) async def post(self) -> Response: ''' diff --git a/src/ahriman/web/views/base.py b/src/ahriman/web/views/base.py index 3820b1c2..e5f31ab3 100644 --- a/src/ahriman/web/views/base.py +++ b/src/ahriman/web/views/base.py @@ -17,14 +17,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from dataclasses import asdict -from typing import Any, Dict - from aiohttp.web import View from ahriman.core.watcher.watcher import Watcher -from ahriman.models.build_status import BuildStatus -from ahriman.models.package import Package class BaseView(View): @@ -39,28 +34,3 @@ class BaseView(View): ''' watcher: Watcher = self.request.app['watcher'] return watcher - - @staticmethod - def package_view(package: Package, status: BuildStatus) -> Dict[str, Any]: - ''' - generate json package view - :param package: package definitions - :param status: package build status - :return: json-friendly dictionary - ''' - return { - 'status': BaseView.status_view(status), - 'package': asdict(package) - } - - @staticmethod - def status_view(status: BuildStatus) -> Dict[str, Any]: - ''' - generate json status view - :param status: build status - :return: json-friendly dictionary - ''' - return { - 'status': status.status.value, - 'timestamp': status.timestamp - } diff --git a/src/ahriman/web/views/package.py b/src/ahriman/web/views/package.py index 179b795e..f87cbdd5 100644 --- a/src/ahriman/web/views/package.py +++ b/src/ahriman/web/views/package.py @@ -41,7 +41,10 @@ class PackageView(BaseView): except KeyError: raise HTTPNotFound() - response = PackageView.package_view(package, status) + response = { + 'package': package.view(), + 'status': status.view() + } return json_response(response) async def delete(self) -> Response: @@ -71,7 +74,7 @@ class PackageView(BaseView): data = await self.request.json() try: - package = Package(**data['package']) if 'package' in data else None + package = Package.from_json(data['package']) if 'package' in data else None status = BuildStatusEnum(data['status']) except Exception as e: raise HTTPBadRequest(text=str(e)) diff --git a/src/ahriman/web/views/packages.py b/src/ahriman/web/views/packages.py index 53d8d44f..2d21c688 100644 --- a/src/ahriman/web/views/packages.py +++ b/src/ahriman/web/views/packages.py @@ -33,8 +33,10 @@ class PackagesView(BaseView): :return: 200 with package description on success ''' response = [ - PackagesView.package_view(package, status) - for package, status in self.service.packages + { + 'package': package.view(), + 'status': status.view() + } for package, status in self.service.packages ] return json_response(response)