add watcher cache support

src/ahriman/core/watcher/watcher.py

@@ -17,7 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Dict, List, Optional, Tuple
+import json
+import logging
+import os
+
+from typing import Any, Dict, List, Optional, Tuple
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.repository import Repository
@@ -30,7 +34,9 @@ class Watcher:
     package status watcher
     :ivar architecture: repository architecture
     :ivar known: list of known packages. For the most cases `packages` should be used instead
+    :ivar logger: class logger
     :ivar repository: repository object
+    :ivar status: daemon status
     '''
 
     def __init__(self, architecture: str, config: Configuration) -> None:
@@ -39,12 +45,21 @@ class Watcher:
         :param architecture: repository architecture
         :param config: configuration instance
         '''
+        self.logger = logging.getLogger('http')
+
         self.architecture = architecture
         self.repository = Repository(architecture, config)
 
         self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
         self.status = BuildStatus()
 
+    @property
+    def cache_path(self) -> str:
+        '''
+        :return: path to dump with json cache
+        '''
+        return os.path.join(self.repository.paths.root, 'cache.json')
+
     @property
     def packages(self) -> List[Tuple[Package, BuildStatus]]:
         '''
@@ -52,6 +67,41 @@ class Watcher:
         '''
         return list(self.known.values())
 
+    def _cache_load(self) -> None:
+        '''
+        update current state from cache
+        '''
+        def parse_single(properties: Dict[str, Any]) -> None:
+            package = Package.from_json(properties['package'])
+            status = BuildStatus(**properties['status'])
+            if package.base in self.known:
+                self.known[package.base] = (package, status)
+
+        if not os.path.isfile(self.cache_path):
+            return
+        with open(self.cache_path) as cache:
+            dump = json.load(cache)
+        for item in dump['packages']:
+            try:
+                parse_single(item)
+            except Exception:
+                self.logger.exception(f'cannot parse item f{item} to package', exc_info=True)
+
+    def _cache_save(self) -> None:
+        '''
+        dump current cache to filesystem
+        '''
+        dump = {
+            'packages': [
+                {
+                    'package': package.view(),
+                    'status': status.view()
+                } for package, status in self.packages
+            ]
+        }
+        with open(self.cache_path, 'w') as cache:
+            json.dump(dump, cache)
+
     def get(self, base: str) -> Tuple[Package, BuildStatus]:
         '''
         get current package base build status
@@ -71,6 +121,7 @@ class Watcher:
             else:
                 _, status = current
             self.known[package.base] = (package, status)
+        self._cache_load()
 
     def remove(self, base: str) -> None:
         '''
@@ -78,6 +129,7 @@ class Watcher:
         :param base: package base
         '''
         self.known.pop(base, None)
+        self._cache_save()
 
     def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
         '''
@@ -90,6 +142,7 @@ class Watcher:
             package, _ = self.known[base]
         full_status = BuildStatus(status)
         self.known[base] = (package, full_status)
+        self._cache_save()
 
     def update_self(self, status: BuildStatusEnum) -> None:
         '''

src/ahriman/models/build_status.py

@@ -20,7 +20,7 @@
 import datetime
 
 from enum import Enum
-from typing import Optional, Union
+from typing import Any, Dict, Optional, Union
 
 
 class BuildStatusEnum(Enum):
@@ -71,3 +71,13 @@ class BuildStatus:
         '''
         self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown
         self.timestamp = timestamp or int(datetime.datetime.utcnow().timestamp())
+
+    def view(self) -> Dict[str, Any]:
+        '''
+        generate json status view
+        :return: json-friendly dictionary
+        '''
+        return {
+            'status': self.status.value,
+            'timestamp': self.timestamp
+        }

src/ahriman/models/package.py

@@ -23,10 +23,10 @@ import aur  # type: ignore
 import logging
 import os
 
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from pyalpm import vercmp  # type: ignore
 from srcinfo.parse import parse_srcinfo  # type: ignore
-from typing import Dict, List, Optional, Set, Type
+from typing import Any, Dict, List, Optional, Set, Type
 
 from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.exceptions import InvalidPackageInfo
@@ -76,31 +76,6 @@ class Package:
         '''
         return f'{self.aur_url}/packages/{self.base}'
 
-    def actual_version(self, paths: RepositoryPaths) -> str:
-        '''
-        additional method to handle VCS package versions
-        :param paths: repository paths instance
-        :return: package version if package is not VCS and current version according to VCS otherwise
-        '''
-        if not self.is_vcs:
-            return self.version
-
-        from ahriman.core.build_tools.task import Task
-
-        clone_dir = os.path.join(paths.cache, self.base)
-        logger = logging.getLogger('build_details')
-        Task.fetch(clone_dir, self.git_url)
-
-        # update pkgver first
-        check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger)
-        # generate new .SRCINFO and put it to parser
-        srcinfo_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger)
-        srcinfo, errors = parse_srcinfo(srcinfo_source)
-        if errors:
-            raise InvalidPackageInfo(errors)
-
-        return self.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel'])
-
     @classmethod
     def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
         '''
@@ -142,6 +117,23 @@ class Package:
 
         return cls(srcinfo['pkgbase'], version, aur_url, packages)
 
+    @classmethod
+    def from_json(cls: Type[Package], dump: Dict[str, Any]) -> Package:
+        '''
+        construct package properties from json dump
+        :param dump: json dump body
+        :return: package properties
+        '''
+        packages = {
+            key: PackageDescription(**value)
+            for key, value in dump.get('packages', {})
+        }
+        return Package(
+            base=dump['base'],
+            version=dump['version'],
+            aur_url=dump['aur_url'],
+            packages=packages)
+
     @staticmethod
     def dependencies(path: str) -> Set[str]:
         '''
@@ -196,6 +188,31 @@ class Package:
         except Exception as e:
             raise InvalidPackageInfo(str(e))
 
+    def actual_version(self, paths: RepositoryPaths) -> str:
+        '''
+        additional method to handle VCS package versions
+        :param paths: repository paths instance
+        :return: package version if package is not VCS and current version according to VCS otherwise
+        '''
+        if not self.is_vcs:
+            return self.version
+
+        from ahriman.core.build_tools.task import Task
+
+        clone_dir = os.path.join(paths.cache, self.base)
+        logger = logging.getLogger('build_details')
+        Task.fetch(clone_dir, self.git_url)
+
+        # update pkgver first
+        check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger)
+        # generate new .SRCINFO and put it to parser
+        srcinfo_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger)
+        srcinfo, errors = parse_srcinfo(srcinfo_source)
+        if errors:
+            raise InvalidPackageInfo(errors)
+
+        return self.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel'])
+
     def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool:
         '''
         check if package is out-of-dated
@@ -206,3 +223,10 @@ class Package:
         remote_version = remote.actual_version(paths)  # either normal version or updated VCS
         result: int = vercmp(self.version, remote_version)
         return result < 0
+
+    def view(self) -> Dict[str, Any]:
+        '''
+        generate json package view
+        :return: json-friendly dictionary
+        '''
+        return asdict(self)

src/ahriman/web/views/ahriman.py

@@ -33,7 +33,7 @@ class AhrimanView(BaseView):
         get current service status
         :return: 200 with service status object
         '''
-        return json_response(AhrimanView.status_view(self.service.status))
+        return json_response(self.service.status.view())
 
     async def post(self) -> Response:
         '''

src/ahriman/web/views/base.py

@@ -17,14 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from dataclasses import asdict
-from typing import Any, Dict
-
 from aiohttp.web import View
 
 from ahriman.core.watcher.watcher import Watcher
-from ahriman.models.build_status import BuildStatus
-from ahriman.models.package import Package
 
 
 class BaseView(View):
@@ -39,28 +34,3 @@ class BaseView(View):
         '''
         watcher: Watcher = self.request.app['watcher']
         return watcher
-
-    @staticmethod
-    def package_view(package: Package, status: BuildStatus) -> Dict[str, Any]:
-        '''
-        generate json package view
-        :param package: package definitions
-        :param status: package build status
-        :return: json-friendly dictionary
-        '''
-        return {
-            'status': BaseView.status_view(status),
-            'package': asdict(package)
-        }
-
-    @staticmethod
-    def status_view(status: BuildStatus) -> Dict[str, Any]:
-        '''
-        generate json status view
-        :param status: build status
-        :return: json-friendly dictionary
-        '''
-        return {
-            'status': status.status.value,
-            'timestamp': status.timestamp
-        }

src/ahriman/web/views/package.py

@@ -41,7 +41,10 @@ class PackageView(BaseView):
         except KeyError:
             raise HTTPNotFound()
 
-        response = PackageView.package_view(package, status)
+        response = {
+            'package': package.view(),
+            'status': status.view()
+        }
         return json_response(response)
 
     async def delete(self) -> Response:
@@ -71,7 +74,7 @@ class PackageView(BaseView):
         data = await self.request.json()
 
         try:
-            package = Package(**data['package']) if 'package' in data else None
+            package = Package.from_json(data['package']) if 'package' in data else None
             status = BuildStatusEnum(data['status'])
         except Exception as e:
             raise HTTPBadRequest(text=str(e))

src/ahriman/web/views/packages.py

@@ -33,8 +33,10 @@ class PackagesView(BaseView):
         :return: 200 with package description on success
         '''
         response = [
-            PackagesView.package_view(package, status)
+            {
-            for package, status in self.service.packages
+                'package': package.view(),
+                'status': status.view()
+            } for package, status in self.service.packages
         ]
         return json_response(response)