diff --git a/docs/configuration.rst b/docs/configuration.rst index bb3daf4c..427ae871 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -59,6 +59,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build: * ``makepkg_flags`` - additional flags passed to ``makepkg`` command, space separated list of strings, optional. * ``makechrootpkg_flags`` - additional flags passed to ``makechrootpkg`` command, space separated list of strings, optional. * ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention. +* ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, int, optional, default ``0``. ``repository`` group -------------------- diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini index df8ed7a2..757f00e3 100644 --- a/package/share/ahriman/settings/ahriman.ini +++ b/package/share/ahriman/settings/ahriman.ini @@ -24,6 +24,7 @@ ignore_packages = makechrootpkg_flags = makepkg_flags = --nocolor --ignorearch triggers = ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger +vcs_allowed_age = 604800 [repository] name = aur-clone diff --git a/src/ahriman/core/build_tools/sources.py b/src/ahriman/core/build_tools/sources.py index dcc882c5..911cf30d 100644 --- a/src/ahriman/core/build_tools/sources.py +++ b/src/ahriman/core/build_tools/sources.py @@ -17,14 +17,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import datetime import shutil from pathlib import Path from typing import List, Optional from ahriman.core.log import LazyLogging -from ahriman.core.util import check_output, walk +from ahriman.core.util import check_output, utcnow, walk from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.remote_source import RemoteSource @@ -215,7 +214,7 @@ class Sources(LazyLogging): author(Optional[str], optional): optional commit author if any (Default value = None) """ if message is None: - message = f"Autogenerated commit at {datetime.datetime.utcnow()}" + message = f"Autogenerated commit at {utcnow()}" args = ["--allow-empty", "--message", message] if author is not None: args.extend(["--author", author]) diff --git a/src/ahriman/core/report/email.py b/src/ahriman/core/report/email.py index 58a206f0..bc6c50ac 100644 --- a/src/ahriman/core/report/email.py +++ b/src/ahriman/core/report/email.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import datetime import smtplib from email.mime.multipart import MIMEMultipart @@ -27,7 +26,7 @@ from typing import Dict, Iterable from ahriman.core.configuration import Configuration from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.report.report import Report -from ahriman.core.util import pretty_datetime +from ahriman.core.util import pretty_datetime, utcnow from ahriman.models.package import Package from ahriman.models.result import Result from ahriman.models.smtp_ssl_settings import SmtpSSLSettings @@ -86,7 +85,7 @@ class Email(Report, JinjaTemplate): message = MIMEMultipart() message["From"] = self.sender message["To"] = ", ".join(self.receivers) - message["Subject"] = f"{self.name} build report at {pretty_datetime(datetime.datetime.utcnow())}" + message["Subject"] = f"{self.name} build report at {pretty_datetime(utcnow())}" message.attach(MIMEText(text, "html")) for filename, content in attachment.items(): diff --git a/src/ahriman/core/repository/repository_properties.py b/src/ahriman/core/repository/repository_properties.py index d4d25274..03197733 100644 --- a/src/ahriman/core/repository/repository_properties.py +++ b/src/ahriman/core/repository/repository_properties.py @@ -45,6 +45,7 @@ class RepositoryProperties(LazyLogging): reporter(Client): build status reporter instance sign(GPG): GPG wrapper instance triggers(TriggerLoader): triggers holder + vcs_allowed_age(int): maximal age of the VCS packages before they will be checked """ def __init__(self, architecture: str, configuration: Configuration, database: SQLite, *, @@ -65,6 +66,7 @@ class RepositoryProperties(LazyLogging): self.database = database self.name = configuration.get("repository", "name") + self.vcs_allowed_age = configuration.getint("build", "vcs_allowed_age", fallback=0) self.paths = configuration.repository_paths try: diff --git a/src/ahriman/core/repository/update_handler.py b/src/ahriman/core/repository/update_handler.py index 0d7b8f7f..f2f57653 100644 --- a/src/ahriman/core/repository/update_handler.py +++ b/src/ahriman/core/repository/update_handler.py @@ -21,6 +21,7 @@ from typing import Iterable, List from ahriman.core.build_tools.sources import Sources from ahriman.core.repository.cleaner import Cleaner +from ahriman.core.util import utcnow from ahriman.models.package import Package from ahriman.models.package_source import PackageSource @@ -53,14 +54,16 @@ class UpdateHandler(Cleaner): Returns: List[Package]: list of packages which are out-of-dated """ + # don't think there are packages older then 1970 + now = utcnow() + min_vcs_build_date = (now.timestamp() - self.vcs_allowed_age) if vcs else now.timestamp() + result: List[Package] = [] for local in self.packages(): with self.in_package_context(local.base): if local.base in self.ignore_list: continue - if local.is_vcs and not vcs: - continue if filter_packages and local.base not in filter_packages: continue source = local.remote.source if local.remote is not None else None @@ -70,7 +73,12 @@ class UpdateHandler(Cleaner): remote = Package.from_official(local.base, self.pacman) else: remote = Package.from_aur(local.base, self.pacman) - if local.is_outdated(remote, self.paths): + + calculate_version = not local.is_newer_than(min_vcs_build_date) + self.logger.debug("set VCS version calculation for %s to %s having minimal build date %s", + local.base, calculate_version, min_vcs_build_date) + + if local.is_outdated(remote, self.paths, calculate_version=calculate_version): self.reporter.set_pending(local.base) result.append(remote) except Exception: @@ -99,7 +107,7 @@ class UpdateHandler(Cleaner): if local is None: self.reporter.set_unknown(remote) result.append(remote) - elif local.is_outdated(remote, self.paths): + elif local.is_outdated(remote, self.paths, calculate_version=True): self.reporter.set_pending(local.base) result.append(remote) except Exception: diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index b47d61a1..2f3c24eb 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -34,8 +34,8 @@ from ahriman.core.exceptions import OptionError, UnsafeRunError from ahriman.models.repository_paths import RepositoryPaths -__all__ = ["check_output", "check_user", "exception_response_text", "filter_json", "full_version", "enum_values", - "package_like", "pretty_datetime", "pretty_size", "safe_filename", "walk"] +__all__ = ["check_output", "check_user", "enum_values", "exception_response_text", "filter_json", "full_version", + "package_like", "pretty_datetime", "pretty_size", "safe_filename", "utcnow", "walk"] def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optional[Path] = None, @@ -295,6 +295,16 @@ def safe_filename(source: str) -> str: return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source) +def utcnow() -> datetime.datetime: + """ + get current time + + Returns: + datetime.datetime: current time in UTC + """ + return datetime.datetime.utcnow() + + def walk(directory_path: Path) -> Generator[Path, None, None]: """ list all file paths in given directory diff --git a/src/ahriman/models/build_status.py b/src/ahriman/models/build_status.py index 76fd35b9..5797ad57 100644 --- a/src/ahriman/models/build_status.py +++ b/src/ahriman/models/build_status.py @@ -19,13 +19,11 @@ # from __future__ import annotations -import datetime - from dataclasses import dataclass, field, fields from enum import Enum from typing import Any, Dict, Type -from ahriman.core.util import filter_json, pretty_datetime +from ahriman.core.util import filter_json, pretty_datetime, utcnow class BuildStatusEnum(str, Enum): @@ -58,7 +56,7 @@ class BuildStatus: """ status: BuildStatusEnum = BuildStatusEnum.Unknown - timestamp: int = field(default_factory=lambda: int(datetime.datetime.utcnow().timestamp())) + timestamp: int = field(default_factory=lambda: int(utcnow().timestamp())) def __post_init__(self) -> None: """ diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 0e95aaa1..03ccb0c4 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +# pylint: disable=too-many-lines from __future__ import annotations import copy @@ -25,7 +26,7 @@ from dataclasses import asdict, dataclass from pathlib import Path from pyalpm import vercmp # type: ignore from srcinfo.parse import parse_srcinfo # type: ignore -from typing import Any, Dict, Iterable, List, Optional, Set, Type +from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb @@ -358,7 +359,24 @@ class Package(LazyLogging): return sorted(result) - def is_outdated(self, remote: Package, paths: RepositoryPaths, *, calculate_version: bool = True) -> bool: + def is_newer_than(self, timestamp: Union[float, int]) -> bool: + """ + check if package was built after the specified timestamp + + Args: + timestamp(int): timestamp to check build date against + + Returns: + bool: True in case if package was built after the specified date and False otherwise. In case if build date + is not set by any of packages, it returns False + """ + return any( + package.build_date > timestamp + for package in self.packages.values() + if package.build_date is not None + ) + + def is_outdated(self, remote: Package, paths: RepositoryPaths, *, calculate_version: bool) -> bool: """ check if package is out-of-dated diff --git a/tests/ahriman/core/repository/test_repository.py b/tests/ahriman/core/repository/test_repository.py index 1827cee6..ee05c55a 100644 --- a/tests/ahriman/core/repository/test_repository.py +++ b/tests/ahriman/core/repository/test_repository.py @@ -17,6 +17,7 @@ def test_load(configuration: Configuration, database: SQLite, mocker: MockerFixt """ must correctly load instance """ + mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") context_mock = mocker.patch("ahriman.core.repository.Repository._set_context") Repository.load("x86_64", configuration, database, report=False, unsafe=False) context_mock.assert_called_once_with() @@ -26,6 +27,7 @@ def test_set_context(configuration: Configuration, database: SQLite, mocker: Moc """ must set context variables """ + mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") set_mock = mocker.patch("ahriman.core._Context.set") instance = Repository.load("x86_64", configuration, database, report=False, unsafe=False) diff --git a/tests/ahriman/core/repository/test_update_handler.py b/tests/ahriman/core/repository/test_update_handler.py index 07b49e50..8bb987d6 100644 --- a/tests/ahriman/core/repository/test_update_handler.py +++ b/tests/ahriman/core/repository/test_update_handler.py @@ -4,6 +4,7 @@ from pathlib import Path from pytest_mock import MockerFixture from ahriman.core.repository.update_handler import UpdateHandler +from ahriman.core.util import utcnow from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource @@ -82,7 +83,7 @@ def test_updates_aur_ignore(update_handler: UpdateHandler, package_ahriman: Pack mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman]) package_load_mock = mocker.patch("ahriman.models.package.Package.from_aur") - update_handler.updates_aur([], vcs=True) + assert not update_handler.updates_aur([], vcs=True) package_load_mock.assert_not_called() @@ -92,11 +93,16 @@ def test_updates_aur_ignore_vcs(update_handler: UpdateHandler, package_ahriman: must skip VCS packages check if requested """ mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman]) + mocker.patch("ahriman.models.package.Package.from_aur", return_value=package_ahriman) mocker.patch("ahriman.models.package.Package.is_vcs", return_value=True) - package_is_outdated_mock = mocker.patch("ahriman.models.package.Package.is_outdated") + package_is_newer_than_mock = mocker.patch("ahriman.models.package.Package.is_newer_than", return_value=True) + package_is_outdated_mock = mocker.patch("ahriman.models.package.Package.is_outdated", return_value=False) + ts1 = utcnow().timestamp() - update_handler.updates_aur([], vcs=False) - package_is_outdated_mock.assert_not_called() + assert not update_handler.updates_aur([], vcs=False) + package_is_newer_than_mock.assert_called_once_with(pytest.helpers.anyvar(float, strict=True)) + assert ts1 < package_is_newer_than_mock.call_args[0][0] < utcnow().timestamp() + package_is_outdated_mock.assert_called_once_with(package_ahriman, update_handler.paths, calculate_version=False) def test_updates_local(update_handler: UpdateHandler, package_ahriman: Package, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/test_util.py b/tests/ahriman/core/test_util.py index dde08f63..20828a7b 100644 --- a/tests/ahriman/core/test_util.py +++ b/tests/ahriman/core/test_util.py @@ -11,8 +11,8 @@ from typing import Any from unittest.mock import MagicMock from ahriman.core.exceptions import BuildError, OptionError, UnsafeRunError -from ahriman.core.util import check_output, check_user, exception_response_text, filter_json, full_version, \ - enum_values, package_like, pretty_datetime, pretty_size, safe_filename, walk +from ahriman.core.util import check_output, check_user, enum_values, exception_response_text, filter_json, \ + full_version, package_like, pretty_datetime, pretty_size, safe_filename, utcnow, walk from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.repository_paths import RepositoryPaths @@ -322,6 +322,15 @@ def test_safe_filename() -> None: assert safe_filename("tolua++-1.0.93-4-x86_64.pkg.tar.zst") == "tolua---1.0.93-4-x86_64.pkg.tar.zst" +def test_utcnow() -> None: + """ + must generate correct timestamp + """ + ts1 = utcnow() + ts2 = utcnow() + assert 1 > (ts2 - ts1).total_seconds() > 0 + + def test_walk(resource_path_root: Path) -> None: """ must traverse directory recursively diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index d32c469d..a37df8ff 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -265,11 +265,31 @@ def test_full_depends(package_ahriman: Package, package_python_schedule: Package assert package_python_schedule.full_depends(pyalpm_handle, [package_python_schedule]) == expected +def test_is_newer_than(package_ahriman: Package, package_python_schedule: Package) -> None: + """ + must correctly check if package is newer than specified timestamp + """ + # base checks, true/false + assert package_ahriman.is_newer_than(package_ahriman.packages[package_ahriman.base].build_date - 1) + assert not package_ahriman.is_newer_than(package_ahriman.packages[package_ahriman.base].build_date + 1) + + # list check + min_date = min(package.build_date for package in package_python_schedule.packages.values()) + assert package_python_schedule.is_newer_than(min_date) + + # null list check + package_python_schedule.packages["python-schedule"].build_date = None + assert package_python_schedule.is_newer_than(min_date) + + package_python_schedule.packages["python2-schedule"].build_date = None + assert not package_python_schedule.is_newer_than(min_date) + + def test_is_outdated_false(package_ahriman: Package, repository_paths: RepositoryPaths) -> None: """ must be not outdated for the same package """ - assert not package_ahriman.is_outdated(package_ahriman, repository_paths) + assert not package_ahriman.is_outdated(package_ahriman, repository_paths, calculate_version=True) def test_is_outdated_true(package_ahriman: Package, repository_paths: RepositoryPaths) -> None: @@ -279,7 +299,7 @@ def test_is_outdated_true(package_ahriman: Package, repository_paths: Repository other = Package.from_json(package_ahriman.view()) other.version = other.version.replace("-1", "-2") - assert package_ahriman.is_outdated(other, repository_paths) + assert package_ahriman.is_outdated(other, repository_paths, calculate_version=True) def test_build_status_pretty_print(package_ahriman: Package) -> None: