improve VCS packages checks

* Unlike older version, currently service will always try to pull AUR
  package to check version. Previously if no-vcs flag is set, it would
  ignore VCS packages completelly
* Introduce build.vcs_allowed_age option. If set, it will skip version
  calculation if package age (now - build_date) is less than this value

docs/configuration.rst

@@ -59,6 +59,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build:
 * ``makepkg_flags`` - additional flags passed to ``makepkg`` command, space separated list of strings, optional.
 * ``makechrootpkg_flags`` - additional flags passed to ``makechrootpkg`` command, space separated list of strings, optional.
 * ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention.
+* ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, int, optional, default ``0``.
 
 ``repository`` group
 --------------------

package/share/ahriman/settings/ahriman.ini

@@ -24,6 +24,7 @@ ignore_packages =
 makechrootpkg_flags =
 makepkg_flags = --nocolor --ignorearch
 triggers = ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger
+vcs_allowed_age = 604800
 
 [repository]
 name = aur-clone

src/ahriman/core/build_tools/sources.py

@@ -17,14 +17,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import datetime
 import shutil
 
 from pathlib import Path
 from typing import List, Optional
 
 from ahriman.core.log import LazyLogging
-from ahriman.core.util import check_output, walk
+from ahriman.core.util import check_output, utcnow, walk
 from ahriman.models.package import Package
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 from ahriman.models.remote_source import RemoteSource
@@ -215,7 +214,7 @@ class Sources(LazyLogging):
             author(Optional[str], optional): optional commit author if any (Default value = None)
         """
         if message is None:
-            message = f"Autogenerated commit at {datetime.datetime.utcnow()}"
+            message = f"Autogenerated commit at {utcnow()}"
         args = ["--allow-empty", "--message", message]
         if author is not None:
             args.extend(["--author", author])

src/ahriman/core/report/email.py

@@ -17,7 +17,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import datetime
 import smtplib
 
 from email.mime.multipart import MIMEMultipart
@@ -27,7 +26,7 @@ from typing import Dict, Iterable
 from ahriman.core.configuration import Configuration
 from ahriman.core.report.jinja_template import JinjaTemplate
 from ahriman.core.report.report import Report
-from ahriman.core.util import pretty_datetime
+from ahriman.core.util import pretty_datetime, utcnow
 from ahriman.models.package import Package
 from ahriman.models.result import Result
 from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
@@ -86,7 +85,7 @@ class Email(Report, JinjaTemplate):
         message = MIMEMultipart()
         message["From"] = self.sender
         message["To"] = ", ".join(self.receivers)
-        message["Subject"] = f"{self.name} build report at {pretty_datetime(datetime.datetime.utcnow())}"
+        message["Subject"] = f"{self.name} build report at {pretty_datetime(utcnow())}"
 
         message.attach(MIMEText(text, "html"))
         for filename, content in attachment.items():

src/ahriman/core/repository/repository_properties.py

@@ -45,6 +45,7 @@ class RepositoryProperties(LazyLogging):
         reporter(Client): build status reporter instance
         sign(GPG): GPG wrapper instance
         triggers(TriggerLoader): triggers holder
+        vcs_allowed_age(int): maximal age of the VCS packages before they will be checked
     """
 
     def __init__(self, architecture: str, configuration: Configuration, database: SQLite, *,
@@ -65,6 +66,7 @@ class RepositoryProperties(LazyLogging):
         self.database = database
 
         self.name = configuration.get("repository", "name")
+        self.vcs_allowed_age = configuration.getint("build", "vcs_allowed_age", fallback=0)
 
         self.paths = configuration.repository_paths
         try:

src/ahriman/core/repository/update_handler.py

@@ -21,6 +21,7 @@ from typing import Iterable, List
 
 from ahriman.core.build_tools.sources import Sources
 from ahriman.core.repository.cleaner import Cleaner
+from ahriman.core.util import utcnow
 from ahriman.models.package import Package
 from ahriman.models.package_source import PackageSource
 
@@ -53,14 +54,16 @@ class UpdateHandler(Cleaner):
         Returns:
             List[Package]: list of packages which are out-of-dated
         """
+        # don't think there are packages older then 1970
+        now = utcnow()
+        min_vcs_build_date = (now.timestamp() - self.vcs_allowed_age) if vcs else now.timestamp()
+
         result: List[Package] = []
 
         for local in self.packages():
             with self.in_package_context(local.base):
                 if local.base in self.ignore_list:
                     continue
-                if local.is_vcs and not vcs:
-                    continue
                 if filter_packages and local.base not in filter_packages:
                     continue
                 source = local.remote.source if local.remote is not None else None
@@ -70,7 +73,12 @@ class UpdateHandler(Cleaner):
                         remote = Package.from_official(local.base, self.pacman)
                     else:
                         remote = Package.from_aur(local.base, self.pacman)
-                    if local.is_outdated(remote, self.paths):
+
+                    calculate_version = not local.is_newer_than(min_vcs_build_date)
+                    self.logger.debug("set VCS version calculation for %s to %s having minimal build date %s",
+                                      local.base, calculate_version, min_vcs_build_date)
+
+                    if local.is_outdated(remote, self.paths, calculate_version=calculate_version):
                         self.reporter.set_pending(local.base)
                         result.append(remote)
                 except Exception:
@@ -99,7 +107,7 @@ class UpdateHandler(Cleaner):
                     if local is None:
                         self.reporter.set_unknown(remote)
                         result.append(remote)
-                    elif local.is_outdated(remote, self.paths):
+                    elif local.is_outdated(remote, self.paths, calculate_version=True):
                         self.reporter.set_pending(local.base)
                         result.append(remote)
                 except Exception:

src/ahriman/core/util.py

@@ -34,8 +34,8 @@ from ahriman.core.exceptions import OptionError, UnsafeRunError
 from ahriman.models.repository_paths import RepositoryPaths
 
 
-__all__ = ["check_output", "check_user", "exception_response_text", "filter_json", "full_version", "enum_values",
+__all__ = ["check_output", "check_user", "enum_values", "exception_response_text", "filter_json", "full_version",
-           "package_like", "pretty_datetime", "pretty_size", "safe_filename", "walk"]
+           "package_like", "pretty_datetime", "pretty_size", "safe_filename", "utcnow", "walk"]
 
 
 def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optional[Path] = None,
@@ -295,6 +295,16 @@ def safe_filename(source: str) -> str:
     return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source)
 
 
+def utcnow() -> datetime.datetime:
+    """
+    get current time
+
+    Returns:
+        datetime.datetime: current time in UTC
+    """
+    return datetime.datetime.utcnow()
+
+
 def walk(directory_path: Path) -> Generator[Path, None, None]:
     """
     list all file paths in given directory

src/ahriman/models/build_status.py

@@ -19,13 +19,11 @@
 #
 from __future__ import annotations
 
-import datetime
-
 from dataclasses import dataclass, field, fields
 from enum import Enum
 from typing import Any, Dict, Type
 
-from ahriman.core.util import filter_json, pretty_datetime
+from ahriman.core.util import filter_json, pretty_datetime, utcnow
 
 
 class BuildStatusEnum(str, Enum):
@@ -58,7 +56,7 @@ class BuildStatus:
     """
 
     status: BuildStatusEnum = BuildStatusEnum.Unknown
-    timestamp: int = field(default_factory=lambda: int(datetime.datetime.utcnow().timestamp()))
+    timestamp: int = field(default_factory=lambda: int(utcnow().timestamp()))
 
     def __post_init__(self) -> None:
         """

src/ahriman/models/package.py

@@ -17,6 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+# pylint: disable=too-many-lines
 from __future__ import annotations
 
 import copy
@@ -25,7 +26,7 @@ from dataclasses import asdict, dataclass
 from pathlib import Path
 from pyalpm import vercmp  # type: ignore
 from srcinfo.parse import parse_srcinfo  # type: ignore
-from typing import Any, Dict, Iterable, List, Optional, Set, Type
+from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union
 
 from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb
@@ -358,7 +359,24 @@ class Package(LazyLogging):
 
         return sorted(result)
 
-    def is_outdated(self, remote: Package, paths: RepositoryPaths, *, calculate_version: bool = True) -> bool:
+    def is_newer_than(self, timestamp: Union[float, int]) -> bool:
+        """
+        check if package was built after the specified timestamp
+
+        Args:
+            timestamp(int): timestamp to check build date against
+
+        Returns:
+            bool: True in case if package was built after the specified date and False otherwise. In case if build date
+                is not set by any of packages, it returns False
+        """
+        return any(
+            package.build_date > timestamp
+            for package in self.packages.values()
+            if package.build_date is not None
+        )
+
+    def is_outdated(self, remote: Package, paths: RepositoryPaths, *, calculate_version: bool) -> bool:
         """
         check if package is out-of-dated
 

tests/ahriman/core/repository/test_repository.py

@@ -17,6 +17,7 @@ def test_load(configuration: Configuration, database: SQLite, mocker: MockerFixt
     """
     must correctly load instance
     """
+    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     context_mock = mocker.patch("ahriman.core.repository.Repository._set_context")
     Repository.load("x86_64", configuration, database, report=False, unsafe=False)
     context_mock.assert_called_once_with()
@@ -26,6 +27,7 @@ def test_set_context(configuration: Configuration, database: SQLite, mocker: Moc
     """
     must set context variables
     """
+    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     set_mock = mocker.patch("ahriman.core._Context.set")
 
     instance = Repository.load("x86_64", configuration, database, report=False, unsafe=False)

tests/ahriman/core/repository/test_update_handler.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from pytest_mock import MockerFixture
 
 from ahriman.core.repository.update_handler import UpdateHandler
+from ahriman.core.util import utcnow
 from ahriman.models.package import Package
 from ahriman.models.package_source import PackageSource
 from ahriman.models.remote_source import RemoteSource
@@ -82,7 +83,7 @@ def test_updates_aur_ignore(update_handler: UpdateHandler, package_ahriman: Pack
     mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
     package_load_mock = mocker.patch("ahriman.models.package.Package.from_aur")
 
-    update_handler.updates_aur([], vcs=True)
+    assert not update_handler.updates_aur([], vcs=True)
     package_load_mock.assert_not_called()
 
 
@@ -92,11 +93,16 @@ def test_updates_aur_ignore_vcs(update_handler: UpdateHandler, package_ahriman:
     must skip VCS packages check if requested
     """
     mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
+    mocker.patch("ahriman.models.package.Package.from_aur", return_value=package_ahriman)
     mocker.patch("ahriman.models.package.Package.is_vcs", return_value=True)
-    package_is_outdated_mock = mocker.patch("ahriman.models.package.Package.is_outdated")
+    package_is_newer_than_mock = mocker.patch("ahriman.models.package.Package.is_newer_than", return_value=True)
+    package_is_outdated_mock = mocker.patch("ahriman.models.package.Package.is_outdated", return_value=False)
+    ts1 = utcnow().timestamp()
 
-    update_handler.updates_aur([], vcs=False)
+    assert not update_handler.updates_aur([], vcs=False)
-    package_is_outdated_mock.assert_not_called()
+    package_is_newer_than_mock.assert_called_once_with(pytest.helpers.anyvar(float, strict=True))
+    assert ts1 < package_is_newer_than_mock.call_args[0][0] < utcnow().timestamp()
+    package_is_outdated_mock.assert_called_once_with(package_ahriman, update_handler.paths, calculate_version=False)
 
 
 def test_updates_local(update_handler: UpdateHandler, package_ahriman: Package, mocker: MockerFixture) -> None:

tests/ahriman/core/test_util.py

@@ -11,8 +11,8 @@ from typing import Any
 from unittest.mock import MagicMock
 
 from ahriman.core.exceptions import BuildError, OptionError, UnsafeRunError
-from ahriman.core.util import check_output, check_user, exception_response_text, filter_json, full_version, \
+from ahriman.core.util import check_output, check_user, enum_values, exception_response_text, filter_json, \
-    enum_values, package_like, pretty_datetime, pretty_size, safe_filename, walk
+    full_version, package_like, pretty_datetime, pretty_size, safe_filename, utcnow, walk
 from ahriman.models.package import Package
 from ahriman.models.package_source import PackageSource
 from ahriman.models.repository_paths import RepositoryPaths
@@ -322,6 +322,15 @@ def test_safe_filename() -> None:
     assert safe_filename("tolua++-1.0.93-4-x86_64.pkg.tar.zst") == "tolua---1.0.93-4-x86_64.pkg.tar.zst"
 
 
+def test_utcnow() -> None:
+    """
+    must generate correct timestamp
+    """
+    ts1 = utcnow()
+    ts2 = utcnow()
+    assert 1 > (ts2 - ts1).total_seconds() > 0
+
+
 def test_walk(resource_path_root: Path) -> None:
     """
     must traverse directory recursively

tests/ahriman/models/test_package.py

@@ -265,11 +265,31 @@ def test_full_depends(package_ahriman: Package, package_python_schedule: Package
     assert package_python_schedule.full_depends(pyalpm_handle, [package_python_schedule]) == expected
 
 
+def test_is_newer_than(package_ahriman: Package, package_python_schedule: Package) -> None:
+    """
+    must correctly check if package is newer than specified timestamp
+    """
+    # base checks, true/false
+    assert package_ahriman.is_newer_than(package_ahriman.packages[package_ahriman.base].build_date - 1)
+    assert not package_ahriman.is_newer_than(package_ahriman.packages[package_ahriman.base].build_date + 1)
+
+    # list check
+    min_date = min(package.build_date for package in package_python_schedule.packages.values())
+    assert package_python_schedule.is_newer_than(min_date)
+
+    # null list check
+    package_python_schedule.packages["python-schedule"].build_date = None
+    assert package_python_schedule.is_newer_than(min_date)
+
+    package_python_schedule.packages["python2-schedule"].build_date = None
+    assert not package_python_schedule.is_newer_than(min_date)
+
+
 def test_is_outdated_false(package_ahriman: Package, repository_paths: RepositoryPaths) -> None:
     """
     must be not outdated for the same package
     """
-    assert not package_ahriman.is_outdated(package_ahriman, repository_paths)
+    assert not package_ahriman.is_outdated(package_ahriman, repository_paths, calculate_version=True)
 
 
 def test_is_outdated_true(package_ahriman: Package, repository_paths: RepositoryPaths) -> None:
@@ -279,7 +299,7 @@ def test_is_outdated_true(package_ahriman: Package, repository_paths: Repository
     other = Package.from_json(package_ahriman.view())
     other.version = other.version.replace("-1", "-2")
 
-    assert package_ahriman.is_outdated(other, repository_paths)
+    assert package_ahriman.is_outdated(other, repository_paths, calculate_version=True)
 
 
 def test_build_status_pretty_print(package_ahriman: Package) -> None: