improve VCS packages checks

* Unlike older version, currently service will always try to pull AUR
  package to check version. Previously if no-vcs flag is set, it would
  ignore VCS packages completelly
* Introduce build.vcs_allowed_age option. If set, it will skip version
  calculation if package age (now - build_date) is less than this value
This commit is contained in:
2022-12-29 03:12:04 +02:00
parent 4b27e102ef
commit effc120a43
13 changed files with 99 additions and 26 deletions

View File

@ -17,14 +17,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
import shutil
from pathlib import Path
from typing import List, Optional
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, walk
from ahriman.core.util import check_output, utcnow, walk
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.remote_source import RemoteSource
@ -215,7 +214,7 @@ class Sources(LazyLogging):
author(Optional[str], optional): optional commit author if any (Default value = None)
"""
if message is None:
message = f"Autogenerated commit at {datetime.datetime.utcnow()}"
message = f"Autogenerated commit at {utcnow()}"
args = ["--allow-empty", "--message", message]
if author is not None:
args.extend(["--author", author])

View File

@ -17,7 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
import smtplib
from email.mime.multipart import MIMEMultipart
@ -27,7 +26,7 @@ from typing import Dict, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.core.util import pretty_datetime
from ahriman.core.util import pretty_datetime, utcnow
from ahriman.models.package import Package
from ahriman.models.result import Result
from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
@ -86,7 +85,7 @@ class Email(Report, JinjaTemplate):
message = MIMEMultipart()
message["From"] = self.sender
message["To"] = ", ".join(self.receivers)
message["Subject"] = f"{self.name} build report at {pretty_datetime(datetime.datetime.utcnow())}"
message["Subject"] = f"{self.name} build report at {pretty_datetime(utcnow())}"
message.attach(MIMEText(text, "html"))
for filename, content in attachment.items():

View File

@ -45,6 +45,7 @@ class RepositoryProperties(LazyLogging):
reporter(Client): build status reporter instance
sign(GPG): GPG wrapper instance
triggers(TriggerLoader): triggers holder
vcs_allowed_age(int): maximal age of the VCS packages before they will be checked
"""
def __init__(self, architecture: str, configuration: Configuration, database: SQLite, *,
@ -65,6 +66,7 @@ class RepositoryProperties(LazyLogging):
self.database = database
self.name = configuration.get("repository", "name")
self.vcs_allowed_age = configuration.getint("build", "vcs_allowed_age", fallback=0)
self.paths = configuration.repository_paths
try:

View File

@ -21,6 +21,7 @@ from typing import Iterable, List
from ahriman.core.build_tools.sources import Sources
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.util import utcnow
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
@ -53,14 +54,16 @@ class UpdateHandler(Cleaner):
Returns:
List[Package]: list of packages which are out-of-dated
"""
# don't think there are packages older then 1970
now = utcnow()
min_vcs_build_date = (now.timestamp() - self.vcs_allowed_age) if vcs else now.timestamp()
result: List[Package] = []
for local in self.packages():
with self.in_package_context(local.base):
if local.base in self.ignore_list:
continue
if local.is_vcs and not vcs:
continue
if filter_packages and local.base not in filter_packages:
continue
source = local.remote.source if local.remote is not None else None
@ -70,7 +73,12 @@ class UpdateHandler(Cleaner):
remote = Package.from_official(local.base, self.pacman)
else:
remote = Package.from_aur(local.base, self.pacman)
if local.is_outdated(remote, self.paths):
calculate_version = not local.is_newer_than(min_vcs_build_date)
self.logger.debug("set VCS version calculation for %s to %s having minimal build date %s",
local.base, calculate_version, min_vcs_build_date)
if local.is_outdated(remote, self.paths, calculate_version=calculate_version):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:
@ -99,7 +107,7 @@ class UpdateHandler(Cleaner):
if local is None:
self.reporter.set_unknown(remote)
result.append(remote)
elif local.is_outdated(remote, self.paths):
elif local.is_outdated(remote, self.paths, calculate_version=True):
self.reporter.set_pending(local.base)
result.append(remote)
except Exception:

View File

@ -34,8 +34,8 @@ from ahriman.core.exceptions import OptionError, UnsafeRunError
from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["check_output", "check_user", "exception_response_text", "filter_json", "full_version", "enum_values",
"package_like", "pretty_datetime", "pretty_size", "safe_filename", "walk"]
__all__ = ["check_output", "check_user", "enum_values", "exception_response_text", "filter_json", "full_version",
"package_like", "pretty_datetime", "pretty_size", "safe_filename", "utcnow", "walk"]
def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optional[Path] = None,
@ -295,6 +295,16 @@ def safe_filename(source: str) -> str:
return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source)
def utcnow() -> datetime.datetime:
"""
get current time
Returns:
datetime.datetime: current time in UTC
"""
return datetime.datetime.utcnow()
def walk(directory_path: Path) -> Generator[Path, None, None]:
"""
list all file paths in given directory

View File

@ -19,13 +19,11 @@
#
from __future__ import annotations
import datetime
from dataclasses import dataclass, field, fields
from enum import Enum
from typing import Any, Dict, Type
from ahriman.core.util import filter_json, pretty_datetime
from ahriman.core.util import filter_json, pretty_datetime, utcnow
class BuildStatusEnum(str, Enum):
@ -58,7 +56,7 @@ class BuildStatus:
"""
status: BuildStatusEnum = BuildStatusEnum.Unknown
timestamp: int = field(default_factory=lambda: int(datetime.datetime.utcnow().timestamp()))
timestamp: int = field(default_factory=lambda: int(utcnow().timestamp()))
def __post_init__(self) -> None:
"""

View File

@ -17,6 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# pylint: disable=too-many-lines
from __future__ import annotations
import copy
@ -25,7 +26,7 @@ from dataclasses import asdict, dataclass
from pathlib import Path
from pyalpm import vercmp # type: ignore
from srcinfo.parse import parse_srcinfo # type: ignore
from typing import Any, Dict, Iterable, List, Optional, Set, Type
from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb
@ -358,7 +359,24 @@ class Package(LazyLogging):
return sorted(result)
def is_outdated(self, remote: Package, paths: RepositoryPaths, *, calculate_version: bool = True) -> bool:
def is_newer_than(self, timestamp: Union[float, int]) -> bool:
"""
check if package was built after the specified timestamp
Args:
timestamp(int): timestamp to check build date against
Returns:
bool: True in case if package was built after the specified date and False otherwise. In case if build date
is not set by any of packages, it returns False
"""
return any(
package.build_date > timestamp
for package in self.packages.values()
if package.build_date is not None
)
def is_outdated(self, remote: Package, paths: RepositoryPaths, *, calculate_version: bool) -> bool:
"""
check if package is out-of-dated