Compare commits

..

6 Commits

Author SHA1 Message Date
7eb637ba93 lookup through archive packages before build 2025-07-31 17:11:54 +03:00
60bae4a56d use generic packages tree for all repos 2025-07-31 15:20:57 +03:00
6478d940f8 implement atomic_move method, move files only with lock 2025-07-31 15:20:57 +03:00
b69ef95890 write tests to support new changes 2025-07-31 15:20:57 +03:00
f7ed9d48e5 store built packages in archive tree instead of repository 2025-07-31 15:20:57 +03:00
358e3dc4d2 feat: expose repository name and architecure in configuration if available
In some cases there are reference to current repository settings. In
order to handle it correctly two ro options have been added

Related to #154
2025-07-31 14:14:22 +03:00
6 changed files with 128 additions and 16 deletions

View File

@ -146,6 +146,8 @@ Build related configuration. Group name can refer to architecture, e.g. ``build:
Base repository settings.
* ``architecture`` - repository architecture, string. This field is read-only and generated automatically from run options if possible.
* ``name`` - repository name, string. This field is read-only and generated automatically from run options if possible.
* ``root`` - root path for application, string, required.
``sign:*`` groups

View File

@ -43,7 +43,6 @@ class Configuration(configparser.RawConfigParser):
SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package
includes(list[Path]): list of includes which were read
path(Path | None): path to root configuration file
repository_id(RepositoryId | None): repository unique identifier
Examples:
Configuration class provides additional method in order to handle application configuration. Since this class is
@ -94,7 +93,7 @@ class Configuration(configparser.RawConfigParser):
},
)
self.repository_id: RepositoryId | None = None
self._repository_id: RepositoryId | None = None
self.path: Path | None = None
self.includes: list[Path] = []
@ -129,6 +128,32 @@ class Configuration(configparser.RawConfigParser):
"""
return self.getpath("settings", "logging")
@property
def repository_id(self) -> RepositoryId | None:
"""
repository identifier
Returns:
RepositoryId: repository unique identifier
"""
return self._repository_id
@repository_id.setter
def repository_id(self, repository_id: RepositoryId | None) -> None:
"""
setter for repository identifier
Args:
repository_id(RepositoryId | None): repository unique identifier
"""
self._repository_id = repository_id
if repository_id is None or repository_id.is_empty:
self.remove_option("repository", "name")
self.remove_option("repository", "architecture")
else:
self.set_option("repository", "name", repository_id.name)
self.set_option("repository", "architecture", repository_id.architecture)
@property
def repository_name(self) -> str:
"""

View File

@ -249,6 +249,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"repository": {
"type": "dict",
"schema": {
"architecture": {
"type": "string",
"empty": False,
},
"name": {
"type": "string",
"empty": False,

View File

@ -17,7 +17,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from collections.abc import Iterable
import shutil
from collections.abc import Generator, Iterable
from pathlib import Path
from tempfile import TemporaryDirectory
@ -25,7 +27,7 @@ from ahriman.core.build_tools.package_archive import PackageArchive
from ahriman.core.build_tools.task import Task
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.repository.package_info import PackageInfo
from ahriman.core.utils import atomic_move, safe_filename
from ahriman.core.utils import atomic_move, filelock, package_like, safe_filename
from ahriman.models.changes import Changes
from ahriman.models.event import EventType
from ahriman.models.package import Package
@ -39,6 +41,28 @@ class Executor(PackageInfo, Cleaner):
trait for common repository update processes
"""
def _archive_lookup(self, package: Package) -> Generator[Path, None, None]:
"""
check if there is a rebuilt package already
Args:
package(Package): package to check
Yields:
Path: list of built packages and signatures if available, empty list otherwise
"""
archive = self.paths.archive_for(package.base)
for path in filter(package_like, archive.iterdir()):
built = Package.from_archive(path, self.pacman)
# check if there is an archive with exact same version
if built.version != package.version:
continue
for single in built.packages.values():
# we allow packages with either same architecture or any
if single.architecture not in ("any", self.architecture):
continue
yield from archive.glob(f"{single.filename}*")
def _archive_rename(self, description: PackageDescription, package_base: str) -> None:
"""
rename package archive removing special symbols
@ -74,7 +98,16 @@ class Executor(PackageInfo, Cleaner):
task = Task(package, self.configuration, self.architecture, self.paths)
patches = self.reporter.package_patches_get(package.base, None)
commit_sha = task.init(path, patches, local_version)
built = task.build(path, PACKAGER=packager)
loaded_package = Package.from_build(path, self.architecture, None)
if prebuilt := list(self._archive_lookup(loaded_package)):
built = []
for artefact in prebuilt:
with filelock(artefact):
shutil.copy(artefact, path)
built.append(path / artefact.name)
else:
built = task.build(path, PACKAGER=packager)
package.with_packages(built, self.pacman)
for src in built:

View File

@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# pylint: disable=too-many-lines
import contextlib
import datetime
import fcntl
import io
@ -47,6 +48,7 @@ __all__ = [
"dataclass_view",
"enum_values",
"extract_user",
"filelock",
"filter_json",
"full_version",
"minmax",
@ -83,17 +85,8 @@ def atomic_move(src: Path, dst: Path) -> None:
>>> atomic_move(src, dst)
"""
lock_path = dst.with_name(f".{dst.name}")
try:
with lock_path.open("ab") as lock_file:
fd = lock_file.fileno()
try:
fcntl.flock(fd, fcntl.LOCK_EX) # lock file and wait lock is until available
shutil.move(src, dst)
finally:
fcntl.flock(fd, fcntl.LOCK_UN) # unlock file first
finally:
lock_path.unlink(missing_ok=True) # remove lock file at the end
with filelock(dst):
shutil.move(src, dst)
# pylint: disable=too-many-locals
@ -264,6 +257,27 @@ def extract_user() -> str | None:
return os.getenv("SUDO_USER") or os.getenv("DOAS_USER") or os.getenv("USER")
@contextlib.contextmanager
def filelock(path: Path) -> Generator[None, None, None]:
"""
lock on file passed as argument
Args:
path(Path): path object on which lock must be performed
"""
lock_path = path.with_name(f".{path.name}")
try:
with lock_path.open("ab") as lock_file:
fd = lock_file.fileno()
try:
fcntl.flock(fd, fcntl.LOCK_EX) # lock file and wait lock is until available
yield
finally:
fcntl.flock(fd, fcntl.LOCK_UN) # unlock file first
finally:
lock_path.unlink(missing_ok=True) # remove lock file at the end
def filter_json(source: dict[str, Any], known_fields: Iterable[str]) -> dict[str, Any]:
"""
filter json object by fields used for json-to-object conversion

View File

@ -20,6 +20,40 @@ def test_architecture(configuration: Configuration) -> None:
assert configuration.architecture == "x86_64"
def test_repository_id(configuration: Configuration, repository_id: RepositoryId) -> None:
"""
must return repository identifier
"""
assert configuration.repository_id == repository_id
assert configuration.get("repository", "name") == repository_id.name
assert configuration.get("repository", "architecture") == repository_id.architecture
def test_repository_id_erase(configuration: Configuration) -> None:
"""
must remove repository identifier properties if empty identifier supplied
"""
configuration.repository_id = None
assert configuration.get("repository", "name", fallback=None) is None
assert configuration.get("repository", "architecture", fallback=None) is None
configuration.repository_id = RepositoryId("", "")
assert configuration.get("repository", "name", fallback=None) is None
assert configuration.get("repository", "architecture", fallback=None) is None
def test_repository_id_update(configuration: Configuration, repository_id: RepositoryId) -> None:
"""
must update repository identifier and related configuration options
"""
repository_id = RepositoryId("i686", repository_id.name)
configuration.repository_id = repository_id
assert configuration.repository_id == repository_id
assert configuration.get("repository", "name") == repository_id.name
assert configuration.get("repository", "architecture") == repository_id.architecture
def test_repository_name(configuration: Configuration) -> None:
"""
must return valid repository name