arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-28 09:17:17 +00:00
Code Issues Projects Releases Wiki Activity

add ability to add manually stored packages

Browse Source
This commit is contained in:
Evgenii Alekseev 2021-10-12 02:42:06 +03:00
parent ab8ca16981
commit 086885fb0e
9 changed files with 155 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/ahriman/application/application.py
View File

@ -105,21 +105,29 @@ class Application:
:param without_dependencies: if set, dependency check will be disabled
"""
known_packages = self._known_packages()
aur_url = self.configuration.get("alpm", "aur_url")
def add_archive(src: Path) -> None:
dst = self.repository.paths.packages / src.name
shutil.move(src, dst)
def add_directory(path: Path) -> None:
for full_path in filter(package_like, path.iterdir()):
add_archive(full_path)
def add_manual(src: str) -> Path:
package = Package.load(src, self.repository.pacman, self.configuration.get("alpm", "aur_url"))
def add_local(path: Path) -> Path:
package = Package.load(path, self.repository.pacman, aur_url)
Sources.init(path)
shutil.copytree(path, self.repository.paths.manual_for(package.base)) # copy package for the build
shutil.copytree(path, self.repository.paths.cache_for(package.base)) # copy package to store in caches
return self.repository.paths.manual_for(package.base)
def add_remote(src: str) -> Path:
package = Package.load(src, self.repository.pacman, aur_url)
Sources.load(self.repository.paths.manual_for(package.base), package.git_url,
self.repository.paths.patches_for(package.base))
return self.repository.paths.manual_for(package.base)
def add_archive(src: Path) -> None:
dst = self.repository.paths.packages / src.name
shutil.move(src, dst)
def process_dependencies(path: Path) -> None:
if without_dependencies:
return
@ -128,12 +136,15 @@ class Application:
def process_single(src: str) -> None:
resolved_source = source.resolve(src)
if resolved_source == PackageSource.Directory:
add_directory(Path(src))
elif resolved_source == PackageSource.Archive:
if resolved_source == PackageSource.Archive:
add_archive(Path(src))
else:
path = add_manual(src)
elif resolved_source == PackageSource.AUR:
path = add_remote(src)
process_dependencies(path)
elif resolved_source == PackageSource.Directory:
add_directory(Path(src))
elif resolved_source == PackageSource.Local:
path = add_local(Path(src))
process_dependencies(path)
for name in names:

89
src/ahriman/core/build_tools/sources.py
View File

@ -33,72 +33,97 @@ class Sources:
logger = logging.getLogger("build_details")
_branch = "master" # in case if BLM would like to change it
_check_output = check_output
@staticmethod
def add(local_path: Path, *pattern: str) -> None:
def add(sources_dir: Path, *pattern: str) -> None:
"""
track found files via git
:param local_path: local path to git repository
:param sources_dir: local path to git repository
:param pattern: glob patterns
"""
# glob directory to find files which match the specified patterns
found_files: List[Path] = []
for glob in pattern:
found_files.extend(local_path.glob(glob))
found_files.extend(sources_dir.glob(glob))
Sources.logger.info("found matching files %s", found_files)
# add them to index
Sources._check_output("git", "add", "--intent-to-add", *[str(fn.relative_to(local_path)) for fn in found_files],
exception=None, cwd=local_path, logger=Sources.logger)
Sources._check_output("git", "add", "--intent-to-add",
*[str(fn.relative_to(sources_dir)) for fn in found_files],
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def diff(local_path: Path, patch_path: Path) -> None:
def branches(sources_dir: Path) -> List[str]:
"""
list current branches. Currently this method is used to define if there is initialized git repository
:param sources_dir: local path to git repository
:return: sorted list of available branches
"""
branches = Sources._check_output("git", "branch", exception=None, cwd=sources_dir, logger=Sources.logger)
return sorted(branches.splitlines())
@staticmethod
def diff(sources_dir: Path, patch_path: Path) -> None:
"""
generate diff from the current version and write it to the output file
:param local_path: local path to git repository
:param sources_dir: local path to git repository
:param patch_path: path to result patch
"""
patch = Sources._check_output("git", "diff", exception=None, cwd=local_path, logger=Sources.logger)
patch = Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
patch_path.write_text(patch)
@staticmethod
def fetch(local_path: Path, remote: str, branch: str = "master") -> None:
def fetch(sources_dir: Path, remote: str) -> None:
"""
either clone repository or update it to origin/`branch`
:param local_path: local path to fetch
:param sources_dir: local path to fetch
:param remote: remote target (from where to fetch)
:param branch: branch name to checkout, master by default
"""
# local directory exists and there is .git directory
if (local_path / ".git").is_dir():
Sources.logger.info("update HEAD to remote to %s", local_path)
Sources._check_output("git", "fetch", "origin", branch,
exception=None, cwd=local_path, logger=Sources.logger)
is_initialized_git = (sources_dir / ".git").is_dir()
if is_initialized_git and not Sources.branches(sources_dir):
# there is git repository, but no remote configured so far
return
if is_initialized_git:
Sources.logger.info("update HEAD to remote to %s", sources_dir)
Sources._check_output("git", "fetch", "origin", Sources._branch,
exception=None, cwd=sources_dir, logger=Sources.logger)
else:
Sources.logger.info("clone remote %s to %s", remote, local_path)
Sources._check_output("git", "clone", remote, str(local_path), exception=None, logger=Sources.logger)
Sources.logger.info("clone remote %s to %s", remote, sources_dir)
Sources._check_output("git", "clone", remote, str(sources_dir), exception=None, logger=Sources.logger)
# and now force reset to our branch
Sources._check_output("git", "checkout", "--force", branch,
exception=None, cwd=local_path, logger=Sources.logger)
Sources._check_output("git", "reset", "--hard", f"origin/{branch}",
exception=None, cwd=local_path, logger=Sources.logger)
Sources._check_output("git", "checkout", "--force", Sources._branch,
exception=None, cwd=sources_dir, logger=Sources.logger)
Sources._check_output("git", "reset", "--hard", f"origin/{Sources._branch}",
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def load(local_path: Path, remote: str, patch_dir: Path) -> None:
def init(sources_dir: Path) -> None:
"""
create empty git repository at the specified path
:param sources_dir: local path to sources
"""
Sources._check_output("git", "init", "--initial-branch", Sources._branch,
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def load(sources_dir: Path, remote: str, patch_dir: Path) -> None:
"""
fetch sources from remote and apply patches
:param local_path: local path to fetch
:param sources_dir: local path to fetch
:param remote: remote target (from where to fetch)
:param patch_dir: path to directory with package patches
"""
Sources.fetch(local_path, remote)
Sources.patch_apply(local_path, patch_dir)
Sources.fetch(sources_dir, remote)
Sources.patch_apply(sources_dir, patch_dir)
@staticmethod
def patch_apply(local_path: Path, patch_dir: Path) -> None:
def patch_apply(sources_dir: Path, patch_dir: Path) -> None:
"""
apply patches if any
:param local_path: local path to directory with git sources
:param sources_dir: local path to directory with git sources
:param patch_dir: path to directory with package patches
"""
# check if even there are patches
@ -110,15 +135,15 @@ class Sources:
for patch in patches:
Sources.logger.info("apply patch %s", patch.name)
Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace", str(patch),
exception=None, cwd=local_path, logger=Sources.logger)
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def patch_create(local_path: Path, patch_path: Path, *pattern: str) -> None:
def patch_create(sources_dir: Path, patch_path: Path, *pattern: str) -> None:
"""
create patch set for the specified local path
:param local_path: local path to git repository
:param sources_dir: local path to git repository
:param patch_path: path to result patch
:param pattern: glob patterns
"""
Sources.add(local_path, *pattern)
Sources.diff(local_path, patch_path)
Sources.add(sources_dir, *pattern)
Sources.diff(sources_dir, patch_path)

16
src/ahriman/core/repository/executor.py
View File

@ -72,9 +72,16 @@ class Executor(Cleaner):
:param packages: list of package names or bases to remove
:return: path to repository database
"""
def remove_single(package: str, fn: Path) -> None:
def remove_base(package_base: str) -> None:
try:
self.repo.remove(package, fn)
self.paths.tree_clear(package_base) # remove all internal files
self.reporter.remove(package_base) # we only update status page in case of base removal
except Exception:
self.logger.exception("could not remove base %s", package_base)
def remove_package(package: str, fn: Path) -> None:
try:
self.repo.remove(package, fn) # remove the package itself
except Exception:
self.logger.exception("could not remove %s", package)
@ -86,7 +93,7 @@ class Executor(Cleaner):
for package, properties in local.packages.items()
if properties.filename is not None
}
self.reporter.remove(local.base) # we only update status page in case of base removal
remove_base(local.base)
elif requested.intersection(local.packages.keys()):
to_remove = {
package: Path(properties.filename)
@ -95,8 +102,9 @@ class Executor(Cleaner):
}
else:
to_remove = {}
for package, filename in to_remove.items():
remove_single(package, filename)
remove_package(package, filename)
return self.repo.repo_path

2
src/ahriman/core/repository/properties.py
View File

@ -58,7 +58,7 @@ class Properties:
self.name = configuration.get("repository", "name")
self.paths = RepositoryPaths(configuration.getpath("repository", "root"), architecture)
self.paths.create_tree()
self.paths.tree_create()
self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[])
self.pacman = Pacman(configuration)

9
src/ahriman/models/package_source.py
View File

@ -30,14 +30,16 @@ class PackageSource(Enum):
package source for addition enumeration
:cvar Auto: automatically determine type of the source
:cvar Archive: source is a package archive
:cvar Directory: source is a directory which contains packages
:cvar AUR: source is an AUR package for which it should search
:cvar Directory: source is a directory which contains packages
:cvar Local: source is locally stored PKGBUILD
"""
Auto = "auto"
Archive = "archive"
Directory = "directory"
AUR = "aur"
Directory = "directory"
Local = "local"
def resolve(self, source: str) -> PackageSource:
"""
@ -47,7 +49,10 @@ class PackageSource(Enum):
"""
if self != PackageSource.Auto:
return self
maybe_path = Path(source)
if (maybe_path / "PKGBUILD").is_file():
return PackageSource.Local
if maybe_path.is_dir():
return PackageSource.Directory
if maybe_path.is_file() and package_like(maybe_path):

43
src/ahriman/models/repository_paths.py
View File

@ -19,6 +19,8 @@
#
from __future__ import annotations
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Set, Type
@ -101,24 +103,12 @@ class RepositoryPaths:
def cache_for(self, package_base: str) -> Path:
"""
get cache path for specific package base
get path to cached PKGBUILD and package sources for the package base
:param package_base: package base name
:return: full path to directory for specified package base cache
"""
return self.cache / package_base
def create_tree(self) -> None:
"""
create ahriman working tree
"""
self.cache.mkdir(mode=0o755, parents=True, exist_ok=True)
self.chroot.mkdir(mode=0o755, parents=True, exist_ok=True)
self.manual.mkdir(mode=0o755, parents=True, exist_ok=True)
self.packages.mkdir(mode=0o755, parents=True, exist_ok=True)
self.patches.mkdir(mode=0o755, parents=True, exist_ok=True)
self.repository.mkdir(mode=0o755, parents=True, exist_ok=True)
self.sources.mkdir(mode=0o755, parents=True, exist_ok=True)
def manual_for(self, package_base: str) -> Path:
"""
get manual path for specific package base
@ -137,8 +127,33 @@ class RepositoryPaths:
def sources_for(self, package_base: str) -> Path:
"""
get sources path for specific package base
get path to directory from where build will start for the package base
:param package_base: package base name
:return: full path to directory for specified package base sources
"""
return self.sources / package_base
def tree_clear(self, package_base: str) -> None:
"""
clear package specific files
:param package_base: package base name
"""
for directory in (
self.cache_for(package_base),
self.manual_for(package_base),
self.patches_for(package_base)):
shutil.rmtree(directory, ignore_errors=True)
def tree_create(self) -> None:
"""
create ahriman working tree
"""
for directory in (
self.cache,
self.chroot,
self.manual,
self.packages,
self.patches,
self.repository,
self.sources):
directory.mkdir(mode=0o755, parents=True, exist_ok=True)

4
tests/ahriman/application/handlers/test_handler_init.py
View File

@ -10,11 +10,11 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc
"""
must run command
"""
create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
tree_create_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init")
Init.run(args, "x86_64", configuration, True)
create_tree_mock.assert_called_once()
tree_create_mock.assert_called_once()
init_mock.assert_called_once()

8
tests/ahriman/core/repository/test_properties.py
View File

@ -9,17 +9,17 @@ def test_create_tree_on_load(configuration: Configuration, mocker: MockerFixture
"""
must create tree on load
"""
create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
tree_create_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
Properties("x86_64", configuration, True)
create_tree_mock.assert_called_once()
tree_create_mock.assert_called_once()
def test_create_dummy_report_client(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must create dummy report client if report is disabled
"""
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
load_mock = mocker.patch("ahriman.core.status.client.Client.load")
properties = Properties("x86_64", configuration, True)
@ -31,7 +31,7 @@ def test_create_full_report_client(configuration: Configuration, mocker: MockerF
"""
must create load report client if report is enabled
"""
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
load_mock = mocker.patch("ahriman.core.status.client.Client.load")
Properties("x86_64", configuration, False)

42
tests/ahriman/models/test_repository_paths.py
View File

@ -23,27 +23,6 @@ def test_cache_for(repository_paths: RepositoryPaths, package_ahriman: Package)
assert path.parent == repository_paths.cache
def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
"""
must create whole tree
"""
paths = {
prop
for prop in dir(repository_paths)
if not prop.startswith("_")
and not prop.endswith("_for")
and prop not in ("architecture", "create_tree", "known_architectures", "root")
}
mkdir_mock = mocker.patch("pathlib.Path.mkdir")
repository_paths.create_tree()
mkdir_mock.assert_has_calls(
[
mock.call(mode=0o755, parents=True, exist_ok=True)
for _ in paths
])
def test_manual_for(repository_paths: RepositoryPaths, package_ahriman: Package) -> None:
"""
must return correct path for manual directory
@ -69,3 +48,24 @@ def test_sources_for(repository_paths: RepositoryPaths, package_ahriman: Package
path = repository_paths.sources_for(package_ahriman.base)
assert path.name == package_ahriman.base
assert path.parent == repository_paths.sources
def test_tree_create(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
"""
must create whole tree
"""
paths = {
prop
for prop in dir(repository_paths)
if not prop.startswith("_")
and not prop.endswith("_for")
and prop not in ("architecture", "known_architectures", "root", "tree_clear", "tree_create")
}
mkdir_mock = mocker.patch("pathlib.Path.mkdir")
repository_paths.tree_create()
mkdir_mock.assert_has_calls(
[
mock.call(mode=0o755, parents=True, exist_ok=True)
for _ in paths
])