arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-23 23:07:17 +00:00
Code Issues Projects Releases Wiki Activity

fix error with missing sources

Browse Source 
In case if package has local cache it will fail to load because no
remote source set. Particially this case can be observed during tree
load
This commit is contained in:
Evgenii Alekseev 2022-05-08 03:56:54 +03:00
parent 73057bc08d
commit a7018d9f7d
11 changed files with 115 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/ahriman/application/application/application_packages.py
View File

@ -86,7 +86,7 @@ class ApplicationPackages(ApplicationProperties):
self.database.remote_update(package)
with tmpdir() as local_path:
Sources.load(local_path, package.remote, self.database.patches_get(package.base))
Sources.load(local_path, package, self.database.patches_get(package.base), self.repository.paths)
self._process_dependencies(local_path, known_packages, without_dependencies)
def _add_directory(self, source: str, *_: Any) -> None:

2
src/ahriman/application/application/application_repository.py
View File

@ -169,7 +169,7 @@ class ApplicationRepository(ApplicationProperties):
process_update(packages, build_result)
# process manual packages
tree = Tree.load(updates, self.database)
tree = Tree.load(updates, self.repository.paths, self.database)
for num, level in enumerate(tree.levels()):
self.logger.info("processing level #%i %s", num, [package.base for package in level])
build_result = self.repository.process_build(level)

56
src/ahriman/core/build_tools/sources.py
View File

@ -24,7 +24,9 @@ from pathlib import Path
from typing import List, Optional
from ahriman.core.util import check_output, walk
from ahriman.models.package import Package
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
class Sources:
@ -43,7 +45,7 @@ class Sources:
_check_output = check_output
@staticmethod
def add(sources_dir: Path, *pattern: str) -> None:
def _add(sources_dir: Path, *pattern: str) -> None:
"""
track found files via git
@ -64,7 +66,7 @@ class Sources:
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def diff(sources_dir: Path) -> str:
def _diff(sources_dir: Path) -> str:
"""
generate diff from the current version and write it to the output file
@ -76,6 +78,21 @@ class Sources:
"""
return Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def _move(pkgbuild_dir: Path, sources_dir: Path) -> None:
"""
move content from pkgbuild_dir to sources_dir
Args:
pkgbuild_dir(Path): path to directory with pkgbuild from which need to move
sources_dir(Path): path to target directory
"""
if pkgbuild_dir == sources_dir:
return # directories are the same, no need to move
for src in walk(pkgbuild_dir):
dst = sources_dir / src.relative_to(pkgbuild_dir)
shutil.move(src, dst)
@staticmethod
def fetch(sources_dir: Path, remote: Optional[RemoteSource]) -> None:
"""
@ -103,7 +120,8 @@ class Sources:
remote.git_url, str(sources_dir),
exception=None, cwd=sources_dir, logger=Sources.logger)
else:
Sources.logger.warning("%s is not initialized, but no remote provided", sources_dir)
# it will cause an exception later
Sources.logger.error("%s is not initialized, but no remote provided", sources_dir)
# and now force reset to our branch
Sources._check_output("git", "checkout", "--force", branch,
@ -114,7 +132,7 @@ class Sources:
# move content if required
# we are using full path to source directory in order to make append possible
pkgbuild_dir = remote.pkgbuild_dir if remote is not None else sources_dir.resolve()
Sources.move((sources_dir / pkgbuild_dir).resolve(), sources_dir)
Sources._move((sources_dir / pkgbuild_dir).resolve(), sources_dir)
@staticmethod
def has_remotes(sources_dir: Path) -> bool:
@ -142,36 +160,26 @@ class Sources:
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def load(sources_dir: Path, remote: Optional[RemoteSource], patch: Optional[str]) -> None:
def load(sources_dir: Path, package: Package, patch: Optional[str], paths: RepositoryPaths) -> None:
"""
fetch sources from remote and apply patches
Args:
sources_dir(Path): local path to fetch
remote(Optional[RemoteSource]): remote target (from where to fetch)
package(Package): package definitions
patch(Optional[str]): optional patch to be applied
paths(RepositoryPaths): repository paths instance
"""
Sources.fetch(sources_dir, remote)
if (cache_dir := paths.cache_for(package.base)).is_dir() and cache_dir != sources_dir:
# no need to clone whole repository, just copy from cache first
shutil.copytree(cache_dir, sources_dir, dirs_exist_ok=True)
Sources.fetch(sources_dir, package.remote)
if patch is None:
Sources.logger.info("no patches found")
return
Sources.patch_apply(sources_dir, patch)
@staticmethod
def move(pkgbuild_dir: Path, sources_dir: Path) -> None:
"""
move content from pkgbuild_dir to sources_dir
Args:
pkgbuild_dir(Path): path to directory with pkgbuild from which need to move
sources_dir(Path): path to target directory
"""
if pkgbuild_dir == sources_dir:
return # directories are the same, no need to move
for src in walk(pkgbuild_dir):
dst = sources_dir / src.relative_to(pkgbuild_dir)
shutil.move(src, dst)
@staticmethod
def patch_apply(sources_dir: Path, patch: str) -> None:
"""
@ -198,6 +206,6 @@ class Sources:
Returns:
str: patch as plain text
"""
Sources.add(sources_dir, *pattern)
diff = Sources.diff(sources_dir)
Sources._add(sources_dir, *pattern)
diff = Sources._diff(sources_dir)
return f"{diff}\n" # otherwise, patch will be broken

18
src/ahriman/core/build_tools/task.py
View File

@ -18,7 +18,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import shutil
from pathlib import Path
from typing import List
@ -66,12 +65,12 @@ class Task:
self.makepkg_flags = configuration.getlist("build", "makepkg_flags", fallback=[])
self.makechrootpkg_flags = configuration.getlist("build", "makechrootpkg_flags", fallback=[])
def build(self, sources_path: Path) -> List[Path]:
def build(self, sources_dir: Path) -> List[Path]:
"""
run package build
Args:
sources_path(Path): path to where sources are
sources_dir(Path): path to where sources are
Returns:
List[Path]: paths of produced packages
@ -85,26 +84,23 @@ class Task:
Task._check_output(
*command,
exception=BuildFailed(self.package.base),
cwd=sources_path,
cwd=sources_dir,
logger=self.build_logger,
user=self.uid)
# well it is not actually correct, but we can deal with it
packages = Task._check_output("makepkg", "--packagelist",
exception=BuildFailed(self.package.base),
cwd=sources_path,
cwd=sources_dir,
logger=self.build_logger).splitlines()
return [Path(package) for package in packages]
def init(self, path: Path, database: SQLite) -> None:
def init(self, sources_dir: Path, database: SQLite) -> None:
"""
fetch package from git
Args:
path(Path): local path to fetch
sources_dir(Path): local path to fetch
database(SQLite): database instance
"""
if self.paths.cache_for(self.package.base).is_dir():
# no need to clone whole repository, just copy from cache first
shutil.copytree(self.paths.cache_for(self.package.base), path, dirs_exist_ok=True)
Sources.load(path, self.package.remote, database.patches_get(self.package.base))
Sources.load(sources_dir, self.package, database.patches_get(self.package.base), self.paths)

13
src/ahriman/core/tree.py
View File

@ -25,6 +25,7 @@ from ahriman.core.build_tools.sources import Sources
from ahriman.core.database import SQLite
from ahriman.core.util import tmpdir
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
class Leaf:
@ -58,19 +59,20 @@ class Leaf:
return self.package.packages.keys()
@classmethod
def load(cls: Type[Leaf], package: Package, database: SQLite) -> Leaf:
def load(cls: Type[Leaf], package: Package, paths: RepositoryPaths, database: SQLite) -> Leaf:
"""
load leaf from package with dependencies
Args:
package(Package): package properties
paths(RepositoryPaths): repository paths instance
database(SQLite): database instance
Returns:
Leaf: loaded class
"""
with tmpdir() as clone_dir:
Sources.load(clone_dir, package.remote, database.patches_get(package.base))
Sources.load(clone_dir, package, database.patches_get(package.base), paths)
dependencies = Package.dependencies(clone_dir)
return cls(package, dependencies)
@ -110,7 +112,7 @@ class Tree:
>>> repository = Repository("x86_64", configuration, database, no_report=False, unsafe=False)
>>> packages = repository.packages()
>>>
>>> tree = Tree.load(packages, database)
>>> tree = Tree.load(packages, configuration.repository_paths, database)
>>> for tree_level in tree.levels():
>>> for package in tree_level:
>>> print(package.base)
@ -138,18 +140,19 @@ class Tree:
self.leaves = leaves
@classmethod
def load(cls: Type[Tree], packages: Iterable[Package], database: SQLite) -> Tree:
def load(cls: Type[Tree], packages: Iterable[Package], paths: RepositoryPaths, database: SQLite) -> Tree:
"""
load tree from packages
Args:
packages(Iterable[Package]): packages list
paths(RepositoryPaths): repository paths instance
database(SQLite): database instance
Returns:
Tree: loaded class
"""
return cls([Leaf.load(package, database) for package in packages])
return cls([Leaf.load(package, paths, database) for package in packages])
def levels(self) -> List[List[Package]]:
"""

2
src/ahriman/models/package.py
View File

@ -282,7 +282,7 @@ class Package:
from ahriman.core.build_tools.sources import Sources
logger = logging.getLogger("build_details")
Sources.load(paths.cache_for(self.base), self.remote, None)
Sources.load(paths.cache_for(self.base), self, None, paths)
try:
# update pkgver first

5
tests/ahriman/application/application/test_application_packages.py
View File

@ -54,8 +54,9 @@ def test_add_aur(application_packages: ApplicationPackages, package_ahriman: Pac
application_packages._add_aur(package_ahriman.base, set(), False)
load_mock.assert_called_once_with(
pytest.helpers.anyvar(int),
package_ahriman.remote,
pytest.helpers.anyvar(int))
package_ahriman,
pytest.helpers.anyvar(int),
application_packages.repository.paths)
dependencies_mock.assert_called_once_with(pytest.helpers.anyvar(int), set(), False)
build_queue_mock.assert_called_once_with(package_ahriman)
update_remote_mock.assert_called_once_with(package_ahriman)

1
tests/ahriman/application/application/test_application_repository.py
View File

@ -195,7 +195,6 @@ def test_update_empty(application_repository: ApplicationRepository, package_ahr
"""
must skip updating repository if no packages supplied
"""
paths = [package.filepath for package in package_ahriman.packages.values()]
tree = Tree([Leaf(package_ahriman, set())])
mocker.patch("ahriman.core.tree.Tree.load", return_value=tree)

86
tests/ahriman/core/build_tools/test_sources.py
View File

@ -5,7 +5,9 @@ from pytest_mock import MockerFixture
from unittest import mock
from ahriman.core.build_tools.sources import Sources
from ahriman.models.package import Package
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
def test_add(mocker: MockerFixture) -> None:
@ -16,7 +18,7 @@ def test_add(mocker: MockerFixture) -> None:
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
local = Path("local")
Sources.add(local, "pattern1", "pattern2")
Sources._add(local, "pattern1", "pattern2")
glob_mock.assert_has_calls([mock.call("pattern1"), mock.call("pattern2")])
check_output_mock.assert_called_once_with(
"git", "add", "--intent-to-add", "1", "2", "1", "2",
@ -30,7 +32,7 @@ def test_add_skip(mocker: MockerFixture) -> None:
mocker.patch("pathlib.Path.glob", return_value=[])
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
Sources.add(Path("local"), "pattern1")
Sources._add(Path("local"), "pattern1")
check_output_mock.assert_not_called()
@ -41,9 +43,29 @@ def test_diff(mocker: MockerFixture) -> None:
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
local = Path("local")
assert Sources.diff(local)
check_output_mock.assert_called_once_with("git", "diff",
exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
assert Sources._diff(local)
check_output_mock.assert_called_once_with(
"git", "diff", exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
def test_move(mocker: MockerFixture) -> None:
"""
must move content between directories
"""
mocker.patch("ahriman.core.build_tools.sources.walk", return_value=[Path("/source/path")])
move_mock = mocker.patch("shutil.move")
Sources._move(Path("/source"), Path("/destination"))
move_mock.assert_called_once_with(Path("/source/path"), Path("/destination/path"))
def test_move_same(mocker: MockerFixture) -> None:
"""
must not do anything in case if directories are the same
"""
walk_mock = mocker.patch("ahriman.core.build_tools.sources.walk")
Sources._move(Path("/same"), Path("/same"))
walk_mock.assert_not_called()
def test_fetch_empty(remote_source: RemoteSource, mocker: MockerFixture) -> None:
@ -65,7 +87,7 @@ def test_fetch_existing(remote_source: RemoteSource, mocker: MockerFixture) -> N
mocker.patch("pathlib.Path.is_dir", return_value=True)
mocker.patch("ahriman.core.build_tools.sources.Sources.has_remotes", return_value=True)
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
local = Path("local")
Sources.fetch(local, remote_source)
@ -86,7 +108,7 @@ def test_fetch_new(remote_source: RemoteSource, mocker: MockerFixture) -> None:
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
local = Path("local")
Sources.fetch(local, remote_source)
@ -107,7 +129,7 @@ def test_fetch_new_without_remote(mocker: MockerFixture) -> None:
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
local = Path("local")
Sources.fetch(local, None)
@ -125,7 +147,7 @@ def test_fetch_relative(remote_source: RemoteSource, mocker: MockerFixture) -> N
must process move correctly on relative directory
"""
mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
Sources.fetch(Path("path"), remote_source)
move_mock.assert_called_once_with(Path("path").resolve(), Path("path"))
@ -139,8 +161,8 @@ def test_has_remotes(mocker: MockerFixture) -> None:
local = Path("local")
assert Sources.has_remotes(local)
check_output_mock.assert_called_once_with("git", "remote",
exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
check_output_mock.assert_called_once_with(
"git", "remote", exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
def test_has_remotes_empty(mocker: MockerFixture) -> None:
@ -163,47 +185,41 @@ def test_init(mocker: MockerFixture) -> None:
exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
def test_load(remote_source: RemoteSource, mocker: MockerFixture) -> None:
def test_load(package_ahriman: Package, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
"""
must load packages sources correctly
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
fetch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.fetch")
patch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.patch_apply")
Sources.load(Path("local"), remote_source, "patch")
fetch_mock.assert_called_once_with(Path("local"), remote_source)
Sources.load(Path("local"), package_ahriman, "patch", repository_paths)
fetch_mock.assert_called_once_with(Path("local"), package_ahriman.remote)
patch_mock.assert_called_once_with(Path("local"), "patch")
def test_load_no_patch(remote_source: RemoteSource, mocker: MockerFixture) -> None:
def test_load_no_patch(package_ahriman: Package, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
"""
must load packages sources correctly without patches
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
mocker.patch("ahriman.core.build_tools.sources.Sources.fetch")
patch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.patch_apply")
Sources.load(Path("local"), remote_source, None)
Sources.load(Path("local"), package_ahriman, None, repository_paths)
patch_mock.assert_not_called()
def test_move(mocker: MockerFixture) -> None:
def test_load_with_cache(package_ahriman: Package, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
"""
must move content between directories
must load sources by using local cache
"""
mocker.patch("ahriman.core.build_tools.sources.walk", return_value=[Path("/source/path")])
move_mock = mocker.patch("shutil.move")
mocker.patch("pathlib.Path.is_dir", return_value=True)
copytree_mock = mocker.patch("shutil.copytree")
mocker.patch("ahriman.core.build_tools.sources.Sources.fetch")
Sources.move(Path("/source"), Path("/destination"))
move_mock.assert_called_once_with(Path("/source/path"), Path("/destination/path"))
def test_move_same(mocker: MockerFixture) -> None:
"""
must not do anything in case if directories are the same
"""
walk_mock = mocker.patch("ahriman.core.build_tools.sources.walk")
Sources.move(Path("/same"), Path("/same"))
walk_mock.assert_not_called()
Sources.load(Path("local"), package_ahriman, None, repository_paths)
copytree_mock.assert_called_once() # we do not check full command here, sorry
def test_patch_apply(mocker: MockerFixture) -> None:
@ -224,8 +240,8 @@ def test_patch_create(mocker: MockerFixture) -> None:
"""
must create patch set for the package
"""
add_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.add")
diff_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.diff")
add_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._add")
diff_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._diff")
Sources.patch_create(Path("local"), "glob")
add_mock.assert_called_once_with(Path("local"), "glob")
@ -236,6 +252,6 @@ def test_patch_create_with_newline(mocker: MockerFixture) -> None:
"""
created patch must have new line at the end
"""
mocker.patch("ahriman.core.build_tools.sources.Sources.add")
mocker.patch("ahriman.core.build_tools.sources.Sources.diff", return_value="diff")
mocker.patch("ahriman.core.build_tools.sources.Sources._add")
mocker.patch("ahriman.core.build_tools.sources.Sources._diff", return_value="diff")
assert Sources.patch_create(Path("local"), "glob").endswith("\n")

9
tests/ahriman/core/build_tools/test_task.py
View File

@ -14,13 +14,10 @@ def test_build(task_ahriman: Task, mocker: MockerFixture) -> None:
check_output_mock.assert_called()
def test_init_with_cache(task_ahriman: Task, database: SQLite, mocker: MockerFixture) -> None:
def test_init(task_ahriman: Task, database: SQLite, mocker: MockerFixture) -> None:
"""
must copy tree instead of fetch
"""
mocker.patch("pathlib.Path.is_dir", return_value=True)
mocker.patch("ahriman.core.build_tools.sources.Sources.load")
copytree_mock = mocker.patch("shutil.copytree")
load_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.load")
task_ahriman.init(Path("ahriman"), database)
copytree_mock.assert_called_once() # we do not check full command here, sorry
load_mock.assert_called_once_with(Path("ahriman"), task_ahriman.package, None, task_ahriman.paths)

14
tests/ahriman/core/test_tree.py
View File

@ -5,6 +5,7 @@ from pytest_mock import MockerFixture
from ahriman.core.database import SQLite
from ahriman.core.tree import Leaf, Tree
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
def test_leaf_is_root_empty(leaf_ahriman: Leaf) -> None:
@ -37,7 +38,8 @@ def test_leaf_is_root_true(leaf_ahriman: Leaf, leaf_python_schedule: Leaf) -> No
assert not leaf_ahriman.is_root([leaf_python_schedule])
def test_leaf_load(package_ahriman: Package, database: SQLite, mocker: MockerFixture) -> None:
def test_leaf_load(package_ahriman: Package, repository_paths: RepositoryPaths,
database: SQLite, mocker: MockerFixture) -> None:
"""
must load with dependencies
"""
@ -46,12 +48,12 @@ def test_leaf_load(package_ahriman: Package, database: SQLite, mocker: MockerFix
dependencies_mock = mocker.patch("ahriman.models.package.Package.dependencies", return_value={"ahriman-dependency"})
rmtree_mock = mocker.patch("shutil.rmtree")
leaf = Leaf.load(package_ahriman, database)
leaf = Leaf.load(package_ahriman, repository_paths, database)
assert leaf.package == package_ahriman
assert leaf.dependencies == {"ahriman-dependency"}
tempdir_mock.assert_called_once_with()
load_mock.assert_called_once_with(
pytest.helpers.anyvar(int), package_ahriman.remote, database.patches_get(package_ahriman.base))
pytest.helpers.anyvar(int), package_ahriman, None, repository_paths)
dependencies_mock.assert_called_once_with(pytest.helpers.anyvar(int))
rmtree_mock.assert_called_once_with(pytest.helpers.anyvar(int), ignore_errors=True)
@ -69,8 +71,8 @@ def test_tree_levels(leaf_ahriman: Leaf, leaf_python_schedule: Leaf) -> None:
assert second == [leaf_ahriman.package]
def test_tree_load(package_ahriman: Package, package_python_schedule: Package, database: SQLite,
mocker: MockerFixture) -> None:
def test_tree_load(package_ahriman: Package, package_python_schedule: Package, repository_paths: RepositoryPaths,
database: SQLite, mocker: MockerFixture) -> None:
"""
must package list
"""
@ -79,5 +81,5 @@ def test_tree_load(package_ahriman: Package, package_python_schedule: Package, d
mocker.patch("ahriman.models.package.Package.dependencies")
mocker.patch("shutil.rmtree")
tree = Tree.load([package_ahriman, package_python_schedule], database)
tree = Tree.load([package_ahriman, package_python_schedule], repository_paths, database)
assert len(tree.leaves) == 2