implement mirrorlist package generator

This commit is contained in:
2023-05-02 18:08:25 +03:00
parent 8e0e57e193
commit 21ea9a4dd1
31 changed files with 1132 additions and 34 deletions

View File

@ -5,25 +5,36 @@ from pytest_mock import MockerFixture
from unittest.mock import MagicMock
from ahriman.application.application.application_packages import ApplicationPackages
from ahriman.core.exceptions import UnknownPackageError
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
from ahriman.models.result import Result
def test_add_archive(
application_packages: ApplicationPackages,
package_ahriman: Package,
mocker: MockerFixture) -> None:
def test_add_archive(application_packages: ApplicationPackages, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must add package from archive
"""
is_file_mock = mocker.patch("pathlib.Path.is_file", return_value=True)
copy_mock = mocker.patch("shutil.copy")
application_packages._add_archive(package_ahriman.base)
is_file_mock.assert_called_once_with()
copy_mock.assert_called_once_with(
Path(package_ahriman.base), application_packages.repository.paths.packages / package_ahriman.base)
def test_add_archive_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None:
"""
must raise UnknownPackageError on unknown path
"""
mocker.patch("pathlib.Path.is_file", return_value=False)
with pytest.raises(UnknownPackageError):
application_packages._add_archive("package")
def test_add_aur(application_packages: ApplicationPackages, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must add package from AUR
@ -37,21 +48,29 @@ def test_add_aur(application_packages: ApplicationPackages, package_ahriman: Pac
update_remote_mock.assert_called_once_with(package_ahriman)
def test_add_directory(
application_packages: ApplicationPackages,
package_ahriman: Package,
mocker: MockerFixture) -> None:
def test_add_directory(application_packages: ApplicationPackages, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must add packages from directory
"""
iterdir_mock = mocker.patch("pathlib.Path.iterdir",
return_value=[package.filepath for package in package_ahriman.packages.values()])
copy_mock = mocker.patch("shutil.copy")
is_dir_mock = mocker.patch("pathlib.Path.is_dir", return_value=True)
filename = package_ahriman.packages[package_ahriman.base].filepath
iterdir_mock = mocker.patch("pathlib.Path.iterdir", return_value=[filename])
add_mock = mocker.patch("ahriman.application.application.application_packages.ApplicationPackages._add_archive")
application_packages._add_directory(package_ahriman.base)
is_dir_mock.assert_called_once_with()
iterdir_mock.assert_called_once_with()
copy_mock.assert_called_once_with(filename, application_packages.repository.paths.packages / filename.name)
add_mock.assert_called_once_with(str(filename))
def test_add_directory_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None:
"""
must raise UnknownPackageError on unknown directory path
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
with pytest.raises(UnknownPackageError):
application_packages._add_directory("package")
def test_add_local(application_packages: ApplicationPackages, package_ahriman: Package, mocker: MockerFixture) -> None:
@ -59,17 +78,46 @@ def test_add_local(application_packages: ApplicationPackages, package_ahriman: P
must add package from local sources
"""
mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman)
is_dir_mock = mocker.patch("pathlib.Path.is_dir", return_value=True)
init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init")
copytree_mock = mocker.patch("shutil.copytree")
build_queue_mock = mocker.patch("ahriman.core.database.SQLite.build_queue_insert")
application_packages._add_local(package_ahriman.base)
is_dir_mock.assert_called_once_with()
copytree_mock.assert_called_once_with(
Path(package_ahriman.base), application_packages.repository.paths.cache_for(package_ahriman.base))
init_mock.assert_called_once_with(application_packages.repository.paths.cache_for(package_ahriman.base))
build_queue_mock.assert_called_once_with(package_ahriman)
def test_add_local_cache(application_packages: ApplicationPackages, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must add package from local source if there is cache
"""
mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman)
mocker.patch("pathlib.Path.is_dir", autospec=True,
side_effect=lambda p: True if p.is_relative_to(application_packages.repository.paths.cache) else False)
init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init")
copytree_mock = mocker.patch("shutil.copytree")
build_queue_mock = mocker.patch("ahriman.core.database.SQLite.build_queue_insert")
application_packages._add_local(package_ahriman.base)
copytree_mock.assert_not_called()
init_mock.assert_not_called()
build_queue_mock.assert_called_once_with(package_ahriman)
def test_add_local_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None:
"""
must raise UnknownPackageError if package wasn't found
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
with pytest.raises(UnknownPackageError):
application_packages._add_local("package")
def test_add_remote(application_packages: ApplicationPackages, package_description_ahriman: PackageDescription,
mocker: MockerFixture) -> None:
"""
@ -87,6 +135,15 @@ def test_add_remote(application_packages: ApplicationPackages, package_descripti
response_mock.raise_for_status.assert_called_once_with()
def test_add_remote_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None:
"""
must add package from remote source
"""
mocker.patch("requests.get", side_effect=Exception())
with pytest.raises(UnknownPackageError):
application_packages._add_remote("url")
def test_add_repository(application_packages: ApplicationPackages, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
@ -112,10 +169,8 @@ def test_add_add_archive(application_packages: ApplicationPackages, package_ahri
add_mock.assert_called_once_with(package_ahriman.base)
def test_add_add_aur(
application_packages: ApplicationPackages,
package_ahriman: Package,
mocker: MockerFixture) -> None:
def test_add_add_aur(application_packages: ApplicationPackages, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must add package from AUR via add function
"""

View File

@ -62,6 +62,8 @@ def test_schema(configuration: Configuration) -> None:
assert schema.pop("email")
assert schema.pop("github")
assert schema.pop("html")
assert schema.pop("mirrorlist")
assert schema.pop("mirrorlist_generator")
assert schema.pop("report")
assert schema.pop("rsync")
assert schema.pop("s3")
@ -76,6 +78,7 @@ def test_schema_invalid_trigger(configuration: Configuration) -> None:
must skip trigger if it caused exception on load
"""
configuration.set_option("build", "triggers", "some.invalid.trigger.path.Trigger")
configuration.remove_option("build", "triggers_known")
assert Validate.schema("x86_64", configuration) == CONFIGURATION_SCHEMA

View File

@ -373,6 +373,24 @@ def test_subparsers_repo_check_option_refresh(parser: argparse.ArgumentParser) -
assert args.refresh == 2
def test_subparsers_repo_create_mirrorlist(parser: argparse.ArgumentParser) -> None:
"""
repo-create-mirrorlist command must imply trigger
"""
args = parser.parse_args(["repo-create-mirrorlist"])
assert args.trigger == ["ahriman.core.support.MirrorlistTrigger"]
def test_subparsers_repo_create_mirrorlist_architecture(parser: argparse.ArgumentParser) -> None:
"""
repo-create-mirrorlist command must correctly parse architecture list
"""
args = parser.parse_args(["repo-create-mirrorlist"])
assert args.architecture is None
args = parser.parse_args(["-a", "x86_64", "repo-create-mirrorlist"])
assert args.architecture == ["x86_64"]
def test_subparsers_repo_daemon(parser: argparse.ArgumentParser) -> None:
"""
repo-daemon command must imply dry run, exit code and package

View File

@ -10,6 +10,13 @@ from ahriman.core.exceptions import InitializeError
from ahriman.models.repository_paths import RepositoryPaths
def test_repository_name(configuration: Configuration) -> None:
"""
must return valid repository name
"""
assert configuration.repository_name == "aur-clone"
def test_repository_paths(configuration: Configuration, repository_paths: RepositoryPaths) -> None:
"""
must return repository paths

View File

@ -73,6 +73,30 @@ def test_validate_is_ip_address(validator: Validator, mocker: MockerFixture) ->
])
def test_validate_path_is_absolute(validator: Validator, mocker: MockerFixture) -> None:
"""
must validate that path is absolute
"""
error_mock = mocker.patch("ahriman.core.configuration.validator.Validator._error")
mocker.patch("pathlib.Path.is_absolute", return_value=False)
validator._validate_path_is_absolute(False, "field", Path("1"))
mocker.patch("pathlib.Path.is_absolute", return_value=True)
validator._validate_path_is_absolute(False, "field", Path("2"))
mocker.patch("pathlib.Path.is_absolute", return_value=False)
validator._validate_path_is_absolute(True, "field", Path("3"))
mocker.patch("pathlib.Path.is_absolute", return_value=True)
validator._validate_path_is_absolute(True, "field", Path("4"))
error_mock.assert_has_calls([
MockCall("field", "Path 2 must be relative"),
MockCall("field", "Path 3 must be absolute"),
])
def test_validate_is_url(validator: Validator, mocker: MockerFixture) -> None:
"""
must validate url correctly
@ -105,12 +129,16 @@ def test_validate_path_exists(validator: Validator, mocker: MockerFixture) -> No
mocker.patch("pathlib.Path.exists", return_value=False)
validator._validate_path_exists(False, "field", Path("1"))
mocker.patch("pathlib.Path.exists", return_value=False)
validator._validate_path_exists(True, "field", Path("2"))
mocker.patch("pathlib.Path.exists", return_value=True)
validator._validate_path_exists(False, "field", Path("2"))
mocker.patch("pathlib.Path.exists", return_value=False)
validator._validate_path_exists(True, "field", Path("3"))
mocker.patch("pathlib.Path.exists", return_value=True)
validator._validate_path_exists(True, "field", Path("4"))
error_mock.assert_has_calls([
MockCall("field", "Path 2 must exist"),
MockCall("field", "Path 2 must not exist"),
MockCall("field", "Path 3 must exist"),
])

View File

@ -0,0 +1,34 @@
import pytest
from ahriman.core.configuration import Configuration
from ahriman.core.support.package_creator import PackageCreator
from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator
@pytest.fixture
def mirrorlist_generator(configuration: Configuration) -> MirrorlistGenerator:
"""
fixture for mirrorlist pkgbuild generator
Args:
configuration(Configuration): configuration fixture
Returns:
MirrorlistGenerator: mirrorlist pkgbuild generator test instance
"""
return MirrorlistGenerator(configuration, "mirrorlist")
@pytest.fixture
def package_creator(configuration: Configuration, mirrorlist_generator: MirrorlistGenerator) -> PackageCreator:
"""
package creator fixture
Args:
configuration(Configuration): configuration fixture
mirrorlist_generator(MirrorlistGenerator):
Returns:
PackageCreator: package creator test instance
"""
return PackageCreator(configuration, mirrorlist_generator)

View File

@ -0,0 +1,14 @@
import pytest
from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator
@pytest.fixture
def pkgbuild_generator() -> PkgbuildGenerator:
"""
fixture for dummy pkgbuild generator
Returns:
PkgbuildGenerator: pkgbuild generator test instance
"""
return PkgbuildGenerator()

View File

@ -0,0 +1,97 @@
from pathlib import Path
from unittest.mock import MagicMock
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator
def test_init_path(configuration: Configuration) -> None:
"""
must set relative path to mirrorlist
"""
assert MirrorlistGenerator(configuration, "mirrorlist").path == Path("etc") / "pacman.d" / "aur-clone-mirrorlist"
configuration.set_option("mirrorlist", "path", "/etc")
assert MirrorlistGenerator(configuration, "mirrorlist").path == Path("etc")
def test_license(configuration: Configuration) -> None:
"""
must generate correct licenses list
"""
assert MirrorlistGenerator(configuration, "mirrorlist").license == ["Unlicense"]
configuration.set_option("mirrorlist", "license", "GPL MPL")
assert MirrorlistGenerator(configuration, "mirrorlist").license == ["GPL", "MPL"]
def test_pkgdesc(configuration: Configuration) -> None:
"""
must generate correct pkgdesc property
"""
assert MirrorlistGenerator(configuration, "mirrorlist").pkgdesc == "aur-clone mirror list for use by pacman"
configuration.set_option("mirrorlist", "description", "description")
assert MirrorlistGenerator(configuration, "mirrorlist").pkgdesc == "description"
def test_pkgname(configuration: Configuration) -> None:
"""
must generate correct pkgname property
"""
assert MirrorlistGenerator(configuration, "mirrorlist").pkgname == "aur-clone-mirrorlist"
configuration.set_option("mirrorlist", "package", "mirrorlist")
assert MirrorlistGenerator(configuration, "mirrorlist").pkgname == "mirrorlist"
def test_url(configuration: Configuration) -> None:
"""
must generate correct url property
"""
assert MirrorlistGenerator(configuration, "mirrorlist").url == ""
configuration.set_option("mirrorlist", "homepage", "homepage")
assert MirrorlistGenerator(configuration, "mirrorlist").url == "homepage"
def test_generate_mirrorlist(mirrorlist_generator: MirrorlistGenerator, mocker: MockerFixture) -> None:
"""
must correctly generate mirrorlist file
"""
path = Path("local")
file_mock = MagicMock()
open_mock = mocker.patch("pathlib.Path.open")
open_mock.return_value.__enter__.return_value = file_mock
mirrorlist_generator._generate_mirrorlist(path)
open_mock.assert_called_once_with("w")
file_mock.writelines.assert_called_once_with(["Server = http://localhost\n"])
def test_package(mirrorlist_generator: MirrorlistGenerator) -> None:
"""
must generate package function correctly
"""
assert mirrorlist_generator.package() == """{
install -Dm644 "$srcdir/mirrorlist" "$pkgdir/etc/pacman.d/aur-clone-mirrorlist"
}"""
def test_patches(mirrorlist_generator: MirrorlistGenerator) -> None:
"""
must generate additional patch list
"""
patches = {patch.key: patch for patch in mirrorlist_generator.patches()}
assert "backup" in patches
assert patches["backup"].value == [str(mirrorlist_generator.path)]
def test_sources(mirrorlist_generator: MirrorlistGenerator) -> None:
"""
must return valid sources files list
"""
assert mirrorlist_generator.sources().get("mirrorlist")

View File

@ -0,0 +1,112 @@
import datetime
import pytest
from pathlib import Path
from pytest_mock import MockerFixture
from unittest.mock import MagicMock, call as MockCall
from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator
from ahriman.core.util import utcnow
from ahriman.models.pkgbuild_patch import PkgbuildPatch
def test_license(pkgbuild_generator: PkgbuildGenerator) -> None:
"""
must return empty license list
"""
assert pkgbuild_generator.license == []
def test_pkgdesc(pkgbuild_generator: PkgbuildGenerator) -> None:
"""
must raise NotImplementedError on missing pkgdesc property
"""
with pytest.raises(NotImplementedError):
assert pkgbuild_generator.pkgdesc
def test_pkgname(pkgbuild_generator: PkgbuildGenerator) -> None:
"""
must raise NotImplementedError on missing pkgname property
"""
with pytest.raises(NotImplementedError):
assert pkgbuild_generator.pkgname
def test_pkgver(pkgbuild_generator: PkgbuildGenerator, mocker: MockerFixture) -> None:
"""
must implement default version as current date
"""
mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.utcnow", return_value=datetime.datetime(2002, 3, 11))
assert pkgbuild_generator.pkgver == utcnow().strftime("20020311")
def test_url(pkgbuild_generator: PkgbuildGenerator) -> None:
"""
must return empty url
"""
assert pkgbuild_generator.url == ""
def test_package(pkgbuild_generator: PkgbuildGenerator) -> None:
"""
must raise NotImplementedError on missing package function
"""
with pytest.raises(NotImplementedError):
pkgbuild_generator.package()
def test_patches(pkgbuild_generator: PkgbuildGenerator) -> None:
"""
must return empty patches list
"""
assert pkgbuild_generator.patches() == []
def test_sources(pkgbuild_generator: PkgbuildGenerator) -> None:
"""
must return empty sources list
"""
assert pkgbuild_generator.sources() == {}
def test_write_pkgbuild(pkgbuild_generator: PkgbuildGenerator, mocker: MockerFixture) -> None:
"""
must write PKGBUILD content to file
"""
path = Path("local")
for prop in ("pkgdesc", "pkgname"):
mocker.patch.object(PkgbuildGenerator, prop, "")
mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.package", return_value="{}")
patches_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.patches",
return_value=[PkgbuildPatch("property", "value")])
sources_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.write_sources",
return_value=[PkgbuildPatch("source", []), PkgbuildPatch("sha512sums", [])])
write_mock = mocker.patch("ahriman.models.pkgbuild_patch.PkgbuildPatch.write")
pkgbuild_generator.write_pkgbuild(path)
patches_mock.assert_called_once_with()
sources_mock.assert_called_once_with(path)
write_mock.assert_has_calls([MockCall(path / "PKGBUILD")] * 11)
def test_write_sources(pkgbuild_generator: PkgbuildGenerator, mocker: MockerFixture) -> None:
"""
must write sources files
"""
path = Path("local")
generator_mock = MagicMock()
sources_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.sources",
return_value={"source": generator_mock})
open_mock = mocker.patch("pathlib.Path.open")
hash_mock = MagicMock()
hash_mock.hexdigest.return_value = "hash"
mocker.patch("hashlib.sha512", return_value=hash_mock)
assert pkgbuild_generator.write_sources(path) == [
PkgbuildPatch("source", ["source"]),
PkgbuildPatch("sha512sums", ["hash"]),
]
generator_mock.assert_called_once_with(path / "source")
sources_mock.assert_called_once_with()
open_mock.assert_called_once_with("rb")

View File

@ -0,0 +1,27 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.support import MirrorlistTrigger
def test_configuration_sections(configuration: Configuration) -> None:
"""
must correctly parse target list
"""
configuration.set_option("mirrorlist", "target", "a b c")
assert MirrorlistTrigger.configuration_sections(configuration) == ["a", "b", "c"]
configuration.remove_option("mirrorlist", "target")
assert MirrorlistTrigger.configuration_sections(configuration) == []
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run report for specified targets
"""
configuration.set_option("mirrorlist", "target", "mirrorlist")
run_mock = mocker.patch("ahriman.core.support.package_creator.PackageCreator.run")
trigger = MirrorlistTrigger("x86_64", configuration)
trigger.on_start()
run_mock.assert_called_once_with()

View File

@ -0,0 +1,40 @@
import pytest
from pytest_mock import MockerFixture
from ahriman.core.database import SQLite
from ahriman.core.support.package_creator import PackageCreator
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
def test_run(package_creator: PackageCreator, database: SQLite, mocker: MockerFixture) -> None:
"""
must correctly process package creation
"""
package = Package(
base=package_creator.generator.pkgname,
version=package_creator.generator.pkgver,
remote=None,
packages={package_creator.generator.pkgname: PackageDescription()},
)
local_path = package_creator.configuration.repository_paths.cache_for(package_creator.generator.pkgname)
rmtree_mock = mocker.patch("shutil.rmtree")
database_mock = mocker.patch("ahriman.core._Context.get", return_value=database)
init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init")
insert_mock = mocker.patch("ahriman.core.database.SQLite.package_update")
mkdir_mock = mocker.patch("pathlib.Path.mkdir")
package_mock = mocker.patch("ahriman.models.package.Package.from_build", return_value=package)
write_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.write_pkgbuild")
package_creator.run()
rmtree_mock.assert_called_once_with(local_path, ignore_errors=True)
mkdir_mock.assert_called_once_with(mode=0o755, parents=True, exist_ok=True)
write_mock.assert_called_once_with(local_path)
init_mock.assert_called_once_with(local_path)
package_mock.assert_called_once_with(local_path, "x86_64")
database_mock.assert_called_once_with(ContextKey("database", SQLite))
insert_mock.assert_called_once_with(package, pytest.helpers.anyvar(int))

View File

@ -81,12 +81,12 @@ def test_check_output_with_user(passwd: Any, mocker: MockerFixture) -> None:
"""
must run command as specified user and set its homedir
"""
assert check_output("python", "-c", "import os; print(os.getenv('HOME'))") != passwd.pw_dir
assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""") != passwd.pw_dir
getpwuid_mock = mocker.patch("ahriman.core.util.getpwuid", return_value=passwd)
user = os.getuid()
assert check_output("python", "-c", "import os; print(os.getenv('HOME'))", user=user) == passwd.pw_dir
assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""", user=user) == passwd.pw_dir
getpwuid_mock.assert_called_once_with(user)

View File

@ -11,6 +11,17 @@ from ahriman.models.package import Package
from ahriman.models.result import Result
def test_known_triggers(configuration: Configuration) -> None:
"""
must return used triggers
"""
configuration.set_option("build", "triggers_known", "a b c")
assert TriggerLoader.known_triggers(configuration) == ["a", "b", "c"]
configuration.remove_option("build", "triggers_known")
assert TriggerLoader.known_triggers(configuration) == []
def test_selected_triggers(configuration: Configuration) -> None:
"""
must return used triggers

View File

@ -25,6 +25,7 @@ ignore_packages =
makechrootpkg_flags =
makepkg_flags = --skippgpcheck
triggers = ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger
triggers_known = ahriman.core.support.MirrorlistTrigger
[repository]
name = aur-clone
@ -33,6 +34,10 @@ root = ../../../
[sign]
target =
[mirrorlist]
target = mirrorlist
servers = http://localhost
[remote-push]
target = gitremote