arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-23 23:07:17 +00:00
Code Issues Projects Releases Wiki Activity

handle packages which have been removed from the repository (#45)

Browse Source 
* handle packages which have been removed from the repository

* manually remove packages which have been removed from the base
This commit is contained in:
Evgenii Alekseev 2021-11-10 01:37:25 +03:00 committed by GitHub
parent 5b678149dc
commit 2752e68ade
13 changed files with 228 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/ahriman/application/application/repository.py
View File

@ -105,27 +105,37 @@ class Repository(Properties):
targets = target or None
self.repository.process_sync(targets, built_packages)
def unknown(self) -> List[Package]:
def unknown(self) -> List[str]:
"""
get packages which were not found in AUR
:return: unknown package list
:return: unknown package archive list
"""
def has_aur(package_base: str, aur_url: str) -> bool:
try:
_ = Package.from_aur(package_base, aur_url)
except Exception:
return False
return True
def has_local(package_base: str) -> bool:
cache_dir = self.repository.paths.cache_for(package_base)
def has_local(probe: Package) -> bool:
cache_dir = self.repository.paths.cache_for(probe.base)
return cache_dir.is_dir() and not Sources.has_remotes(cache_dir)
return [
package
for package in self.repository.packages()
if not has_aur(package.base, package.aur_url) and not has_local(package.base)
]
def unknown_aur(probe: Package) -> List[str]:
packages: List[str] = []
for single in probe.packages:
try:
_ = Package.from_aur(single, probe.aur_url)
except Exception:
packages.append(single)
return packages
def unknown_local(probe: Package) -> List[str]:
cache_dir = self.repository.paths.cache_for(probe.base)
local = Package.from_build(cache_dir, probe.aur_url)
packages = set(probe.packages.keys()).difference(local.packages.keys())
return list(packages)
result = []
for package in self.repository.packages():
if has_local(package):
result.extend(unknown_local(package)) # there is local package
else:
result.extend(unknown_aur(package)) # local package not found
return result
def update(self, updates: Iterable[Package]) -> None:
"""

10
src/ahriman/application/formatters/status_printer.py
View File

@ -17,11 +17,10 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import List, Optional
from typing import Optional
from ahriman.application.formatters.printer import Printer
from ahriman.models.build_status import BuildStatus
from ahriman.models.property import Property
class StatusPrinter(Printer):
@ -36,13 +35,6 @@ class StatusPrinter(Printer):
"""
self.content = status
def properties(self) -> List[Property]:
"""
convert content into printable data
:return: list of content properties
"""
return []
def title(self) -> Optional[str]:
"""
generate entry title from content

42
src/ahriman/application/formatters/string_printer.py Normal file
View File

@ -0,0 +1,42 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import Optional
from ahriman.application.formatters.printer import Printer
class StringPrinter(Printer):
"""
print content of the random string
"""
def __init__(self, content: str) -> None:
"""
default constructor
:param content: any content string
"""
self.content = content
def title(self) -> Optional[str]:
"""
generate entry title from content
:return: content title if it can be generated and None otherwise
"""
return self.content

9
src/ahriman/application/handlers/remove_unknown.py
View File

@ -22,10 +22,9 @@ import argparse
from typing import Type
from ahriman.application.application import Application
from ahriman.application.formatters.package_printer import PackagePrinter
from ahriman.application.formatters.string_printer import StringPrinter
from ahriman.application.handlers.handler import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.build_status import BuildStatus
class RemoveUnknown(Handler):
@ -46,8 +45,8 @@ class RemoveUnknown(Handler):
application = Application(architecture, configuration, no_report)
unknown_packages = application.unknown()
if args.dry_run:
for package in unknown_packages:
PackagePrinter(package, BuildStatus()).print(args.info)
for package in sorted(unknown_packages):
StringPrinter(package).print(args.info)
return
application.remove(package.base for package in unknown_packages)
application.remove(unknown_packages)

18
src/ahriman/core/exceptions.py
View File

@ -153,8 +153,12 @@ class UnknownPackage(ValueError):
exception for status watcher which will be thrown on unknown package
"""
def __init__(self, base: str) -> None:
ValueError.__init__(self, f"Package base {base} is unknown")
def __init__(self, package_base: str) -> None:
"""
default constructor
:param package_base: package base name
"""
ValueError.__init__(self, f"Package base {package_base} is unknown")
class UnsafeRun(RuntimeError):
@ -165,9 +169,9 @@ class UnsafeRun(RuntimeError):
def __init__(self, current_uid: int, root_uid: int) -> None:
"""
default constructor
:param current_uid: current user ID
:param root_uid: ID of the owner of root directory
"""
RuntimeError.__init__(
self,
f"""Current UID {current_uid} differs from root owner {root_uid}.
Note that for the most actions it is unsafe to run application as different user.
If you are 100% sure that it must be there try --unsafe option""")
RuntimeError.__init__(self, f"Current UID {current_uid} differs from root owner {root_uid}. "
f"Note that for the most actions it is unsafe to run application as different user."
f" If you are 100% sure that it must be there try --unsafe option")

30
src/ahriman/core/repository/executor.py
View File

@ -20,14 +20,13 @@
import shutil
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from typing import Iterable, List, Optional, Set
from ahriman.core.build_tools.task import Task
from ahriman.core.report.report import Report
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.upload.upload import Upload
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
class Executor(Cleaner):
@ -35,6 +34,14 @@ class Executor(Cleaner):
trait for common repository update processes
"""
def load_archives(self, packages: Iterable[Path]) -> List[Package]:
"""
load packages from list of archives
:param packages: paths to package archives
:return: list of read packages
"""
raise NotImplementedError
def packages(self) -> List[Package]:
"""
generate list of repository packages
@ -152,23 +159,24 @@ class Executor(Cleaner):
package_path = self.paths.repository / name
self.repo.add(package_path)
# we are iterating over bases, not single packages
updates: Dict[str, Package] = {}
for filename in packages:
try:
local = Package.load(str(filename), PackageSource.Archive, self.pacman, self.aur_url)
updates.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception("could not load package from %s", filename)
current_packages = self.packages()
removed_packages: List[str] = [] # list of packages which have been removed from the base
updates = self.load_archives(packages)
for local in updates.values():
for local in updates:
try:
for description in local.packages.values():
update_single(description.filename, local.base)
self.reporter.set_success(local)
current_package_archives: Set[str] = next(
(set(current.packages) for current in current_packages if current.base == local.base), set())
removed_packages.extend(current_package_archives.difference(local.packages))
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception("could not process %s", local.base)
self.clear_packages()
self.process_remove(removed_packages)
return self.repo.repo_path

35
src/ahriman/core/repository/repository.py
View File

@ -18,7 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pathlib import Path
from typing import Dict, List
from typing import Dict, Iterable, List
from ahriman.core.repository.executor import Executor
from ahriman.core.repository.update_handler import UpdateHandler
@ -32,20 +32,35 @@ class Repository(Executor, UpdateHandler):
base repository control class
"""
def load_archives(self, packages: Iterable[Path]) -> List[Package]:
"""
load packages from list of archives
:param packages: paths to package archives
:return: list of read packages
"""
result: Dict[str, Package] = {}
# we are iterating over bases, not single packages
for full_path in packages:
try:
local = Package.load(str(full_path), PackageSource.Archive, self.pacman, self.aur_url)
current = result.setdefault(local.base, local)
if current.version != local.version:
# force version to max of them
self.logger.warning("version of %s differs, found %s and %s",
current.base, current.version, local.version)
if current.is_outdated(local, self.paths, calculate_version=False):
current.version = local.version
current.packages.update(local.packages)
except Exception:
self.logger.exception("could not load package from %s", full_path)
return list(result.values())
def packages(self) -> List[Package]:
"""
generate list of repository packages
:return: list of packages properties
"""
result: Dict[str, Package] = {}
for full_path in filter(package_like, self.paths.repository.iterdir()):
try:
local = Package.load(str(full_path), PackageSource.Archive, self.pacman, self.aur_url)
result.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception("could not load package from %s", full_path)
continue
return list(result.values())
return self.load_archives(filter(package_like, self.paths.repository.iterdir()))
def packages_built(self) -> List[Path]:
"""

5
src/ahriman/models/package.py
View File

@ -257,14 +257,15 @@ class Package:
return self.version
def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool:
def is_outdated(self, remote: Package, paths: RepositoryPaths, calculate_version: bool = True) -> bool:
"""
check if package is out-of-dated
:param remote: package properties from remote source
:param paths: repository paths instance. Required for VCS packages cache
:param calculate_version: expand version to actual value (by calculating git versions)
:return: True if the package is out-of-dated and False otherwise
"""
remote_version = remote.actual_version(paths) # either normal version or updated VCS
remote_version = remote.actual_version(paths) if calculate_version else remote.version
result: int = vercmp(self.version, remote_version)
return result < 0

3
tests/ahriman/application/application/test_application_repository.py
View File

@ -147,6 +147,7 @@ def test_unknown_no_aur(application_repository: Repository, package_ahriman: Pac
"""
mocker.patch("ahriman.core.repository.repository.Repository.packages", return_value=[package_ahriman])
mocker.patch("ahriman.models.package.Package.from_aur", side_effect=Exception())
mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman)
mocker.patch("pathlib.Path.is_dir", return_value=True)
mocker.patch("ahriman.core.build_tools.sources.Sources.has_remotes", return_value=False)
@ -163,7 +164,7 @@ def test_unknown_no_aur_no_local(application_repository: Repository, package_ahr
mocker.patch("pathlib.Path.is_dir", return_value=False)
packages = application_repository.unknown()
assert packages == [package_ahriman]
assert packages == list(package_ahriman.packages.keys())
def test_unknown_no_local(application_repository: Repository, package_ahriman: Package, mocker: MockerFixture) -> None:

10
tests/ahriman/application/formatters/conftest.py
View File

@ -5,6 +5,7 @@ from ahriman.application.formatters.aur_printer import AurPrinter
from ahriman.application.formatters.configuration_printer import ConfigurationPrinter
from ahriman.application.formatters.package_printer import PackagePrinter
from ahriman.application.formatters.status_printer import StatusPrinter
from ahriman.application.formatters.string_printer import StringPrinter
from ahriman.application.formatters.update_printer import UpdatePrinter
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
@ -48,6 +49,15 @@ def status_printer() -> StatusPrinter:
return StatusPrinter(BuildStatus())
@pytest.fixture
def string_printer() -> StringPrinter:
"""
fixture for any string printer
:return: any string printer test instance
"""
return StringPrinter("hello, world")
@pytest.fixture
def update_printer(package_ahriman: Package) -> UpdatePrinter:
"""

15
tests/ahriman/application/formatters/test_string_printer.py Normal file
View File

@ -0,0 +1,15 @@
from ahriman.application.formatters.string_printer import StringPrinter
def test_properties(string_printer: StringPrinter) -> None:
"""
must return empty properties list
"""
assert not string_printer.properties()
def test_title(string_printer: StringPrinter) -> None:
"""
must return non empty title
"""
assert string_printer.title() is not None

43
tests/ahriman/core/repository/test_executor.py
View File

@ -11,6 +11,14 @@ from ahriman.core.upload.upload import Upload
from ahriman.models.package import Package
def test_load_archives(executor: Executor) -> None:
"""
must raise NotImplemented for missing load_archives method
"""
with pytest.raises(NotImplementedError):
executor.load_archives([])
def test_packages(executor: Executor) -> None:
"""
must raise NotImplemented for missing method
@ -182,11 +190,13 @@ def test_process_update(executor: Executor, package_ahriman: Package, mocker: Mo
"""
must run update process
"""
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman])
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
move_mock = mocker.patch("shutil.move")
repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add")
sign_package_mock = mocker.patch("ahriman.core.sign.gpg.GPG.process_sign_package", side_effect=lambda fn, _: [fn])
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_success")
remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove")
# must return complete
assert executor.process_update([package.filepath for package in package_ahriman.packages.values()])
@ -201,6 +211,8 @@ def test_process_update(executor: Executor, package_ahriman: Package, mocker: Mo
# must clear directory
from ahriman.core.repository.cleaner import Cleaner
Cleaner.clear_packages.assert_called_once()
# clear removed packages
remove_mock.assert_called_once_with([])
def test_process_update_group(executor: Executor, package_python_schedule: Package,
@ -209,9 +221,11 @@ def test_process_update_group(executor: Executor, package_python_schedule: Packa
must group single packages under one base
"""
mocker.patch("shutil.move")
mocker.patch("ahriman.models.package.Package.load", return_value=package_python_schedule)
mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_python_schedule])
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule])
repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_success")
remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove")
executor.process_update([package.filepath for package in package_python_schedule.packages.values()])
repo_add_mock.assert_has_calls([
@ -219,6 +233,7 @@ def test_process_update_group(executor: Executor, package_python_schedule: Packa
for package in package_python_schedule.packages.values()
], any_order=True)
status_client_mock.assert_called_once_with(package_python_schedule)
remove_mock.assert_called_once_with([])
def test_process_empty_filename(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
@ -226,7 +241,8 @@ def test_process_empty_filename(executor: Executor, package_ahriman: Package, mo
must skip update for package which does not have path
"""
package_ahriman.packages[package_ahriman.base].filename = None
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman])
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
executor.process_update([package.filepath for package in package_ahriman.packages.values()])
@ -235,18 +251,27 @@ def test_process_update_failed(executor: Executor, package_ahriman: Package, moc
must process update for failed package
"""
mocker.patch("shutil.move", side_effect=Exception())
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman])
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_failed")
executor.process_update([package.filepath for package in package_ahriman.packages.values()])
status_client_mock.assert_called_once()
def test_process_update_failed_on_load(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
def test_process_update_removed_package(executor: Executor, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must process update even with failed package load
must remove packages which have been removed from the new base
"""
mocker.patch("shutil.move")
mocker.patch("ahriman.models.package.Package.load", side_effect=Exception())
without_python2 = Package.from_json(package_python_schedule.view())
del without_python2.packages["python2-schedule"]
assert executor.process_update([package.filepath for package in package_ahriman.packages.values()])
mocker.patch("shutil.move")
mocker.patch("ahriman.core.alpm.repo.Repo.add")
mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[without_python2])
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule])
remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove")
executor.process_update([package.filepath for package in without_python2.packages.values()])
remove_mock.assert_called_once_with(["python2-schedule"])

48
tests/ahriman/core/repository/test_repository.py
View File

@ -5,8 +5,8 @@ from ahriman.core.repository import Repository
from ahriman.models.package import Package
def test_packages(package_ahriman: Package, package_python_schedule: Package,
repository: Repository, mocker: MockerFixture) -> None:
def test_load_archives(package_ahriman: Package, package_python_schedule: Package,
repository: Repository, mocker: MockerFixture) -> None:
"""
must return all packages grouped by package base
"""
@ -17,12 +17,9 @@ def test_packages(package_ahriman: Package, package_python_schedule: Package,
packages={package: props})
for package, props in package_python_schedule.packages.items()
] + [package_ahriman]
mocker.patch("pathlib.Path.iterdir",
return_value=[Path("a.pkg.tar.xz"), Path("b.pkg.tar.xz"), Path("c.pkg.tar.xz")])
mocker.patch("ahriman.models.package.Package.load", side_effect=single_packages)
packages = repository.packages()
packages = repository.load_archives([Path("a.pkg.tar.xz"), Path("b.pkg.tar.xz"), Path("c.pkg.tar.xz")])
assert len(packages) == 2
assert {package.base for package in packages} == {package_ahriman.base, package_python_schedule.base}
@ -33,21 +30,48 @@ def test_packages(package_ahriman: Package, package_python_schedule: Package,
assert set(archives) == expected
def test_packages_failed(repository: Repository, mocker: MockerFixture) -> None:
def test_load_archives_failed(repository: Repository, mocker: MockerFixture) -> None:
"""
must skip packages which cannot be loaded
"""
mocker.patch("pathlib.Path.iterdir", return_value=[Path("a.pkg.tar.xz")])
mocker.patch("ahriman.models.package.Package.load", side_effect=Exception())
assert not repository.packages()
assert not repository.load_archives([Path("a.pkg.tar.xz")])
def test_packages_not_package(repository: Repository, mocker: MockerFixture) -> None:
def test_load_archives_not_package(repository: Repository) -> None:
"""
must skip not packages from iteration
"""
mocker.patch("pathlib.Path.iterdir", return_value=[Path("a.tar.xz")])
assert not repository.packages()
assert not repository.load_archives([Path("a.tar.xz")])
def test_load_archives_different_version(repository: Repository, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must load packages with different versions choosing maximal
"""
single_packages = [
Package(base=package_python_schedule.base,
version=package_python_schedule.version,
aur_url=package_python_schedule.aur_url,
packages={package: props})
for package, props in package_python_schedule.packages.items()
]
single_packages[0].version = "0.0.1-1"
mocker.patch("ahriman.models.package.Package.load", side_effect=single_packages)
packages = repository.load_archives([Path("a.pkg.tar.xz"), Path("b.pkg.tar.xz")])
assert len(packages) == 1
assert packages[0].version == package_python_schedule.version
def test_packages(repository: Repository, mocker: MockerFixture) -> None:
"""
must return repository packages
"""
load_mock = mocker.patch("ahriman.core.repository.repository.Repository.load_archives")
repository.packages()
load_mock.assert_called_once() # it uses filter object so we cannot verity argument list =/
def test_packages_built(repository: Repository, mocker: MockerFixture) -> None: