arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-06-28 06:41:43 +00:00
Code Issues Projects Releases Wiki Activity

implement single-function patches (#69)

Browse Source
This commit is contained in:
Evgenii Alekseev
2022-10-30 03:11:03 +03:00
committed by GitHub
parent 1e8388af5d
commit 73e311a41c
25 changed files with 632 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/ahriman/application/ahriman.py
View File

@ -93,6 +93,7 @@ def _parser() -> argparse.ArgumentParser:
_set_patch_add_parser(subparsers)
_set_patch_list_parser(subparsers)
_set_patch_remove_parser(subparsers)
_set_patch_set_add_parser(subparsers)
_set_repo_backup_parser(subparsers)
_set_repo_check_parser(subparsers)
_set_repo_clean_parser(subparsers)
@ -320,7 +321,7 @@ def _set_package_status_update_parser(root: SubParserAction) -> argparse.Argumen
def _set_patch_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for new patch subcommand
add parser for new single-function patch subcommand
Args:
root(SubParserAction): subparsers for the commands
@ -328,16 +329,18 @@ def _set_patch_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
Returns:
argparse.ArgumentParser: created argument parser
"""
parser = root.add_parser("patch-add", help="add patch set", description="create or update source patches",
epilog="In order to add a patch set for the package you will need to clone "
"the AUR package manually, add required changes (e.g. external patches, "
"edit PKGBUILD) and run command, e.g. ``ahriman patch path/to/directory``. "
"By default it tracks *.patch and *.diff files, but this behavior can be changed "
"by using --track option",
parser = root.add_parser("patch-add", help="add patch for PKGBUILD function",
description="create or update patched PKGBUILD function or variable",
epilog="Unlike ``patch-set-add``, this function allows to patch only one PKGBUILD f"
"unction, e.g. typing ``ahriman patch-add ahriman version`` it will change the "
"``version`` inside PKGBUILD, typing ``ahriman patch-add ahriman build()`` "
"it will change ``build()`` function inside PKGBUILD",
formatter_class=_formatter)
parser.add_argument("package", help="path to directory with changed files for patch addition/update")
parser.add_argument("-t", "--track", help="files which has to be tracked", action="append",
default=["*.diff", "*.patch"])
parser.add_argument("package", help="package base")
parser.add_argument("variable", help="PKGBUILD variable or function name. If variable is a function, "
"it must end with ()")
parser.add_argument("patch", help="path to file which contains function or variable value. If not set, "
"the value will be read from stdin", type=Path, nargs="?")
parser.set_defaults(handler=handlers.Patch, action=Action.Update, architecture=[""], lock=None, no_report=True)
return parser
@ -356,6 +359,8 @@ def _set_patch_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
description="list available patches for the package", formatter_class=_formatter)
parser.add_argument("package", help="package base", nargs="?")
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("-v", "--variable", help="if set, show only patches for specified PKGBUILD variables",
action="append")
parser.set_defaults(handler=handlers.Patch, action=Action.List, architecture=[""], lock=None, no_report=True)
return parser
@ -373,10 +378,39 @@ def _set_patch_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("patch-remove", help="remove patch set", description="remove patches for the package",
formatter_class=_formatter)
parser.add_argument("package", help="package base")
parser.add_argument("-v", "--variable", help="should be used for single-function patches in case if you wold like "
"to remove only specified PKGBUILD variables. In case if not set, "
"it will remove all patches related to the package",
action="append")
parser.set_defaults(handler=handlers.Patch, action=Action.Remove, architecture=[""], lock=None, no_report=True)
return parser
def _set_patch_set_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for new full-diff patch subcommand
Args:
root(SubParserAction): subparsers for the commands
Returns:
argparse.ArgumentParser: created argument parser
"""
parser = root.add_parser("patch-set-add", help="add patch set", description="create or update source patches",
epilog="In order to add a patch set for the package you will need to clone "
"the AUR package manually, add required changes (e.g. external patches, "
"edit PKGBUILD) and run command, e.g. ``ahriman patch-set-add path/to/directory``. "
"By default it tracks *.patch and *.diff files, but this behavior can be changed "
"by using --track option",
formatter_class=_formatter)
parser.add_argument("package", help="path to directory with changed files for patch addition/update", type=Path)
parser.add_argument("-t", "--track", help="files which has to be tracked", action="append",
default=["*.diff", "*.patch"])
parser.set_defaults(handler=handlers.Patch, action=Action.Update, architecture=[""], lock=None, no_report=True,
variable=None)
return parser
def _set_repo_backup_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for repository backup subcommand

82
src/ahriman/application/handlers/patch.py
View File

@ -18,17 +18,19 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
import sys
from pathlib import Path
from typing import List, Optional, Type
from typing import List, Optional, Tuple, Type
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter
from ahriman.core.formatters import PatchPrinter
from ahriman.models.action import Action
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
class Patch(Handler):
@ -52,51 +54,93 @@ class Patch(Handler):
application = Application(architecture, configuration, no_report, unsafe)
application.on_start()
if args.action == Action.List:
Patch.patch_set_list(application, args.package, args.exit_code)
if args.action == Action.Update and args.variable is not None:
patch = Patch.patch_create_from_function(args.variable, args.patch)
Patch.patch_set_create(application, args.package, patch)
elif args.action == Action.Update and args.variable is None:
package_base, patch = Patch.patch_create_from_diff(args.package, args.track)
Patch.patch_set_create(application, package_base, patch)
elif args.action == Action.List:
Patch.patch_set_list(application, args.package, args.variable, args.exit_code)
elif args.action == Action.Remove:
Patch.patch_set_remove(application, args.package)
elif args.action == Action.Update:
Patch.patch_set_create(application, Path(args.package), args.track)
Patch.patch_set_remove(application, args.package, args.variable)
@staticmethod
def patch_set_create(application: Application, sources_dir: Path, track: List[str]) -> None:
def patch_create_from_diff(sources_dir: Path, track: List[str]) -> Tuple[str, PkgbuildPatch]:
"""
create PKGBUILD plain diff patches from sources directory
Args:
sources_dir(Path): path to directory with the package sources
track(List[str]): track files which match the glob before creating the patch
Returns:
Tuple[str, PkgbuildPatch]: package base and created PKGBUILD patch based on the diff from master HEAD
to current files
"""
package = Package.from_build(sources_dir)
patch = Sources.patch_create(sources_dir, *track)
return package.base, PkgbuildPatch(None, patch)
@staticmethod
def patch_create_from_function(variable: str, patch_path: Optional[Path]) -> PkgbuildPatch:
"""
create single-function patch set for the package base
Args:
variable(str): function or variable name inside PKGBUILD
patch_path(Path): optional path to patch content. If not set, it will be read from stdin
Returns:
PkgbuildPatch: created patch for the PKGBUILD function
"""
if patch_path is None:
print("Post new function or variable value below. Press Ctrl-D to finish:", file=sys.stderr)
patch = "".join(list(sys.stdin))
else:
patch = patch_path.read_text(encoding="utf8")
patch = patch.strip() # remove spaces around the patch
return PkgbuildPatch(variable, patch)
@staticmethod
def patch_set_create(application: Application, package_base: str, patch: PkgbuildPatch) -> None:
"""
create patch set for the package base
Args:
application(Application): application instance
sources_dir(Path): path to directory with the package sources
track(List[str]): track files which match the glob before creating the patch
package_base(str): package base
patch(PkgbuildPatch): patch descriptor
"""
package = Package.from_build(sources_dir)
patch = Sources.patch_create(sources_dir, *track)
application.database.patches_insert(package.base, patch)
application.database.patches_insert(package_base, patch)
@staticmethod
def patch_set_list(application: Application, package_base: Optional[str], exit_code: bool) -> None:
def patch_set_list(application: Application, package_base: Optional[str], variables: List[str],
exit_code: bool) -> None:
"""
list patches available for the package base
Args:
application(Application): application instance
package_base(Optional[str]): package base
variables(List[str]): extract patches only for specified PKGBUILD variables
exit_code(bool): exit with error on empty search result
:
"""
patches = application.database.patches_list(package_base)
patches = application.database.patches_list(package_base, variables)
Patch.check_if_empty(exit_code, not patches)
for base, patch in patches.items():
content = base if package_base is None else patch
StringPrinter(content).print(verbose=True)
PatchPrinter(base, patch).print(verbose=True, separator=" = ")
@staticmethod
def patch_set_remove(application: Application, package_base: str) -> None:
def patch_set_remove(application: Application, package_base: str, variables: List[str]) -> None:
"""
remove patch set for the package base
Args:
application(Application): application instance
package_base(str): package base
variables(List[str]): remove patches only for specified PKGBUILD variables
"""
application.database.patches_remove(package_base)
application.database.patches_remove(package_base, variables)

35
src/ahriman/core/build_tools/sources.py
View File

@ -45,24 +45,22 @@ class Sources(LazyLogging):
_check_output = check_output
@staticmethod
def extend_architectures(sources_dir: Path, architecture: str) -> None:
def extend_architectures(sources_dir: Path, architecture: str) -> List[PkgbuildPatch]:
"""
extend existing PKGBUILD with repository architecture
Args:
sources_dir(Path): local path to directory with source files
architecture(str): repository architecture
"""
pkgbuild_path = sources_dir / "PKGBUILD"
if not pkgbuild_path.is_file():
return
Returns:
List[PkgbuildPatch]: generated patch for PKGBUILD architectures if required
"""
architectures = Package.supported_architectures(sources_dir)
if "any" in architectures: # makepkg does not like when there is any other arch except for any
return
return []
architectures.add(architecture)
patch = PkgbuildPatch("arch", list(architectures))
patch.write(pkgbuild_path)
return [PkgbuildPatch("arch", list(architectures))]
@staticmethod
def fetch(sources_dir: Path, remote: Optional[RemoteSource]) -> None:
@ -134,14 +132,14 @@ class Sources(LazyLogging):
exception=None, cwd=sources_dir, logger=instance.logger)
@staticmethod
def load(sources_dir: Path, package: Package, patch: Optional[str], paths: RepositoryPaths) -> None:
def load(sources_dir: Path, package: Package, patches: List[PkgbuildPatch], paths: RepositoryPaths) -> None:
"""
fetch sources from remote and apply patches
Args:
sources_dir(Path): local path to fetch
package(Package): package definitions
patch(Optional[str]): optional patch to be applied
patches(List[PkgbuildPatch]): optional patch to be applied
paths(RepositoryPaths): repository paths instance
"""
instance = Sources()
@ -150,9 +148,9 @@ class Sources(LazyLogging):
shutil.copytree(cache_dir, sources_dir, dirs_exist_ok=True)
instance.fetch(sources_dir, package.remote)
if patch is not None:
patches.extend(instance.extend_architectures(sources_dir, paths.architecture))
for patch in patches:
instance.patch_apply(sources_dir, patch)
instance.extend_architectures(sources_dir, paths.architecture)
@staticmethod
def patch_create(sources_dir: Path, *pattern: str) -> str:
@ -248,15 +246,18 @@ class Sources(LazyLogging):
dst = sources_dir / src.relative_to(pkgbuild_dir)
shutil.move(src, dst)
def patch_apply(self, sources_dir: Path, patch: str) -> None:
def patch_apply(self, sources_dir: Path, patch: PkgbuildPatch) -> None:
"""
apply patches if any
Args:
sources_dir(Path): local path to directory with git sources
patch(str): patch to be applied
patch(PkgbuildPatch): patch to be applied
"""
# create patch
self.logger.info("apply patch from database")
Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace",
exception=None, cwd=sources_dir, input_data=patch, logger=self.logger)
self.logger.info("apply patch %s from database at %s", patch.key, sources_dir)
if patch.is_plain_diff:
Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace",
exception=None, cwd=sources_dir, input_data=patch.serialize(), logger=self.logger)
else:
patch.write(sources_dir / "PKGBUILD")

43
src/ahriman/core/database/migrations/m003_patch_variables.py Normal file
View File

@ -0,0 +1,43 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ["steps"]
steps = [
"""
alter table patches rename to patches_
""",
"""
create table patches (
package_base text not null,
variable text,
patch blob not null
)
""",
"""
create unique index patches_package_base_variable on patches (package_base, coalesce(variable, ''))
""",
"""
insert into patches (package_base, patch) select package_base, patch from patches_
""",
"""
drop table patches_
""",
]

54
src/ahriman/core/database/operations/patch_operations.py
View File

@ -17,10 +17,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from collections import defaultdict
from sqlite3 import Connection
from typing import Dict, Optional
from typing import Dict, List, Optional, Tuple
from ahriman.core.database.operations import Operations
from ahriman.models.pkgbuild_patch import PkgbuildPatch
class PatchOperations(Operations):
@ -28,7 +31,7 @@ class PatchOperations(Operations):
operations for patches
"""
def patches_get(self, package_base: str) -> Optional[str]:
def patches_get(self, package_base: str) -> List[PkgbuildPatch]:
"""
retrieve patches for the package
@ -36,62 +39,77 @@ class PatchOperations(Operations):
package_base(str): package base to search for patches
Returns:
Optional[str]: plain text patch for the package
List[PkgbuildPatch]: plain text patch for the package
"""
return self.patches_list(package_base).get(package_base)
return self.patches_list(package_base, []).get(package_base, [])
def patches_insert(self, package_base: str, patch: str) -> None:
def patches_insert(self, package_base: str, patch: PkgbuildPatch) -> None:
"""
insert or update patch in database
Args:
package_base(str): package base to insert
patch(str): patch content
patch(PkgbuildPatch): patch content
"""
def run(connection: Connection) -> None:
connection.execute(
"""
insert into patches
(package_base, patch)
(package_base, variable, patch)
values
(:package_base, :patch)
on conflict (package_base) do update set
(:package_base, :variable, :patch)
on conflict (package_base, coalesce(variable, '')) do update set
patch = :patch
""",
{"package_base": package_base, "patch": patch})
{"package_base": package_base, "variable": patch.key, "patch": patch.value})
return self.with_connection(run, commit=True)
def patches_list(self, package_base: Optional[str]) -> Dict[str, str]:
def patches_list(self, package_base: Optional[str], variables: List[str]) -> Dict[str, List[PkgbuildPatch]]:
"""
extract all patches
Args:
package_base(Optional[str]): optional filter by package base
variables(List[str]): extract patches only for specified PKGBUILD variables
Returns:
Dict[str, str]: map of package base to patch content
Dict[str, List[PkgbuildPatch]]: map of package base to patch content
"""
def run(connection: Connection) -> Dict[str, str]:
return {
cursor["package_base"]: cursor["patch"]
def run(connection: Connection) -> List[Tuple[str, PkgbuildPatch]]:
return [
(cursor["package_base"], PkgbuildPatch(cursor["variable"], cursor["patch"]))
for cursor in connection.execute(
"""select * from patches where :package_base is null or package_base = :package_base""",
{"package_base": package_base})
}
]
return self.with_connection(run)
# we could use itertools & operator but why?
patches: Dict[str, List[PkgbuildPatch]] = defaultdict(list)
for package, patch in self.with_connection(run):
if variables and patch.key not in variables:
continue
patches[package].append(patch)
return dict(patches)
def patches_remove(self, package_base: str) -> None:
def patches_remove(self, package_base: str, variables: List[str]) -> None:
"""
remove patch set
Args:
package_base(str): package base to clear patches
variables(List[str]): remove patches only for specified PKGBUILD variables
"""
def run_many(connection: Connection) -> None:
connection.executemany(
"""delete from patches where package_base = :package_base and variable = :variable""",
[{"package_base": package_base, "variable": variable} for variable in variables])
def run(connection: Connection) -> None:
connection.execute(
"""delete from patches where package_base = :package_base""",
{"package_base": package_base})
if variables:
return self.with_connection(run_many, commit=True)
return self.with_connection(run, commit=True)

1
src/ahriman/core/formatters/__init__.py
View File

@ -24,6 +24,7 @@ from ahriman.core.formatters.aur_printer import AurPrinter
from ahriman.core.formatters.build_printer import BuildPrinter
from ahriman.core.formatters.configuration_printer import ConfigurationPrinter
from ahriman.core.formatters.package_printer import PackagePrinter
from ahriman.core.formatters.patch_printer import PatchPrinter
from ahriman.core.formatters.status_printer import StatusPrinter
from ahriman.core.formatters.update_printer import UpdatePrinter
from ahriman.core.formatters.user_printer import UserPrinter

56
src/ahriman/core/formatters/patch_printer.py Normal file
View File

@ -0,0 +1,56 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import List
from ahriman.core.formatters import StringPrinter
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.property import Property
class PatchPrinter(StringPrinter):
"""
print content of the PKGBUILD patch
Attributes:
patches(List[PkgbuildPatch]): PKGBUILD patch object
"""
def __init__(self, package_base: str, patches: List[PkgbuildPatch]) -> None:
"""
default constructor
Args:
package_base(str): package base
patches(List[PkgbuildPatch]): PKGBUILD patch object
"""
StringPrinter.__init__(self, package_base)
self.patches = patches
def properties(self) -> List[Property]:
"""
convert content into printable data
Returns:
List[Property]: list of content properties
"""
return [
Property(patch.key or "Full source diff", patch.value, is_required=True)
for patch in self.patches
]

2
src/ahriman/core/repository/executor.py
View File

@ -109,7 +109,7 @@ class Executor(Cleaner):
try:
self.paths.tree_clear(package_base) # remove all internal files
self.database.build_queue_clear(package_base)
self.database.patches_remove(package_base)
self.database.patches_remove(package_base, [])
self.reporter.remove(package_base) # we only update status page in case of base removal
except Exception:
self.logger.exception("could not remove base %s", package_base)

2
src/ahriman/models/package.py
View File

@ -305,7 +305,7 @@ class Package(LazyLogging):
from ahriman.core.build_tools.sources import Sources
Sources.load(paths.cache_for(self.base), self, None, paths)
Sources.load(paths.cache_for(self.base), self, [], paths)
try:
# update pkgver first

27
src/ahriman/models/pkgbuild_patch.py
View File

@ -21,7 +21,7 @@ import shlex
from dataclasses import dataclass, field
from pathlib import Path
from typing import List, Union
from typing import List, Optional, Union
@dataclass(frozen=True)
@ -30,16 +30,23 @@ class PkgbuildPatch:
wrapper for patching PKBGUILDs
Attributes:
key(str): name of the property in PKGBUILD, e.g. version, url etc
key(Optional[str]): name of the property in PKGBUILD, e.g. version, url etc. If not set, patch will be
considered as full PKGBUILD diffs
value(Union[str, List[str]]): value of the stored PKGBUILD property. It must be either string or list of string
values
unsafe(bool): if set, value will be not quoted, might break PKGBUILD
"""
key: str
key: Optional[str]
value: Union[str, List[str]]
unsafe: bool = field(default=False, kw_only=True)
def __post_init__(self) -> None:
"""
remove empty key
"""
object.__setattr__(self, "key", self.key or None)
@property
def is_function(self) -> bool:
"""
@ -48,7 +55,17 @@ class PkgbuildPatch:
Returns:
bool: True in case if key ends with parentheses and False otherwise
"""
return self.key.endswith("()")
return self.key is not None and self.key.endswith("()")
@property
def is_plain_diff(self) -> bool:
"""
check if patch is full diff one or just single-variable patch
Returns:
bool: True in case key set and False otherwise
"""
return self.key is None
def quote(self, value: str) -> str:
"""
@ -74,6 +91,8 @@ class PkgbuildPatch:
if isinstance(self.value, list): # list like
value = " ".join(map(self.quote, self.value))
return f"""{self.key}=({value})"""
if self.is_plain_diff: # no additional logic for plain diffs
return self.value
# we suppose that function values are only supported in string-like values
if self.is_function:
return f"{self.key} {self.value}" # no quoting enabled here