arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-14 22:45:47 +00:00
Code Issues Projects Releases Wiki Activity

port part of settings to database (#54)

Browse Source
This commit is contained in:
Evgenii Alekseev
2022-03-31 01:48:06 +03:00
committed by GitHub
parent d4eadf0013
commit 83931f5cf4
117 changed files with 2768 additions and 1044 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/ahriman/application/ahriman.py
View File

@ -94,6 +94,7 @@ def _parser() -> argparse.ArgumentParser:
_set_repo_sync_parser(subparsers)
_set_repo_update_parser(subparsers)
_set_user_add_parser(subparsers)
_set_user_list_parser(subparsers)
_set_user_remove_parser(subparsers)
_set_web_parser(subparsers)
@ -126,6 +127,8 @@ def _set_help_commands_unsafe(root: SubParserAction) -> argparse.ArgumentParser:
"""
parser = root.add_parser("help-commands-unsafe", help="list unsafe commands",
description="list unsafe commands as defined in default args", formatter_class=_formatter)
parser.add_argument("--command", help="instead of showing commands, just test command line for unsafe subcommand "
"and return 0 in case if command is safe and 1 otherwise")
parser.set_defaults(handler=handlers.UnsafeCommands, architecture=[""], lock=None, no_report=True, quiet=True,
unsafe=True, parser=_parser)
return parser
@ -273,7 +276,7 @@ def _set_patch_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
parser = root.add_parser("patch-list", help="list patch sets",
description="list available patches for the package", formatter_class=_formatter)
parser.add_argument("package", help="package base")
parser.add_argument("package", help="package base", nargs="?")
parser.set_defaults(handler=handlers.Patch, action=Action.List, architecture=[""], lock=None, no_report=True)
return parser
@ -318,12 +321,10 @@ def _set_repo_clean_parser(root: SubParserAction) -> argparse.ArgumentParser:
"you should not run this command manually. Also in case if you are going to clear "
"the chroot directories you will need root privileges.",
formatter_class=_formatter)
parser.add_argument("--build", help="clear directory with package sources", action="store_true")
parser.add_argument("--cache", help="clear directory with package caches", action="store_true")
parser.add_argument("--chroot", help="clear build chroot", action="store_true")
parser.add_argument("--manual", help="clear directory with manually added packages", action="store_true")
parser.add_argument("--manual", help="clear manually added packages queue", action="store_true")
parser.add_argument("--packages", help="clear directory with built packages", action="store_true")
parser.add_argument("--patches", help="clear directory with patches", action="store_true")
parser.set_defaults(handler=handlers.Clean, quiet=True, unsafe=True)
return parser
@ -487,7 +488,6 @@ def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
formatter_class=_formatter)
parser.add_argument("username", help="username for web service")
parser.add_argument("--as-service", help="add user as service user", action="store_true")
parser.add_argument("--no-reload", help="do not reload authentication module", action="store_true")
parser.add_argument("-p", "--password", help="user password. Blank password will be treated as empty password, "
"which is in particular must be used for OAuth2 authorization type.")
parser.add_argument("-r", "--role", help="user access level",
@ -498,6 +498,22 @@ def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
return parser
def _set_user_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for user list subcommand
:param root: subparsers for the commands
:return: created argument parser
"""
parser = root.add_parser("user-list", help="user known users and their access",
description="list users from the user mapping and their roles",
formatter_class=_formatter)
parser.add_argument("username", help="filter users by username", nargs="?")
parser.add_argument("-r", "--role", help="filter users by role", type=UserAccess, choices=UserAccess)
parser.set_defaults(handler=handlers.User, action=Action.List, architecture=[""], lock=None, no_report=True, # nosec
password="", quiet=True, role=UserAccess.Read, unsafe=True)
return parser
def _set_user_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for user removal subcommand
@ -508,7 +524,6 @@ def _set_user_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
description="remove user from the user mapping and update the configuration",
formatter_class=_formatter)
parser.add_argument("username", help="username for web service")
parser.add_argument("--no-reload", help="do not reload authentication module", action="store_true")
parser.add_argument("-s", "--secure", help="set file permissions to user-only", action="store_true")
parser.set_defaults(handler=handlers.User, action=Action.Remove, architecture=[""], lock=None, no_report=True, # nosec
password="", quiet=True, role=UserAccess.Read, unsafe=True)

14
src/ahriman/application/application/packages.py
View File

@ -25,7 +25,7 @@ from typing import Any, Iterable, Set
from ahriman.application.application.properties import Properties
from ahriman.core.build_tools.sources import Sources
from ahriman.core.util import package_like
from ahriman.core.util import package_like, tmpdir
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
from ahriman.models.result import Result
@ -67,10 +67,11 @@ class Packages(Properties):
:param without_dependencies: if set, dependency check will be disabled
"""
package = Package.load(source, PackageSource.AUR, self.repository.pacman, self.repository.aur_url)
local_path = self.repository.paths.manual_for(package.base)
self.repository.database.build_queue_insert(package)
Sources.load(local_path, package.git_url, self.repository.paths.patches_for(package.base))
self._process_dependencies(local_path, known_packages, without_dependencies)
with tmpdir() as local_path:
Sources.load(local_path, package.git_url, self.database.patches_get(package.base))
self._process_dependencies(local_path, known_packages, without_dependencies)
def _add_directory(self, source: str, *_: Any) -> None:
"""
@ -92,10 +93,9 @@ class Packages(Properties):
cache_dir = self.repository.paths.cache_for(package.base)
shutil.copytree(Path(source), cache_dir) # copy package to store in caches
Sources.init(cache_dir) # we need to run init command in directory where we do have permissions
self.repository.database.build_queue_insert(package)
dst = self.repository.paths.manual_for(package.base)
shutil.copytree(cache_dir, dst) # copy package for the build
self._process_dependencies(dst, known_packages, without_dependencies)
self._process_dependencies(cache_dir, known_packages, without_dependencies)
def _add_remote(self, source: str, *_: Any) -> None:
"""

5
src/ahriman/application/application/properties.py
View File

@ -20,6 +20,7 @@
import logging
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.repository import Repository
@ -28,6 +29,7 @@ class Properties:
application base properties class
:ivar architecture: repository architecture
:ivar configuration: configuration instance
:ivar database: database instance
:ivar logger: application logger
:ivar repository: repository instance
"""
@ -43,4 +45,5 @@ class Properties:
self.logger = logging.getLogger("root")
self.configuration = configuration
self.architecture = architecture
self.repository = Repository(architecture, configuration, no_report, unsafe)
self.database = SQLite.load(configuration)
self.repository = Repository(architecture, configuration, self.database, no_report, unsafe)

12
src/ahriman/application/application/repository.py
View File

@ -42,28 +42,22 @@ class Repository(Properties):
"""
raise NotImplementedError
def clean(self, build: bool, cache: bool, chroot: bool, manual: bool, packages: bool, patches: bool) -> None:
def clean(self, cache: bool, chroot: bool, manual: bool, packages: bool) -> None:
"""
run all clean methods. Warning: some functions might not be available under non-root
:param build: clear directory with package sources
:param cache: clear directory with package caches
:param chroot: clear build chroot
:param manual: clear directory with manually added packages
:param packages: clear directory with built packages
:param patches: clear directory with patches
"""
if build:
self.repository.clear_build()
if cache:
self.repository.clear_cache()
if chroot:
self.repository.clear_chroot()
if manual:
self.repository.clear_manual()
self.repository.clear_queue()
if packages:
self.repository.clear_packages()
if patches:
self.repository.clear_patches()
def report(self, target: Iterable[str], result: Result) -> None:
"""
@ -154,7 +148,7 @@ class Repository(Properties):
process_update(packages, Result())
# process manual packages
tree = Tree.load(updates, self.repository.paths)
tree = Tree.load(updates, self.database)
for num, level in enumerate(tree.levels()):
self.logger.info("processing level #%i %s", num, [package.base for package in level])
build_result = self.repository.process_build(level)

2
src/ahriman/application/handlers/clean.py
View File

@ -43,4 +43,4 @@ class Clean(Handler):
:param unsafe: if set no user check will be performed before path creation
"""
Application(architecture, configuration, no_report, unsafe).clean(
args.build, args.cache, args.chroot, args.manual, args.packages, args.patches)
args.cache, args.chroot, args.manual, args.packages)

4
src/ahriman/application/handlers/handler.py
View File

@ -27,7 +27,7 @@ from typing import List, Type
from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import MissingArchitecture, MultipleArchitectures
from ahriman.core.exceptions import ExitCode, MissingArchitecture, MultipleArchitectures
from ahriman.models.repository_paths import RepositoryPaths
@ -78,6 +78,8 @@ class Handler:
with Lock(args, architecture, configuration):
cls.run(args, architecture, configuration, args.no_report, args.unsafe)
return True
except ExitCode:
return False
except Exception:
# we are basically always want to print error to stderr instead of default logger
logging.getLogger("stderr").exception("process exception")

28
src/ahriman/application/handlers/patch.py
View File

@ -18,15 +18,15 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
import shutil
from pathlib import Path
from typing import List, Type
from typing import List, Optional, Type
from ahriman.application.application import Application
from ahriman.application.handlers.handler import Handler
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.formatters.string_printer import StringPrinter
from ahriman.models.action import Action
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
@ -37,8 +37,6 @@ class Patch(Handler):
patch control handler
"""
_print = print
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
configuration: Configuration, no_report: bool, unsafe: bool) -> None:
@ -69,25 +67,20 @@ class Patch(Handler):
"""
package = Package.load(sources_dir, PackageSource.Local, application.repository.pacman,
application.repository.aur_url)
patch_dir = application.repository.paths.patches_for(package.base)
Patch.patch_set_remove(application, package.base) # remove old patches
patch_dir.mkdir(mode=0o755, parents=True)
Sources.patch_create(Path(sources_dir), patch_dir / "00-main.patch", *track)
patch = Sources.patch_create(Path(sources_dir), *track)
application.database.patches_insert(package.base, patch)
@staticmethod
def patch_set_list(application: Application, package_base: str) -> None:
def patch_set_list(application: Application, package_base: Optional[str]) -> None:
"""
list patches available for the package base
:param application: application instance
:param package_base: package base
"""
patch_dir = application.repository.paths.patches_for(package_base)
if not patch_dir.is_dir():
return
for patch_path in sorted(patch_dir.glob("*.patch")):
Patch._print(patch_path.name)
patches = application.database.patches_list(package_base)
for base, patch in patches.items():
content = base if package_base is None else patch
StringPrinter(content).print(verbose=True)
@staticmethod
def patch_set_remove(application: Application, package_base: str) -> None:
@ -96,5 +89,4 @@ class Patch(Handler):
:param application: application instance
:param package_base: package base
"""
patch_dir = application.repository.paths.patches_for(package_base)
shutil.rmtree(patch_dir, ignore_errors=True)
application.database.patches_remove(package_base)

24
src/ahriman/application/handlers/unsafe_commands.py
View File

@ -18,11 +18,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
import shlex
from typing import List, Type
from ahriman.application.handlers.handler import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExitCode
from ahriman.core.formatters.string_printer import StringPrinter
@ -44,9 +46,25 @@ class UnsafeCommands(Handler):
:param no_report: force disable reporting
:param unsafe: if set no user check will be performed before path creation
"""
unsafe_commands = UnsafeCommands.get_unsafe_commands(args.parser())
for command in unsafe_commands:
StringPrinter(command).print(verbose=True)
parser = args.parser()
unsafe_commands = UnsafeCommands.get_unsafe_commands(parser)
if args.command is None:
for command in unsafe_commands:
StringPrinter(command).print(verbose=True)
else:
UnsafeCommands.check_unsafe(args.command, unsafe_commands, parser)
@staticmethod
def check_unsafe(command: str, unsafe_commands: List[str], parser: argparse.ArgumentParser) -> None:
"""
check if command is unsafe
:param command: command to check
:param unsafe_commands: list of unsafe commands
:param parser: generated argument parser
"""
args = parser.parse_args(shlex.split(command))
if args.command in unsafe_commands:
raise ExitCode()
@staticmethod
def get_unsafe_commands(parser: argparse.ArgumentParser) -> List[str]:

54
src/ahriman/application/handlers/user.py
View File

@ -23,12 +23,12 @@ import getpass
from pathlib import Path
from typing import Type
from ahriman.application.application import Application
from ahriman.application.handlers.handler import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.formatters.user_printer import UserPrinter
from ahriman.models.action import Action
from ahriman.models.user import User as MUser
from ahriman.models.user_access import UserAccess
class User(Handler):
@ -51,33 +51,35 @@ class User(Handler):
"""
salt = User.get_salt(configuration)
user = User.user_create(args)
auth_configuration = User.configuration_get(configuration.include)
database = SQLite.load(configuration)
User.user_clear(auth_configuration, user)
if args.action == Action.Update:
User.configuration_create(auth_configuration, user, salt, args.as_service)
User.configuration_write(auth_configuration, args.secure)
if not args.no_reload:
client = Application(architecture, configuration, no_report=False, unsafe=unsafe).repository.reporter
client.reload_auth()
if args.action == Action.List:
for found_user in database.user_list(user.username, user.access):
UserPrinter(found_user).print(verbose=True)
elif args.action == Action.Remove:
database.user_remove(user.username)
elif args.action == Action.Update:
User.configuration_create(auth_configuration, user, salt, args.as_service, args.secure)
database.user_update(user.hash_password(salt))
@staticmethod
def configuration_create(configuration: Configuration, user: MUser, salt: str, as_service_user: bool) -> None:
def configuration_create(configuration: Configuration, user: MUser, salt: str,
as_service_user: bool, secure: bool) -> None:
"""
put new user to configuration
enable configuration if it has been disabled
:param configuration: configuration instance
:param user: user descriptor
:param salt: password hash salt
:param as_service_user: add user as service user, also set password and user to configuration
:param secure: if true then set file permissions to 0o600
"""
section = Configuration.section_name("auth", user.access.value)
configuration.set_option("auth", "salt", salt)
configuration.set_option(section, user.username, user.hash_password(salt))
if as_service_user:
configuration.set_option("web", "username", user.username)
configuration.set_option("web", "password", user.password)
User.configuration_write(configuration, secure)
@staticmethod
def configuration_get(include_path: Path) -> Configuration:
@ -90,6 +92,8 @@ class User(Handler):
configuration = Configuration()
configuration.load(target)
configuration.architecture = "" # not user anyway
return configuration
@staticmethod
@ -99,12 +103,11 @@ class User(Handler):
:param configuration: configuration instance
:param secure: if true then set file permissions to 0o600
"""
if configuration.path is None:
return # should never happen actually
with configuration.path.open("w") as ahriman_configuration:
path, _ = configuration.check_loaded()
with path.open("w") as ahriman_configuration:
configuration.write(ahriman_configuration)
if secure:
configuration.path.chmod(0o600)
path.chmod(0o600)
@staticmethod
def get_salt(configuration: Configuration, salt_length: int = 20) -> str:
@ -118,19 +121,6 @@ class User(Handler):
return salt
return MUser.generate_password(salt_length)
@staticmethod
def user_clear(configuration: Configuration, user: MUser) -> None:
"""
remove user user from configuration file in case if it exists
:param configuration: configuration instance
:param user: user descriptor
"""
for role in UserAccess:
section = Configuration.section_name("auth", role.value)
if not configuration.has_option(section, user.username):
continue
configuration.remove_option(section, user.username)
@staticmethod
def user_create(args: argparse.Namespace) -> MUser:
"""

3
src/ahriman/application/lock.py
View File

@ -32,7 +32,6 @@ from ahriman.core.exceptions import DuplicateRun
from ahriman.core.status.client import Client
from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.repository_paths import RepositoryPaths
class Lock:
@ -56,7 +55,7 @@ class Lock:
self.force = args.force
self.unsafe = args.unsafe
self.paths = RepositoryPaths(configuration.getpath("repository", "root"), architecture)
self.paths = configuration.repository_paths
self.reporter = Client() if args.no_report else Client.load(configuration)
def __enter__(self) -> Lock:

31
src/ahriman/core/auth/auth.py
View File

@ -21,12 +21,11 @@ from __future__ import annotations
import logging
from typing import Dict, Optional, Type
from typing import Optional, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import DuplicateUser
from ahriman.core.database.sqlite import SQLite
from ahriman.models.auth_settings import AuthSettings
from ahriman.models.user import User
from ahriman.models.user_access import UserAccess
@ -63,40 +62,22 @@ class Auth:
return """<button type="button" class="btn btn-link" data-bs-toggle="modal" data-bs-target="#loginForm" style="text-decoration: none">login</button>"""
@classmethod
def load(cls: Type[Auth], configuration: Configuration) -> Auth:
def load(cls: Type[Auth], configuration: Configuration, database: SQLite) -> Auth:
"""
load authorization module from settings
:param configuration: configuration instance
:param database: database instance
:return: authorization module according to current settings
"""
provider = AuthSettings.from_option(configuration.get("auth", "target", fallback="disabled"))
if provider == AuthSettings.Configuration:
from ahriman.core.auth.mapping import Mapping
return Mapping(configuration)
return Mapping(configuration, database)
if provider == AuthSettings.OAuth:
from ahriman.core.auth.oauth import OAuth
return OAuth(configuration)
return OAuth(configuration, database)
return cls(configuration)
@staticmethod
def get_users(configuration: Configuration) -> Dict[str, User]:
"""
load users from settings
:param configuration: configuration instance
:return: map of username to its descriptor
"""
users: Dict[str, User] = {}
for role in UserAccess:
section = configuration.section_name("auth", role.value)
if not configuration.has_section(section):
continue
for user, password in configuration[section].items():
normalized_user = user.lower()
if normalized_user in users:
raise DuplicateUser(normalized_user)
users[normalized_user] = User(normalized_user, password, role)
return users
async def check_credentials(self, username: Optional[str], password: Optional[str]) -> bool: # pylint: disable=no-self-use
"""
validate user password

13
src/ahriman/core/auth/mapping.py
View File

@ -20,7 +20,9 @@
from typing import Optional
from ahriman.core.auth.auth import Auth
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.models.auth_settings import AuthSettings
from ahriman.models.user import User
from ahriman.models.user_access import UserAccess
@ -30,18 +32,20 @@ class Mapping(Auth):
"""
user authorization based on mapping from configuration file
:ivar salt: random generated string to salt passwords
:ivar _users: map of username to its descriptor
:ivar database: database instance
"""
def __init__(self, configuration: Configuration, provider: AuthSettings = AuthSettings.Configuration) -> None:
def __init__(self, configuration: Configuration, database: SQLite,
provider: AuthSettings = AuthSettings.Configuration) -> None:
"""
default constructor
:param configuration: configuration instance
:param database: database instance
:param provider: authorization type definition
"""
Auth.__init__(self, configuration, provider)
self.database = database
self.salt = configuration.get("auth", "salt")
self._users = self.get_users(configuration)
async def check_credentials(self, username: Optional[str], password: Optional[str]) -> bool:
"""
@ -61,8 +65,7 @@ class Mapping(Auth):
:param username: username
:return: user descriptor if username is known and None otherwise
"""
normalized_user = username.lower()
return self._users.get(normalized_user)
return self.database.user_get(username)
async def known_username(self, username: Optional[str]) -> bool:
"""

7
src/ahriman/core/auth/oauth.py
View File

@ -23,6 +23,7 @@ from typing import Optional, Type
from ahriman.core.auth.mapping import Mapping
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.exceptions import InvalidOption
from ahriman.models.auth_settings import AuthSettings
@ -38,13 +39,15 @@ class OAuth(Mapping):
:ivar scopes: list of scopes required by the application
"""
def __init__(self, configuration: Configuration, provider: AuthSettings = AuthSettings.OAuth) -> None:
def __init__(self, configuration: Configuration, database: SQLite,
provider: AuthSettings = AuthSettings.OAuth) -> None:
"""
default constructor
:param configuration: configuration instance
:param database: database instance
:param provider: authorization type definition
"""
Mapping.__init__(self, configuration, provider)
Mapping.__init__(self, configuration, database, provider)
self.client_id = configuration.get("auth", "client_id")
self.client_secret = configuration.get("auth", "client_secret")
# in order to use OAuth feature the service must be publicity available

43
src/ahriman/core/build_tools/sources.py
View File

@ -47,6 +47,8 @@ class Sources:
found_files: List[Path] = []
for glob in pattern:
found_files.extend(sources_dir.glob(glob))
if not found_files:
return # no additional files found
Sources.logger.info("found matching files %s", found_files)
# add them to index
Sources._check_output("git", "add", "--intent-to-add",
@ -54,14 +56,13 @@ class Sources:
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def diff(sources_dir: Path, patch_path: Path) -> None:
def diff(sources_dir: Path) -> str:
"""
generate diff from the current version and write it to the output file
:param sources_dir: local path to git repository
:param patch_path: path to result patch
:return: patch as plain string
"""
patch = Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
patch_path.write_text(patch)
return Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def fetch(sources_dir: Path, remote: Optional[str]) -> None:
@ -112,41 +113,39 @@ class Sources:
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def load(sources_dir: Path, remote: str, patch_dir: Path) -> None:
def load(sources_dir: Path, remote: str, patch: Optional[str]) -> None:
"""
fetch sources from remote and apply patches
:param sources_dir: local path to fetch
:param remote: remote target (from where to fetch)
:param patch_dir: path to directory with package patches
:param patch: optional patch to be applied
"""
Sources.fetch(sources_dir, remote)
Sources.patch_apply(sources_dir, patch_dir)
if patch is None:
Sources.logger.info("no patches found")
return
Sources.patch_apply(sources_dir, patch)
@staticmethod
def patch_apply(sources_dir: Path, patch_dir: Path) -> None:
def patch_apply(sources_dir: Path, patch: str) -> None:
"""
apply patches if any
:param sources_dir: local path to directory with git sources
:param patch_dir: path to directory with package patches
:param patch: patch to be applied
"""
# check if even there are patches
if not patch_dir.is_dir():
return # no patches provided
# find everything that looks like patch and sort it
patches = sorted(patch_dir.glob("*.patch"))
Sources.logger.info("found %s patches", patches)
for patch in patches:
Sources.logger.info("apply patch %s", patch.name)
Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace", str(patch),
exception=None, cwd=sources_dir, logger=Sources.logger)
# create patch
Sources.logger.info("apply patch from database")
Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace",
exception=None, cwd=sources_dir, input_data=patch, logger=Sources.logger)
@staticmethod
def patch_create(sources_dir: Path, patch_path: Path, *pattern: str) -> None:
def patch_create(sources_dir: Path, *pattern: str) -> str:
"""
create patch set for the specified local path
:param sources_dir: local path to git repository
:param patch_path: path to result patch
:param pattern: glob patterns
:return: patch as plain text
"""
Sources.add(sources_dir, *pattern)
Sources.diff(sources_dir, patch_path)
diff = Sources.diff(sources_dir)
return f"{diff}\n" # otherwise, patch will be broken

20
src/ahriman/core/build_tools/task.py
View File

@ -21,10 +21,11 @@ import logging
import shutil
from pathlib import Path
from typing import List, Optional
from typing import List
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.exceptions import BuildFailed
from ahriman.core.util import check_output
from ahriman.models.package import Package
@ -61,9 +62,10 @@ class Task:
self.makepkg_flags = configuration.getlist("build", "makepkg_flags", fallback=[])
self.makechrootpkg_flags = configuration.getlist("build", "makechrootpkg_flags", fallback=[])
def build(self) -> List[Path]:
def build(self, sources_path: Path) -> List[Path]:
"""
run package build
:param sources_path: path to where sources are
:return: paths of produced packages
"""
command = [self.build_command, "-r", str(self.paths.chroot)]
@ -75,24 +77,24 @@ class Task:
Task._check_output(
*command,
exception=BuildFailed(self.package.base),
cwd=self.paths.sources_for(self.package.base),
cwd=sources_path,
logger=self.build_logger,
user=self.uid)
# well it is not actually correct, but we can deal with it
packages = Task._check_output("makepkg", "--packagelist",
exception=BuildFailed(self.package.base),
cwd=self.paths.sources_for(self.package.base),
cwd=sources_path,
logger=self.build_logger).splitlines()
return [Path(package) for package in packages]
def init(self, path: Optional[Path] = None) -> None:
def init(self, path: Path, database: SQLite) -> None:
"""
fetch package from git
:param path: optional local path to fetch. If not set default path will be used
:param path: local path to fetch
:param database: database instance
"""
git_path = path or self.paths.sources_for(self.package.base)
if self.paths.cache_for(self.package.base).is_dir():
# no need to clone whole repository, just copy from cache first
shutil.copytree(self.paths.cache_for(self.package.base), git_path)
Sources.load(git_path, self.package.git_url, self.paths.patches_for(self.package.base))
shutil.copytree(self.paths.cache_for(self.package.base), path, dirs_exist_ok=True)
Sources.load(path, self.package.git_url, database.patches_get(self.package.base))

25
src/ahriman/core/configuration.py
View File

@ -28,6 +28,7 @@ from pathlib import Path
from typing import Any, Dict, Generator, List, Optional, Tuple, Type
from ahriman.core.exceptions import InitializeException
from ahriman.models.repository_paths import RepositoryPaths
class Configuration(configparser.RawConfigParser):
@ -72,6 +73,14 @@ class Configuration(configparser.RawConfigParser):
"""
return self.getpath("settings", "logging")
@property
def repository_paths(self) -> RepositoryPaths:
"""
:return: repository paths instance
"""
_, architecture = self.check_loaded()
return RepositoryPaths(self.getpath("repository", "root"), architecture)
@classmethod
def from_path(cls: Type[Configuration], path: Path, architecture: str, quiet: bool) -> Configuration:
"""
@ -134,6 +143,15 @@ class Configuration(configparser.RawConfigParser):
return path
return self.path.parent / path
def check_loaded(self) -> Tuple[Path, str]:
"""
check if service was actually loaded
:return: configuration root path and architecture if loaded
"""
if self.path is None or self.architecture is None:
raise InitializeException("Configuration path and/or architecture are not set")
return self.path, self.architecture
def dump(self) -> Dict[str, Dict[str, str]]:
"""
dump configuration to dictionary
@ -233,12 +251,11 @@ class Configuration(configparser.RawConfigParser):
"""
reload configuration if possible or raise exception otherwise
"""
if self.path is None or self.architecture is None:
raise InitializeException("Configuration path and/or architecture are not set")
path, architecture = self.check_loaded()
for section in self.sections(): # clear current content
self.remove_section(section)
self.load(self.path)
self.merge_sections(self.architecture)
self.load(path)
self.merge_sections(architecture)
def set_option(self, section: str, option: str, value: Optional[str]) -> None:
"""

19
src/ahriman/core/database/__init__.py Normal file
View File

@ -0,0 +1,19 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

43
src/ahriman/core/database/data/__init__.py Normal file
View File

@ -0,0 +1,43 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
from ahriman.core.database.data.patches import migrate_patches
from ahriman.core.database.data.users import migrate_users_data
from ahriman.core.database.data.package_statuses import migrate_package_statuses
from ahriman.models.migration_result import MigrationResult
from ahriman.models.repository_paths import RepositoryPaths
def migrate_data(result: MigrationResult, connection: Connection,
configuration: Configuration, paths: RepositoryPaths) -> None:
"""
perform data migration
:param result: result of the schema migration
:param connection: database connection
:param configuration: configuration instance
:param paths: repository paths instance
"""
# initial data migration
if result.old_version == 0:
migrate_package_statuses(connection, paths)
migrate_users_data(connection, configuration)
migrate_patches(connection, paths)

80
src/ahriman/core/database/data/package_statuses.py Normal file
View File

@ -0,0 +1,80 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import json
from sqlite3 import Connection
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for package statuses
:param connection: database connection
:param paths: repository paths instance
"""
def insert_base(metadata: Package, last_status: BuildStatus) -> None:
connection.execute(
"""
insert into package_bases
(package_base, version, aur_url)
values
(:package_base, :version, :aur_url)
""",
dict(package_base=metadata.base, version=metadata.version, aur_url=metadata.aur_url))
connection.execute(
"""
insert into package_statuses
(package_base, status, last_updated)
values
(:package_base, :status, :last_updated)""",
dict(package_base=metadata.base, status=last_status.status.value, last_updated=last_status.timestamp))
def insert_packages(metadata: Package) -> None:
package_list = []
for name, description in metadata.packages.items():
package_list.append(dict(package=name, package_base=metadata.base, **description.view()))
connection.executemany(
"""
insert into packages
(package, package_base, architecture, archive_size, build_date, depends, description,
filename, "groups", installed_size, licenses, provides, url)
values
(:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description,
:filename, :groups, :installed_size, :licenses, :provides, :url)
""",
package_list)
cache_path = paths.root / "status_cache.json"
if not cache_path.is_file():
return # no file found
with cache_path.open() as cache:
dump = json.load(cache)
for item in dump.get("packages", []):
package = Package.from_json(item["package"])
status = BuildStatus.from_json(item["status"])
insert_base(package, status)
insert_packages(package)
connection.commit()
cache_path.unlink()

44
src/ahriman/core/database/data/patches.py Normal file
View File

@ -0,0 +1,44 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.models.repository_paths import RepositoryPaths
def migrate_patches(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for patches
:param connection: database connection
:param paths: repository paths instance
"""
root = paths.root / "patches"
if not root.is_dir():
return # no directory found
for package in root.iterdir():
patch_path = package / "00-main.patch"
if not patch_path.is_file():
continue # not exist
content = patch_path.read_text(encoding="utf8")
connection.execute(
"""insert into patches (package_base, patch) values (:package_base, :patch)""",
{"package_base": package.name, "patch": content})
connection.commit()

40
src/ahriman/core/database/data/users.py Normal file
View File

@ -0,0 +1,40 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
def migrate_users_data(connection: Connection, configuration: Configuration) -> None:
"""
perform migration for users
:param connection: database connection
:param configuration: configuration instance
"""
for section in configuration.sections():
for option, value in configuration[section].items():
if not section.startswith("auth:"):
continue
permission = section[5:]
connection.execute(
"""insert into users (username, permission, password) values (:username, :permission, :password)""",
{"username": option.lower(), "permission": permission, "password": value})
connection.commit()

125
src/ahriman/core/database/migrations/__init__.py Normal file
View File

@ -0,0 +1,125 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import logging
from importlib import import_module
from pathlib import Path
from pkgutil import iter_modules
from sqlite3 import Connection
from typing import List, Type
from ahriman.models.migration import Migration
from ahriman.models.migration_result import MigrationResult
class Migrations:
"""
simple migration wrapper for the sqlite
idea comes from https://www.ash.dev/blog/simple-migration-system-in-sqlite/
:ivar connection: database connection
:ivar logger: class logger
"""
def __init__(self, connection: Connection) -> None:
"""
default constructor
:param connection: database connection
"""
self.connection = connection
self.logger = logging.getLogger("database")
@classmethod
def migrate(cls: Type[Migrations], connection: Connection) -> MigrationResult:
"""
perform migrations implicitly
:param connection: database connection
:return: current schema version
"""
return cls(connection).run()
def migrations(self) -> List[Migration]:
"""
extract all migrations from the current package
idea comes from https://julienharbulot.com/python-dynamical-import.html
"""
migrations: List[Migration] = []
package_dir = Path(__file__).resolve().parent
modules = [module_name for (_, module_name, _) in iter_modules([str(package_dir)])]
for index, module_name in enumerate(sorted(modules)):
module = import_module(f"{__name__}.{module_name}")
steps: List[str] = getattr(module, "steps", [])
self.logger.debug("found migration %s at index %s with steps count %s", module_name, index, len(steps))
migrations.append(Migration(index, module_name, steps))
return migrations
def run(self) -> MigrationResult:
"""
perform migrations
:return: current schema version
"""
migrations = self.migrations()
current_version = self.user_version()
expected_version = len(migrations)
result = MigrationResult(current_version, expected_version)
if not result.is_outdated:
self.logger.info("no migrations required")
return result
previous_isolation = self.connection.isolation_level
try:
self.connection.isolation_level = None
cursor = self.connection.cursor()
try:
cursor.execute("begin exclusive")
for migration in migrations[current_version:]:
self.logger.info("applying migration %s at index %s", migration.name, migration.index)
for statement in migration.steps:
cursor.execute(statement)
self.logger.info("migration %s at index %s has been applied", migration.name, migration.index)
cursor.execute(f"pragma user_version = {expected_version}") # no support for ? placeholders
except Exception:
self.logger.exception("migration failed with exception")
cursor.execute("rollback")
raise
else:
cursor.execute("commit")
finally:
cursor.close()
finally:
self.connection.isolation_level = previous_isolation
self.logger.info("migrations have been performed from version %s to %s", result.old_version, result.new_version)
return result
def user_version(self) -> int:
"""
get schema version from sqlite database
;return: current schema version
"""
cursor = self.connection.execute("pragma user_version")
current_version: int = cursor.fetchone()["user_version"]
return current_version

73
src/ahriman/core/database/migrations/m000_initial.py Normal file
View File

@ -0,0 +1,73 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
steps = [
"""
create table build_queue (
package_base text not null unique,
properties json not null
)
""",
"""
create table package_bases (
package_base text not null unique,
version text not null,
aur_url text not null
)
""",
"""
create table package_statuses (
package_base text not null unique,
status text not null,
last_updated integer
)
""",
"""
create table packages (
package text not null,
package_base text not null,
architecture text,
archive_size integer,
build_date integer,
depends json,
description text,
filename text,
"groups" json,
installed_size integer,
licenses json,
provides json,
url text,
unique (package, architecture)
)
""",
"""
create table patches (
package_base text not null unique,
patch blob not null
)
""",
"""
create table users (
username text not null unique,
access text not null,
password text
)
""",
]

19
src/ahriman/core/database/operations/__init__.py Normal file
View File

@ -0,0 +1,19 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

93
src/ahriman/core/database/operations/auth_operations.py Normal file
View File

@ -0,0 +1,93 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
from sqlite3 import Connection
from typing import List, Optional
from ahriman.core.database.operations.operations import Operations
from ahriman.models.user import User
from ahriman.models.user_access import UserAccess
class AuthOperations(Operations):
"""
authorization operations
"""
def user_get(self, username: str) -> Optional[User]:
"""
get user by username
:param username: username
:return: user if it was found
"""
return next(iter(self.user_list(username, None)), None)
def user_list(self, username: Optional[str], access: Optional[UserAccess]) -> List[User]:
"""
get users by filter
:param username: optional filter by username
:param access: optional filter by role
:return: list of users who match criteria
"""
username_filter = username.lower() if username is not None else username
access_filter = access.value if access is not None else access
def run(connection: Connection) -> List[User]:
return [
User(cursor["username"], cursor["password"], UserAccess(cursor["access"]))
for cursor in connection.execute(
"""
select * from users
where (:username is null or username = :username) and (:access is null or access = :access)
""",
{"username": username_filter, "access": access_filter})
]
return self.with_connection(run)
def user_remove(self, username: str) -> None:
"""
remove user from storage
:param username: username
"""
def run(connection: Connection) -> None:
connection.execute("""delete from users where username = :username""", {"username": username.lower()})
return self.with_connection(run, commit=True)
def user_update(self, user: User) -> None:
"""
get user by username
:param user: user descriptor
"""
def run(connection: Connection) -> None:
connection.execute(
"""
insert into users
(username, access, password)
values
(:username, :access, :password)
on conflict (username) do update set
access = :access, password = :password
""",
{"username": user.username.lower(), "access": user.access.value, "password": user.password})
self.with_connection(run, commit=True)

77
src/ahriman/core/database/operations/build_operations.py Normal file
View File

@ -0,0 +1,77 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from typing import List, Optional
from ahriman.core.database.operations.operations import Operations
from ahriman.models.package import Package
class BuildOperations(Operations):
"""
operations for main functions
"""
def build_queue_clear(self, package_base: Optional[str]) -> None:
"""
remove packages from build queue
:param package_base: optional filter by package base
"""
def run(connection: Connection) -> None:
connection.execute(
"""
delete from build_queue
where :package_base is null or package_base = :package_base
""",
{"package_base": package_base})
return self.with_connection(run, commit=True)
def build_queue_get(self) -> List[Package]:
"""
retrieve packages from build queue
:return: list of packages to be built
"""
def run(connection: Connection) -> List[Package]:
return [
Package.from_json(row["properties"])
for row in connection.execute("""select * from build_queue""")
]
return self.with_connection(run)
def build_queue_insert(self, package: Package) -> None:
"""
insert packages to build queue
:param package: package to be inserted
"""
def run(connection: Connection) -> None:
connection.execute(
"""
insert into build_queue
(package_base, properties)
values
(:package_base, :properties)
on conflict (package_base) do update set
properties = :properties
""",
{"package_base": package.base, "properties": package.view()})
return self.with_connection(run, commit=True)

71
src/ahriman/core/database/operations/operations.py Normal file
View File

@ -0,0 +1,71 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import sqlite3
from pathlib import Path
from sqlite3 import Connection, Cursor
from typing import Any, Dict, Tuple, TypeVar, Callable
T = TypeVar("T")
class Operations:
"""
base operation class
:ivar logger: class logger
:ivar path: path to the database file
"""
def __init__(self, path: Path) -> None:
"""
default constructor
:param path: path to the database file
"""
self.path = path
self.logger = logging.getLogger("database")
@staticmethod
def factory(cursor: Cursor, row: Tuple[Any, ...]) -> Dict[str, Any]:
"""
dictionary factory based on official documentation
:param cursor: cursor descriptor
:param row: fetched row
:return: row converted to dictionary
"""
result = {}
for index, column in enumerate(cursor.description):
result[column[0]] = row[index]
return result
def with_connection(self, query: Callable[[Connection], T], commit: bool = False) -> T:
"""
perform operation in connection
:param query: function to be called with connection
:param commit: if True commit() will be called on success
:return: result of the `query` call
"""
with sqlite3.connect(self.path, detect_types=sqlite3.PARSE_DECLTYPES) as connection:
connection.row_factory = self.factory
result = query(connection)
if commit:
connection.commit()
return result

199
src/ahriman/core/database/operations/package_operations.py Normal file
View File

@ -0,0 +1,199 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from typing import Dict, Generator, Iterable, List, Tuple
from ahriman.core.database.operations.operations import Operations
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
class PackageOperations(Operations):
"""
package operations
"""
@staticmethod
def _package_remove_package_base(connection: Connection, package_base: str) -> None:
"""
remove package base information
:param connection: database connection
:param package_base: package base name
"""
connection.execute("""delete from package_statuses where package_base = :package_base""",
{"package_base": package_base})
connection.execute("""delete from package_bases where package_base = :package_base""",
{"package_base": package_base})
@staticmethod
def _package_remove_packages(connection: Connection, package_base: str, current_packages: Iterable[str]) -> None:
"""
remove packages belong to the package base
:param connection: database connection
:param package_base: package base name
:param current_packages: current packages list which has to be left in database
"""
packages = [
package
for package in connection.execute(
"""select package from packages where package_base = :package_base""", {"package_base": package_base})
if package["package"] not in current_packages
]
connection.executemany("""delete from packages where package = :package""", packages)
@staticmethod
def _package_update_insert_base(connection: Connection, package: Package) -> None:
"""
insert base package into table
:param connection: database connection
:param package: package properties
"""
connection.execute(
"""
insert into package_bases
(package_base, version, aur_url)
values
(:package_base, :version, :aur_url)
on conflict (package_base) do update set
version = :version, aur_url = :aur_url
""",
dict(package_base=package.base, version=package.version, aur_url=package.aur_url))
@staticmethod
def _package_update_insert_packages(connection: Connection, package: Package) -> None:
"""
insert packages into table
:param connection: database connection
:param package: package properties
"""
package_list = []
for name, description in package.packages.items():
package_list.append(dict(package=name, package_base=package.base, **description.view()))
connection.executemany(
"""
insert into packages
(package, package_base, architecture, archive_size,
build_date, depends, description, filename,
"groups", installed_size, licenses, provides, url)
values
(:package, :package_base, :architecture, :archive_size,
:build_date, :depends, :description, :filename,
:groups, :installed_size, :licenses, :provides, :url)
on conflict (package, architecture) do update set
package_base = :package_base, archive_size = :archive_size,
build_date = :build_date, depends = :depends, description = :description, filename = :filename,
"groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides, url = :url
""",
package_list)
@staticmethod
def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus) -> None:
"""
insert base package status into table
:param connection: database connection
:param package_base: package base name
:param status: new build status
"""
connection.execute(
"""
insert into package_statuses (package_base, status, last_updated)
values
(:package_base, :status, :last_updated)
on conflict (package_base) do update set
status = :status, last_updated = :last_updated
""",
dict(package_base=package_base, status=status.status.value, last_updated=status.timestamp))
@staticmethod
def _packages_get_select_package_bases(connection: Connection) -> Dict[str, Package]:
"""
select package bases from the table
:param connection: database connection
:return: map of the package base to its descriptor (without packages themselves)
"""
return {
row["package_base"]: Package(row["package_base"], row["version"], row["aur_url"], {})
for row in connection.execute("""select * from package_bases""")
}
@staticmethod
def _packages_get_select_packages(connection: Connection, packages: Dict[str, Package]) -> Dict[str, Package]:
"""
select packages from the table
:param connection: database connection
:param packages: packages descriptor map
:return: map of the package base to its descriptor including individual packages
"""
for row in connection.execute("""select * from packages"""):
if row["package_base"] not in packages:
continue # normally must never happen though
packages[row["package_base"]].packages[row["package"]] = PackageDescription.from_json(row)
return packages
@staticmethod
def _packages_get_select_statuses(connection: Connection) -> Dict[str, BuildStatus]:
"""
select package build statuses from the table
:param connection: database connection
:return: map of the package base to its status
"""
return {
row["package_base"]: BuildStatus.from_json({"status": row["status"], "timestamp": row["last_updated"]})
for row in connection.execute("""select * from package_statuses""")
}
def package_remove(self, package_base: str) -> None:
"""
remove package from database
:param package_base: package base name
"""
def run(connection: Connection) -> None:
self._package_remove_packages(connection, package_base, [])
self._package_remove_package_base(connection, package_base)
return self.with_connection(run, commit=True)
def package_update(self, package: Package, status: BuildStatus) -> None:
"""
update package status
:param package: package properties
:param status: new build status
"""
def run(connection: Connection) -> None:
self._package_update_insert_base(connection, package)
self._package_update_insert_status(connection, package.base, status)
self._package_update_insert_packages(connection, package)
self._package_remove_packages(connection, package.base, package.packages.keys())
return self.with_connection(run, commit=True)
def packages_get(self) -> List[Tuple[Package, BuildStatus]]:
"""
get package list and their build statuses from database
:return: list of package properties and their statuses
"""
def run(connection: Connection) -> Generator[Tuple[Package, BuildStatus], None, None]:
packages = self._packages_get_select_package_bases(connection)
statuses = self._packages_get_select_statuses(connection)
for package_base, package in self._packages_get_select_packages(connection, packages).items():
yield package, statuses.get(package_base, BuildStatus())
return self.with_connection(lambda connection: list(run(connection)))

85
src/ahriman/core/database/operations/patch_operations.py Normal file
View File

@ -0,0 +1,85 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from typing import Dict, Optional
from ahriman.core.database.operations.operations import Operations
class PatchOperations(Operations):
"""
operations for patches
"""
def patches_get(self, package_base: str) -> Optional[str]:
"""
retrieve patches for the package
:param package_base: package base to search for patches
:return: plain text patch for the package
"""
return self.patches_list(package_base).get(package_base)
def patches_insert(self, package_base: str, patch: str) -> None:
"""
insert or update patch in database
:param package_base: package base to insert
:param patch: patch content
"""
def run(connection: Connection) -> None:
connection.execute(
"""
insert into patches
(package_base, patch)
values
(:package_base, :patch)
on conflict (package_base) do update set
patch = :patch
""",
{"package_base": package_base, "patch": patch})
return self.with_connection(run, commit=True)
def patches_list(self, package_base: Optional[str]) -> Dict[str, str]:
"""
extract all patches
:param package_base: optional filter by package base
:return: map of package base to patch content
"""
def run(connection: Connection) -> Dict[str, str]:
return {
cursor["package_base"]: cursor["patch"]
for cursor in connection.execute(
"""select * from patches where :package_base is null or package_base = :package_base""",
{"package_base": package_base})
}
return self.with_connection(run)
def patches_remove(self, package_base: str) -> None:
"""
remove patch set
:param package_base: package base to clear patches
"""
def run(connection: Connection) -> None:
connection.execute(
"""delete from patches where package_base = :package_base""",
{"package_base": package_base})
return self.with_connection(run, commit=True)

70
src/ahriman/core/database/sqlite.py Normal file
View File

@ -0,0 +1,70 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import json
import sqlite3
from sqlite3 import Connection
from typing import Type
from ahriman.core.configuration import Configuration
from ahriman.core.database.data import migrate_data
from ahriman.core.database.migrations import Migrations
from ahriman.core.database.operations.auth_operations import AuthOperations
from ahriman.core.database.operations.build_operations import BuildOperations
from ahriman.core.database.operations.package_operations import PackageOperations
from ahriman.core.database.operations.patch_operations import PatchOperations
class SQLite(AuthOperations, BuildOperations, PackageOperations, PatchOperations):
"""
wrapper for sqlite3 database
"""
@classmethod
def load(cls: Type[SQLite], configuration: Configuration) -> SQLite:
"""
construct instance from configuration
:param configuration: configuration instance
:return: fully initialized instance of the database
"""
database = cls(configuration.getpath("settings", "database"))
database.init(configuration)
return database
def init(self, configuration: Configuration) -> None:
"""
perform database migrations
:param configuration: configuration instance
"""
# custom types support
sqlite3.register_adapter(dict, json.dumps)
sqlite3.register_adapter(list, json.dumps)
sqlite3.register_converter("json", json.loads)
paths = configuration.repository_paths
def run(connection: Connection) -> None:
result = Migrations.migrate(connection)
migrate_data(result, connection, configuration, paths)
self.with_connection(run)
paths.chown(self.path)

30
src/ahriman/core/exceptions.py
View File

@ -26,12 +26,12 @@ class BuildFailed(RuntimeError):
base exception for failed builds
"""
def __init__(self, package: str) -> None:
def __init__(self, package_base: str) -> None:
"""
default constructor
:param package: package base raised exception
:param package_base: package base raised exception
"""
RuntimeError.__init__(self, f"Package {package} build failed, check logs for details")
RuntimeError.__init__(self, f"Package {package_base} build failed, check logs for details")
class DuplicateRun(RuntimeError):
@ -47,18 +47,11 @@ class DuplicateRun(RuntimeError):
self, "Another application instance is run. This error can be suppressed by using --force flag.")
class DuplicateUser(ValueError):
class ExitCode(RuntimeError):
"""
exception which will be thrown in case if there are two users with different settings
special exception which has to be thrown to return non-zero status without error message
"""
def __init__(self, username: str) -> None:
"""
default constructor
:param username: username with duplicates
"""
ValueError.__init__(self, f"Found duplicate user with username {username}")
class InitializeException(RuntimeError):
"""
@ -113,6 +106,19 @@ class InvalidPackageInfo(RuntimeError):
RuntimeError.__init__(self, f"There are errors during reading package information: `{details}`")
class MigrationError(RuntimeError):
"""
exception which will be raised on migration error
"""
def __init__(self, details: str) -> None:
"""
default constructor
:param details: error details
"""
RuntimeError.__init__(self, details)
class MissingArchitecture(ValueError):
"""
exception which will be raised if architecture is required, but missing

46
src/ahriman/core/formatters/user_printer.py Normal file
View File

@ -0,0 +1,46 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import List
from ahriman.core.formatters.string_printer import StringPrinter
from ahriman.models.property import Property
from ahriman.models.user import User
class UserPrinter(StringPrinter):
"""
print properties of user
:ivar user: stored user
"""
def __init__(self, user: User) -> None:
"""
default constructor
:param user: user to print
"""
StringPrinter.__init__(self, user.username)
self.user = user
def properties(self) -> List[Property]:
"""
convert content into printable data
:return: list of content properties
"""
return [Property("role", self.user.access.value, is_required=True)]

25
src/ahriman/core/repository/cleaner.py
View File

@ -37,14 +37,6 @@ class Cleaner(Properties):
"""
raise NotImplementedError
def clear_build(self) -> None:
"""
clear sources directory
"""
self.logger.info("clear package sources directory")
for package in self.paths.sources.iterdir():
shutil.rmtree(package)
def clear_cache(self) -> None:
"""
clear cache directory
@ -61,14 +53,6 @@ class Cleaner(Properties):
for chroot in self.paths.chroot.iterdir():
shutil.rmtree(chroot)
def clear_manual(self) -> None:
"""
clear directory with manual package updates
"""
self.logger.info("clear manual packages")
for package in self.paths.manual.iterdir():
shutil.rmtree(package)
def clear_packages(self) -> None:
"""
clear directory with built packages (NOT repository itself)
@ -77,10 +61,9 @@ class Cleaner(Properties):
for package in self.packages_built():
package.unlink()
def clear_patches(self) -> None:
def clear_queue(self) -> None:
"""
clear directory with patches
clear packages which were queued for the update
"""
self.logger.info("clear patches directory")
for package in self.paths.patches.iterdir():
shutil.rmtree(package)
self.logger.info("clear build queue")
self.database.build_queue_clear(None)

25
src/ahriman/core/repository/executor.py
View File

@ -26,6 +26,7 @@ from ahriman.core.build_tools.task import Task
from ahriman.core.report.report import Report
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.upload.upload import Upload
from ahriman.core.util import tmpdir
from ahriman.models.package import Package
from ahriman.models.result import Result
@ -56,25 +57,25 @@ class Executor(Cleaner):
:param updates: list of packages properties to build
:return: `packages_built`
"""
def build_single(package: Package) -> None:
def build_single(package: Package, local_path: Path) -> None:
self.reporter.set_building(package.base)
task = Task(package, self.configuration, self.paths)
task.init()
built = task.build()
task.init(local_path, self.database)
built = task.build(local_path)
for src in built:
dst = self.paths.packages / src.name
shutil.move(src, dst)
result = Result()
for single in updates:
try:
build_single(single)
result.add_success(single)
except Exception:
self.reporter.set_failed(single.base)
result.add_failed(single)
self.logger.exception("%s (%s) build exception", single.base, self.architecture)
self.clear_build()
with tmpdir() as build_dir:
try:
build_single(single, build_dir)
result.add_success(single)
except Exception:
self.reporter.set_failed(single.base)
result.add_failed(single)
self.logger.exception("%s (%s) build exception", single.base, self.architecture)
return result
@ -87,6 +88,8 @@ class Executor(Cleaner):
def remove_base(package_base: str) -> None:
try:
self.paths.tree_clear(package_base) # remove all internal files
self.database.build_queue_clear(package_base)
self.database.patches_remove(package_base)
self.reporter.remove(package_base) # we only update status page in case of base removal
except Exception:
self.logger.exception("could not remove base %s", package_base)

10
src/ahriman/core/repository/properties.py
View File

@ -22,11 +22,11 @@ import logging
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.repo import Repo
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.exceptions import UnsafeRun
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.client import Client
from ahriman.core.util import check_user
from ahriman.models.repository_paths import RepositoryPaths
class Properties:
@ -35,6 +35,7 @@ class Properties:
:ivar architecture: repository architecture
:ivar aur_url: base AUR url
:ivar configuration: configuration instance
:ivar database: database instance
:ivar ignore_list: package bases which will be ignored during auto updates
:ivar logger: class logger
:ivar name: repository name
@ -45,22 +46,25 @@ class Properties:
:ivar sign: GPG wrapper instance
"""
def __init__(self, architecture: str, configuration: Configuration, no_report: bool, unsafe: bool) -> None:
def __init__(self, architecture: str, configuration: Configuration, database: SQLite,
no_report: bool, unsafe: bool) -> None:
"""
default constructor
:param architecture: repository architecture
:param configuration: configuration instance
:param database: database instance
:param no_report: force disable reporting
:param unsafe: if set no user check will be performed before path creation
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration
self.database = database
self.aur_url = configuration.get("alpm", "aur_url")
self.name = configuration.get("repository", "name")
self.paths = RepositoryPaths(configuration.getpath("repository", "root"), architecture)
self.paths = configuration.repository_paths
try:
check_user(self.paths, unsafe)
self.paths.tree_create()

13
src/ahriman/core/repository/update_handler.py
View File

@ -90,7 +90,7 @@ class UpdateHandler(Cleaner):
else:
self.reporter.set_success(local)
except Exception:
self.logger.exception("could not procees package at %s", dirname)
self.logger.exception("could not process package at %s", dirname)
return result
@ -102,16 +102,15 @@ class UpdateHandler(Cleaner):
result: List[Package] = []
known_bases = {package.base for package in self.packages()}
for dirname in self.paths.manual.iterdir():
try:
local = Package.load(str(dirname), PackageSource.Local, self.pacman, self.aur_url)
try:
for local in self.database.build_queue_get():
result.append(local)
if local.base not in known_bases:
self.reporter.set_unknown(local)
else:
self.reporter.set_pending(local.base)
except Exception:
self.logger.exception("could not add package from %s", dirname)
self.clear_manual()
except Exception:
self.logger.exception("could not load packages from database")
self.clear_queue()
return result

5
src/ahriman/core/status/client.py
View File

@ -77,11 +77,6 @@ class Client:
"""
return BuildStatus()
def reload_auth(self) -> None:
"""
reload authentication module call
"""
def remove(self, base: str) -> None:
"""
remove packages from watcher

86
src/ahriman/core/status/watcher.py
View File

@ -17,13 +17,12 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import json
import logging
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from typing import Dict, List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.exceptions import UnknownPackage
from ahriman.core.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
@ -34,33 +33,29 @@ class Watcher:
"""
package status watcher
:ivar architecture: repository architecture
:ivar database: database instance
:ivar known: list of known packages. For the most cases `packages` should be used instead
:ivar logger: class logger
:ivar repository: repository object
:ivar status: daemon status
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, architecture: str, configuration: Configuration, database: SQLite) -> None:
"""
default constructor
:param architecture: repository architecture
:param configuration: configuration instance
:param database: database instance
"""
self.logger = logging.getLogger("http")
self.architecture = architecture
self.repository = Repository(architecture, configuration, no_report=True, unsafe=False)
self.database = database
self.repository = Repository(architecture, configuration, database, no_report=True, unsafe=False)
self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
self.status = BuildStatus()
@property
def cache_path(self) -> Path:
"""
:return: path to dump with json cache
"""
return self.repository.paths.root / "status_cache.json"
@property
def packages(self) -> List[Tuple[Package, BuildStatus]]:
"""
@ -68,48 +63,6 @@ class Watcher:
"""
return list(self.known.values())
def _cache_load(self) -> None:
"""
update current state from cache
"""
def parse_single(properties: Dict[str, Any]) -> None:
package = Package.from_json(properties["package"])
status = BuildStatus.from_json(properties["status"])
if package.base in self.known:
self.known[package.base] = (package, status)
if not self.cache_path.is_file():
return
with self.cache_path.open() as cache:
try:
dump = json.load(cache)
except Exception:
self.logger.exception("cannot parse json from file")
dump = {}
for item in dump.get("packages", []):
try:
parse_single(item)
except Exception:
self.logger.exception("cannot parse item %s to package", item)
def _cache_save(self) -> None:
"""
dump current cache to filesystem
"""
dump = {
"packages": [
{
"package": package.view(),
"status": status.view()
} for package, status in self.packages
]
}
try:
with self.cache_path.open("w") as cache:
json.dump(dump, cache)
except Exception:
self.logger.exception("cannot dump cache")
def get(self, base: str) -> Tuple[Package, BuildStatus]:
"""
get current package base build status
@ -131,31 +84,34 @@ class Watcher:
else:
status = BuildStatus()
self.known[package.base] = (package, status)
self._cache_load()
def remove(self, base: str) -> None:
for package, status in self.database.packages_get():
if package.base in self.known:
self.known[package.base] = (package, status)
def remove(self, package_base: str) -> None:
"""
remove package base from known list if any
:param base: package base
:param package_base: package base
"""
self.known.pop(base, None)
self._cache_save()
self.known.pop(package_base, None)
self.database.package_remove(package_base)
def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
def update(self, package_base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
"""
update package status and description
:param base: package base to update
:param package_base: package base to update
:param status: new build status
:param package: optional new package description. In case if not set current properties will be used
"""
if package is None:
try:
package, _ = self.known[base]
package, _ = self.known[package_base]
except KeyError:
raise UnknownPackage(base)
raise UnknownPackage(package_base)
full_status = BuildStatus(status)
self.known[base] = (package, full_status)
self._cache_save()
self.known[package_base] = (package, full_status)
self.database.package_update(package, full_status)
def update_self(self, status: BuildStatusEnum) -> None:
"""

19
src/ahriman/core/status/web_client.py
View File

@ -67,13 +67,6 @@ class WebClient(Client):
"""
return f"{self.address}/user-api/v1/login"
@property
def _reload_auth_url(self) -> str:
"""
:return: full url for web service to reload authentication module
"""
return f"{self.address}/service-api/v1/reload-auth"
@property
def _status_url(self) -> str:
"""
@ -198,18 +191,6 @@ class WebClient(Client):
self.logger.exception("could not get service status")
return BuildStatus()
def reload_auth(self) -> None:
"""
reload authentication module call
"""
try:
response = self.__session.post(self._reload_auth_url)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not reload auth module: %s", exception_response_text(e))
except Exception:
self.logger.exception("could not reload auth module")
def remove(self, base: str) -> None:
"""
remove packages from watcher

14
src/ahriman/core/tree.py
View File

@ -26,8 +26,8 @@ from pathlib import Path
from typing import Iterable, List, Set, Type
from ahriman.core.build_tools.sources import Sources
from ahriman.core.database.sqlite import SQLite
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
class Leaf:
@ -54,16 +54,16 @@ class Leaf:
return self.package.packages.keys()
@classmethod
def load(cls: Type[Leaf], package: Package, paths: RepositoryPaths) -> Leaf:
def load(cls: Type[Leaf], package: Package, database: SQLite) -> Leaf:
"""
load leaf from package with dependencies
:param package: package properties
:param paths: repository paths instance
:param database: database instance
:return: loaded class
"""
clone_dir = Path(tempfile.mkdtemp())
try:
Sources.load(clone_dir, package.git_url, paths.patches_for(package.base))
Sources.load(clone_dir, package.git_url, database.patches_get(package.base))
dependencies = Package.dependencies(clone_dir)
finally:
shutil.rmtree(clone_dir, ignore_errors=True)
@ -95,14 +95,14 @@ class Tree:
self.leaves = leaves
@classmethod
def load(cls: Type[Tree], packages: Iterable[Package], paths: RepositoryPaths) -> Tree:
def load(cls: Type[Tree], packages: Iterable[Package], database: SQLite) -> Tree:
"""
load tree from packages
:param packages: packages list
:param paths: repository paths instance
:param database: database instance
:return: loaded class
"""
return cls([Leaf.load(package, paths) for package in packages])
return cls([Leaf.load(package, database) for package in packages])
def levels(self) -> List[List[Package]]:
"""

18
src/ahriman/core/util.py
View File

@ -19,9 +19,12 @@
#
import datetime
import os
import subprocess
import requests
import shutil
import subprocess
import tempfile
from contextlib import contextmanager
from logging import Logger
from pathlib import Path
from typing import Any, Dict, Generator, Iterable, Optional, Union
@ -143,6 +146,19 @@ def pretty_size(size: Optional[float], level: int = 0) -> str:
return pretty_size(size / 1024, level + 1)
@contextmanager
def tmpdir() -> Generator[Path, None, None]:
"""
wrapper for tempfile to remove directory after all
:return: path to the created directory
"""
path = Path(tempfile.mkdtemp())
try:
yield path
finally:
shutil.rmtree(path, ignore_errors=True)
def walk(directory_path: Path) -> Generator[Path, None, None]:
"""
list all file paths in given directory

35
src/ahriman/models/migration.py Normal file
View File

@ -0,0 +1,35 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from typing import List
@dataclass
class Migration:
"""
migration implementation
:ivar index: migration position
:ivar name: migration name
:ivar steps: migration steps
"""
index: int
name: str
steps: List[str]

50
src/ahriman/models/migration_result.py Normal file
View File

@ -0,0 +1,50 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from ahriman.core.exceptions import MigrationError
@dataclass
class MigrationResult:
"""
migration result implementation model
:ivar old_version: old schema version before migrations
:ivar new_version: new schema version after migrations
"""
old_version: int
new_version: int
@property
def is_outdated(self) -> bool:
"""
:return: True in case if it requires migrations and False otherwise
"""
self.validate()
return self.new_version > self.old_version
def validate(self) -> None:
"""
perform version validation
"""
if self.old_version < 0 or self.old_version > self.new_version:
raise MigrationError(f"Invalid current schema version, expected less or equal to {self.new_version}, "
f"got {self.old_version}")

2
src/ahriman/models/package.py
View File

@ -239,7 +239,7 @@ class Package:
from ahriman.core.build_tools.sources import Sources
logger = logging.getLogger("build_details")
Sources.load(paths.cache_for(self.base), self.git_url, paths.patches_for(self.base))
Sources.load(paths.cache_for(self.base), self.git_url, None)
try:
# update pkgver first

9
src/ahriman/models/package_description.py
View File

@ -19,7 +19,7 @@
#
from __future__ import annotations
from dataclasses import dataclass, field, fields
from dataclasses import asdict, dataclass, field, fields
from pathlib import Path
from pyalpm import Package # type: ignore
from typing import Any, Dict, List, Optional, Type
@ -94,3 +94,10 @@ class PackageDescription:
licenses=package.licenses,
provides=package.provides,
url=package.url)
def view(self) -> Dict[str, Any]:
"""
generate json package view
:return: json-friendly dictionary
"""
return asdict(self)

55
src/ahriman/models/repository_paths.py
View File

@ -52,16 +52,9 @@ class RepositoryPaths:
"""
:return: directory for devtools chroot
"""
# for the chroot directory devtools will create own tree and we don"t have to specify architecture here
# for the chroot directory devtools will create own tree, and we don"t have to specify architecture here
return self.root / "chroot"
@property
def manual(self) -> Path:
"""
:return: directory for manual updates (i.e. from add command)
"""
return self.root / "manual" / self.architecture
@property
def packages(self) -> Path:
"""
@ -69,13 +62,6 @@ class RepositoryPaths:
"""
return self.root / "packages" / self.architecture
@property
def patches(self) -> Path:
"""
:return: directory for source patches
"""
return self.root / "patches"
@property
def repository(self) -> Path:
"""
@ -90,13 +76,6 @@ class RepositoryPaths:
"""
return self.owner(self.root)
@property
def sources(self) -> Path:
"""
:return: directory for downloaded PKGBUILDs for current build
"""
return self.root / "sources" / self.architecture
@classmethod
def known_architectures(cls: Type[RepositoryPaths], root: Path) -> Set[str]:
"""
@ -151,30 +130,6 @@ class RepositoryPaths:
set_owner(path)
path = path.parent
def manual_for(self, package_base: str) -> Path:
"""
get manual path for specific package base
:param package_base: package base name
:return: full path to directory for specified package base manual updates
"""
return self.manual / package_base
def patches_for(self, package_base: str) -> Path:
"""
get patches path for specific package base
:param package_base: package base name
:return: full path to directory for specified package base patches
"""
return self.patches / package_base
def sources_for(self, package_base: str) -> Path:
"""
get path to directory from where build will start for the package base
:param package_base: package base name
:return: full path to directory for specified package base sources
"""
return self.sources / package_base
def tree_clear(self, package_base: str) -> None:
"""
clear package specific files
@ -182,9 +137,7 @@ class RepositoryPaths:
"""
for directory in (
self.cache_for(package_base),
self.manual_for(package_base),
self.patches_for(package_base),
self.sources_for(package_base)):
):
shutil.rmtree(directory, ignore_errors=True)
def tree_create(self) -> None:
@ -194,10 +147,8 @@ class RepositoryPaths:
for directory in (
self.cache,
self.chroot,
self.manual,
self.packages,
self.patches,
self.repository,
self.sources):
):
directory.mkdir(mode=0o755, parents=True, exist_ok=True)
self.chown(directory)

8
src/ahriman/models/user.py
View File

@ -79,18 +79,18 @@ class User:
verified = False # the absence of evidence is not the evidence of absence (c) Gin Rummy
return verified
def hash_password(self, salt: str) -> str:
def hash_password(self, salt: str) -> User:
"""
generate hashed password from plain text
:param salt: salt for hashed password
:return: hashed string to store in configuration
:return: user with hashed password to store in configuration
"""
if not self.password:
# in case of empty password we leave it empty. This feature is used by any external (like OAuth) provider
# when we do not store any password here
return ""
return self
password_hash: str = self._HASHER.hash(self.password + salt)
return password_hash
return User(self.username, password_hash, self.access)
def verify_access(self, required: UserAccess) -> bool:
"""

5
src/ahriman/web/routes.py
View File

@ -22,7 +22,6 @@ from pathlib import Path
from ahriman.web.views.index import IndexView
from ahriman.web.views.service.add import AddView
from ahriman.web.views.service.reload_auth import ReloadAuthView
from ahriman.web.views.service.remove import RemoveView
from ahriman.web.views.service.request import RequestView
from ahriman.web.views.service.search import SearchView
@ -45,8 +44,6 @@ def setup_routes(application: Application, static_path: Path) -> None:
POST /service-api/v1/add add new packages to repository
POST /service-api/v1/reload-auth reload authentication module
POST /service-api/v1/remove remove existing package from repository
POST /service-api/v1/request request to add new packages to repository
@ -81,8 +78,6 @@ def setup_routes(application: Application, static_path: Path) -> None:
application.router.add_post("/service-api/v1/add", AddView)
application.router.add_post("/service-api/v1/reload-auth", ReloadAuthView)
application.router.add_post("/service-api/v1/remove", RemoveView)
application.router.add_post("/service-api/v1/request", RequestView)

9
src/ahriman/web/views/base.py
View File

@ -24,6 +24,7 @@ from typing import Any, Dict, List, Optional, Type
from ahriman.core.auth.auth import Auth
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.spawn import Spawn
from ahriman.core.status.watcher import Watcher
from ahriman.models.user_access import UserAccess
@ -42,6 +43,14 @@ class BaseView(View):
configuration: Configuration = self.request.app["configuration"]
return configuration
@property
def database(self) -> SQLite:
"""
:return: database instance
"""
database: SQLite = self.request.app["database"]
return database
@property
def service(self) -> Watcher:
"""

51
src/ahriman/web/views/service/reload_auth.py
View File

@ -1,51 +0,0 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import HTTPNoContent, Response
from ahriman.core.auth.auth import Auth
from ahriman.models.user_access import UserAccess
from ahriman.web.views.base import BaseView
class ReloadAuthView(BaseView):
"""
reload authentication module web view
:cvar POST_PERMISSION: post permissions of self
"""
POST_PERMISSION = UserAccess.Write
async def post(self) -> Response:
"""
reload authentication module. No parameters supported here
:return: 204 on success
"""
self.configuration.reload()
try:
import aiohttp_security # type: ignore
self.request.app[aiohttp_security.api.AUTZ_KEY].validator =\
self.request.app["validator"] =\
Auth.load(self.configuration)
except (ImportError, KeyError):
self.request.app.logger.warning("could not update authentication module validator", exc_info=True)
raise
raise HTTPNoContent()

4
src/ahriman/web/views/user/login.py
View File

@ -58,7 +58,7 @@ class LoginView(BaseView):
identity = UserIdentity.from_username(username, self.validator.max_age)
if identity is not None and await self.validator.known_username(username):
await remember(self.request, response, identity.to_identity())
return response
raise response
raise HTTPUnauthorized()
@ -81,6 +81,6 @@ class LoginView(BaseView):
identity = UserIdentity.from_username(username, self.validator.max_age)
if identity is not None and await self.validator.check_credentials(username, data.get("password")):
await remember(self.request, response, identity.to_identity())
return response
raise response
raise HTTPUnauthorized()

8
src/ahriman/web/web.py
View File

@ -25,6 +25,7 @@ from aiohttp import web
from ahriman.core.auth.auth import Auth
from ahriman.core.configuration import Configuration
from ahriman.core.database.sqlite import SQLite
from ahriman.core.exceptions import InitializeException
from ahriman.core.spawn import Spawn
from ahriman.core.status.watcher import Watcher
@ -93,8 +94,11 @@ def setup_service(architecture: str, configuration: Configuration, spawner: Spaw
application.logger.info("setup configuration")
application["configuration"] = configuration
application.logger.info("setup database and perform migrations")
database = application["database"] = SQLite.load(configuration)
application.logger.info("setup watcher")
application["watcher"] = Watcher(architecture, configuration)
application["watcher"] = Watcher(architecture, configuration, database)
application.logger.info("setup process spawner")
application["spawn"] = spawner
@ -108,7 +112,7 @@ def setup_service(architecture: str, configuration: Configuration, spawner: Spaw
check_host=configuration.getboolean("web", "debug_check_host", fallback=False))
application.logger.info("setup authorization")
validator = application["validator"] = Auth.load(configuration)
validator = application["validator"] = Auth.load(configuration, database)
if validator.enabled:
from ahriman.web.middlewares.auth_handler import setup_auth
setup_auth(application, validator)