arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2026-01-16 11:13:43 +00:00
Code Issues Projects Releases Wiki Activity

update tests

Browse Source
This commit is contained in:
Evgenii Alekseev
2023-08-29 04:48:41 +03:00
committed by Evgenii Alekseev
parent 252f7ba6d4
commit 896384c081
90 changed files with 1172 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/ahriman/application/handlers/handler.py
View File

@ -25,7 +25,7 @@ from multiprocessing import Pool
from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError
from ahriman.core.log.log_loader import LogLoader
from ahriman.core.log import Log
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@ -65,8 +65,8 @@ class Handler:
try:
configuration = Configuration.from_path(args.configuration, repository_id)
log_handler = LogLoader.handler(args.log_handler)
LogLoader.load(configuration, log_handler, quiet=args.quiet, report=args.report)
log_handler = Log.handler(args.log_handler)
Log.load(configuration, log_handler, quiet=args.quiet, report=args.report)
with Lock(args, repository_id, configuration):
cls.run(args, repository_id, configuration, report=args.report)
@ -116,7 +116,7 @@ class Handler:
args(argparse.Namespace): command line args
Returns:
tuple[str | None, str]: list of repository names and architectures for which tree is created
list[RepositoryId]: list of repository names and architectures for which tree is created
Raises:
MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed

4
src/ahriman/application/handlers/setup.py
View File

@ -66,8 +66,8 @@ class Setup(Handler):
Setup.configuration_create_makepkg(args.packager, args.makeflags_jobs, application.repository.paths)
Setup.executable_create(application.repository.paths, repository_id)
repository_server = f"file://{application.repository.paths.repository}" if args.server is None else args.server
Setup.configuration_create_devtools(repository_id, args.from_configuration, args.mirror, args.multilib,
repository_server)
Setup.configuration_create_devtools(
repository_id, args.from_configuration, args.mirror, args.multilib, repository_server)
Setup.configuration_create_sudo(application.repository.paths, repository_id)
application.repository.repo.init()

3
src/ahriman/application/handlers/web.py
View File

@ -77,8 +77,7 @@ class Web(Handler):
"""
# read architecture from the same argument list
yield from ["--architecture", repository_id.architecture]
if repository_id.name is not None:
yield from ["--repository", repository_id.name]
yield from ["--repository", repository_id.name]
# read configuration path from current settings
if (configuration_path := configuration.path) is not None:
yield from ["--configuration", str(configuration_path)]

2
src/ahriman/application/lock.py
View File

@ -70,7 +70,7 @@ class Lock(LazyLogging):
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
lock_suffix = f"{repository_id.name}_{repository_id.architecture}" if repository_id.name is not None else repository_id.architecture
lock_suffix = f"{repository_id.name}_{repository_id.architecture}"
self.path: Path | None = \
args.lock.with_stem(f"{args.lock.stem}_{lock_suffix}") if args.lock is not None else None

21
src/ahriman/core/configuration/configuration.py
View File

@ -89,6 +89,17 @@ class Configuration(configparser.RawConfigParser):
self.path: Path | None = None
self.includes: list[Path] = []
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
_, repository_id = self.check_loaded()
return repository_id.architecture
@property
def include(self) -> Path:
"""
@ -163,23 +174,23 @@ class Configuration(configparser.RawConfigParser):
# the valid order is global < per architecture < per repository < per repository and architecture
return [
Configuration.section_name(section, repository_id.architecture), # architecture specific override
Configuration.section_name(section, repository_id.name),
Configuration.section_name(section, repository_id.name, repository_id.architecture),
Configuration.section_name(section, repository_id.name), # override with repository name
Configuration.section_name(section, repository_id.name, repository_id.architecture), # both
]
@staticmethod
def section_name(section: str, *suffixes: str) -> str:
def section_name(section: str, *suffixes: str | None) -> str:
"""
generate section name for sections which depends on context
Args:
section(str): section name
*suffixes(str): session suffix, e.g. repository architecture
*suffixes(str | None): session suffix, e.g. repository architecture
Returns:
str: correct section name for repository specific section
"""
for suffix in suffixes:
for suffix in filter(bool, suffixes):
section = f"{section}:{suffix}"
return section

2
src/ahriman/core/database/migrations/m001_package_source.py
View File

@ -80,7 +80,7 @@ def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> N
version=row["version"],
remote=RemoteSource.from_json(row),
packages={},
packager=row["packager"] or None,
packager=row.get("packager") or None,
) for row in connection.execute("""select * from package_bases""")
}

77
src/ahriman/core/database/migrations/m011_repository_name.py
View File

@ -51,6 +51,9 @@ steps = [
alter table build_queue add column repository text not null default ''
""",
"""
alter table build_queue add column architecture text not null default ''
""",
"""
alter table build_queue rename to build_queue_
""",
"""
@ -58,7 +61,8 @@ steps = [
package_base text not null,
properties json not null,
repository text not null,
primary key (package_base, repository)
architecture text not null,
unique (package_base, architecture, repository)
)
""",
"""
@ -72,6 +76,9 @@ steps = [
alter table package_bases add column repository text not null default ''
""",
"""
alter table package_bases add column architecture text not null default ''
""",
"""
alter table package_bases rename to package_bases_
""",
"""
@ -85,7 +92,8 @@ steps = [
source text,
packager text,
repository text not null,
primary key (package_base, repository)
architecture text not null,
unique (package_base, architecture, repository)
)
""",
"""
@ -99,6 +107,9 @@ steps = [
alter table package_statuses add column repository text not null default ''
""",
"""
alter table package_statuses add column architecture text not null default ''
""",
"""
alter table package_statuses rename to package_statuses_
""",
"""
@ -107,7 +118,8 @@ steps = [
status text not null,
last_updated integer,
repository text not null,
primary key (package_base, repository)
architecture text not null,
unique (package_base, architecture, repository)
)
""",
"""
@ -142,7 +154,7 @@ steps = [
opt_depends json,
check_depends json,
repository text not null,
primary key (package, architecture, repository)
unique (package, architecture, repository)
)
""",
"""
@ -151,39 +163,14 @@ steps = [
"""
drop table packages_
""",
# patches
"""
alter table patches add column repository text not null default ''
""",
"""
drop index patches_package_base_variable
""",
"""
alter table patches rename to patches_
""",
"""
create table patches (
package_base text not null,
variable text,
patch blob not null,
repository text not null
)
""",
"""
create unique index patches_package_base_variable_repository
on patches (package_base, coalesce(variable, ''), repository)
""",
"""
insert into patches select * from patches_
""",
"""
drop table patches_
""",
# logs
"""
alter table logs add column repository text not null default ''
""",
"""
alter table logs add column architecture text not null default ''
""",
"""
drop index logs_package_base_version
""",
"""
@ -195,14 +182,16 @@ steps = [
created real not null,
record text,
version text not null,
repository text not null
repository text not null,
architecture text not null
)
""",
"""
insert into logs select * from logs_
""",
"""
create index logs_package_base_version on logs (package_base, version)
create index logs_package_base_version_architecture_repository
on logs (package_base, version, architecture, repository)
""",
"""
drop table logs_
@ -231,15 +220,13 @@ def migrate_package_repository(connection: Connection, configuration: Configurat
"""
_, repository_id = configuration.check_loaded()
connection.execute("""update build_queue set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update package_bases set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update package_statuses set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update build_queue set repository = :repository, architecture = :architecture""",
{"repository": repository_id.name, "architecture": repository_id.architecture})
connection.execute("""update package_bases set repository = :repository, architecture = :architecture""",
{"repository": repository_id.name, "architecture": repository_id.architecture})
connection.execute("""update package_statuses set repository = :repository, architecture = :architecture""",
{"repository": repository_id.name, "architecture": repository_id.architecture})
connection.execute("""update packages set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update patches set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update logs set repository = :repository""",
{"repository": repository_id.name, })
{"repository": repository_id.name})
connection.execute("""update logs set repository = :repository, architecture = :architecture""",
{"repository": repository_id.name, "architecture": repository_id.architecture})

29
src/ahriman/core/database/operations/build_operations.py
View File

@ -39,9 +39,14 @@ class BuildOperations(Operations):
connection.execute(
"""
delete from build_queue
where (:package_base is null or package_base = :package_base) and repository = :repository
where (:package_base is null or package_base = :package_base)
and repository = :repository and architecture = :architecture
""",
{"package_base": package_base, "repository": self.repository_id.name})
{
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
return self.with_connection(run, commit=True)
@ -56,8 +61,11 @@ class BuildOperations(Operations):
return [
Package.from_json(row["properties"])
for row in connection.execute(
"""select properties from build_queue where repository = :repository""",
{"repository": self.repository_id.name}
"""
select properties from build_queue
where repository = :repository and architecture = :architecture
""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture}
)
]
@ -74,12 +82,17 @@ class BuildOperations(Operations):
connection.execute(
"""
insert into build_queue
(package_base, properties, repository)
(package_base, properties, repository, architecture)
values
(:package_base, :properties, :repository)
on conflict (package_base, repository) do update set
(:package_base, :properties, :repository, :architecture)
on conflict (package_base, architecture, repository) do update set
properties = :properties
""",
{"package_base": package.base, "properties": package.view(), "repository": self.repository_id.name})
{
"package_base": package.base,
"properties": package.view(),
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
return self.with_connection(run, commit=True)

24
src/ahriman/core/database/operations/logs_operations.py
View File

@ -46,10 +46,16 @@ class LogsOperations(Operations):
for row in connection.execute(
"""
select created, record from logs
where package_base = :package_base and repository = :repository
where package_base = :package_base and repository = :repository and architecture = :architecture
order by created limit :limit offset :offset
""",
{"package_base": package_base, "repository": self.repository_id.name, "limit": limit, "offset": offset})
{
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
"limit": limit,
"offset": offset,
})
]
return self.with_connection(run)
@ -67,9 +73,9 @@ class LogsOperations(Operations):
connection.execute(
"""
insert into logs
(package_base, version, created, record, repository)
(package_base, version, created, record, repository, architecture)
values
(:package_base, :version, :created, :record, :repository)
(:package_base, :version, :created, :record, :repository, :architecture)
""",
{
"package_base": log_record_id.package_base,
@ -77,6 +83,7 @@ class LogsOperations(Operations):
"created": created,
"record": record,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
}
)
@ -95,10 +102,15 @@ class LogsOperations(Operations):
connection.execute(
"""
delete from logs
where package_base = :package_base and repository = :repository
where package_base = :package_base and repository = :repository and architecture = :architecture
and (:version is null or version <> :version)
""",
{"package_base": package_base, "version": version, "repository": self.repository_id.name}
{
"package_base": package_base,
"version": version,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
}
)
return self.with_connection(run, commit=True)

58
src/ahriman/core/database/operations/package_operations.py
View File

@ -43,14 +43,23 @@ class PackageOperations(Operations):
connection.execute(
"""
delete from package_statuses
where package_base = :package_base and repository = :repository
where package_base = :package_base and repository = :repository and architecture = :architecture
""",
{"package_base": package_base, "repository": self.repository_id.name})
{
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
connection.execute(
"""
delete from package_bases
where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
where package_base = :package_base and repository = :repository and architecture = :architecture
""",
{
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
def _package_remove_packages(self, connection: Connection, package_base: str,
current_packages: Iterable[str]) -> None:
@ -66,15 +75,19 @@ class PackageOperations(Operations):
package
for package in connection.execute(
"""
select package, repository from packages
where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
select package, repository, architecture from packages
where package_base = :package_base and repository = :repository and architecture = :architecture""",
{
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
if package["package"] not in current_packages
]
connection.executemany(
"""
delete from packages
where package = :package and repository = :repository
where package = :package and repository = :repository and architecture = :architecture
""",
packages)
@ -89,10 +102,12 @@ class PackageOperations(Operations):
connection.execute(
"""
insert into package_bases
(package_base, version, source, branch, git_url, path, web_url, packager, repository)
(package_base, version, source, branch, git_url, path, web_url, packager,
repository, architecture)
values
(:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager, :repository)
on conflict (package_base, repository) do update set
(:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager,
:repository, :architecture)
on conflict (package_base, architecture, repository) do update set
version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url,
source = :source, packager = :packager
""",
@ -106,6 +121,7 @@ class PackageOperations(Operations):
"source": package.remote.source.value,
"packager": package.packager,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
}
)
@ -161,10 +177,10 @@ class PackageOperations(Operations):
connection.execute(
"""
insert into package_statuses
(package_base, status, last_updated, repository)
(package_base, status, last_updated, repository, architecture)
values
(:package_base, :status, :last_updated, :repository)
on conflict (package_base, repository) do update set
(:package_base, :status, :last_updated, :repository, :architecture)
on conflict (package_base, architecture, repository) do update set
status = :status, last_updated = :last_updated
""",
{
@ -172,6 +188,7 @@ class PackageOperations(Operations):
"status": status.status.value,
"last_updated": status.timestamp,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
def _packages_get_select_package_bases(self, connection: Connection) -> dict[str, Package]:
@ -192,8 +209,8 @@ class PackageOperations(Operations):
packages={},
packager=row["packager"] or None,
) for row in connection.execute(
"""select * from package_bases where repository = :repository""",
{"repository": self.repository_id.name}
"""select * from package_bases where repository = :repository and architecture = :architecture""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture}
)
}
@ -209,8 +226,9 @@ class PackageOperations(Operations):
dict[str, Package]: map of the package base to its descriptor including individual packages
"""
for row in connection.execute(
"""select * from packages where repository = :repository""",
{"repository": self.repository_id.name}):
"""select * from packages where repository = :repository and architecture = :architecture""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture}
):
if row["package_base"] not in packages:
continue # normally must never happen though
packages[row["package_base"]].packages[row["package"]] = PackageDescription.from_json(row)
@ -229,8 +247,8 @@ class PackageOperations(Operations):
return {
row["package_base"]: BuildStatus.from_json({"status": row["status"], "timestamp": row["last_updated"]})
for row in connection.execute(
"""select * from package_statuses where repository = :repository""",
{"repository": self.repository_id.name}
"""select * from package_statuses where repository = :repository and architecture = :architecture""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture}
)
}

38
src/ahriman/core/database/operations/patch_operations.py
View File

@ -54,19 +54,13 @@ class PatchOperations(Operations):
connection.execute(
"""
insert into patches
(package_base, variable, patch, repository)
(package_base, variable, patch)
values
(:package_base, :variable, :patch, :repository)
on conflict (package_base, coalesce(variable, ''), repository) do update set
(:package_base, :variable, :patch)
on conflict (package_base, coalesce(variable, '')) do update set
patch = :patch
""",
{
"package_base": package_base,
"variable": patch.key,
"patch": patch.value,
"repository": self.repository_id.name,
}
)
{"package_base": package_base, "variable": patch.key, "patch": patch.value})
return self.with_connection(run, commit=True)
@ -85,10 +79,8 @@ class PatchOperations(Operations):
return [
(row["package_base"], PkgbuildPatch(row["variable"], row["patch"]))
for row in connection.execute(
"""
select * from patches
where (:package_base is null or package_base = :package_base) and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
"""select * from patches where :package_base is null or package_base = :package_base""",
{"package_base": package_base})
]
# we could use itertools & operator but why?
@ -109,23 +101,13 @@ class PatchOperations(Operations):
"""
def run_many(connection: Connection) -> None:
connection.executemany(
"""
delete from patches
where package_base = :package_base and variable = :variable and repository = :repository
""",
[
{
"package_base": package_base,
"variable": variable,
"repository": self.repository_id.name,
} for variable in variables
]
)
"""delete from patches where package_base = :package_base and variable = :variable""",
[{"package_base": package_base, "variable": variable} for variable in variables])
def run(connection: Connection) -> None:
connection.execute(
"""delete from patches where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
"""delete from patches where package_base = :package_base""",
{"package_base": package_base})
if variables:
return self.with_connection(run_many, commit=True)

1
src/ahriman/core/log/__init__.py
View File

@ -18,3 +18,4 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.log.lazy_logging import LazyLogging
from ahriman.core.log.log import Log

6
src/ahriman/core/log/log_loader.py → src/ahriman/core/log/log.py
View File

@ -27,7 +27,7 @@ from ahriman.core.log.http_log_handler import HttpLogHandler
from ahriman.models.log_handler import LogHandler
class LogLoader:
class Log:
"""
simple static method class which setups application loggers
@ -63,7 +63,7 @@ class LogLoader:
del JournalHandler
return LogHandler.Journald # journald import was found
except ImportError:
if LogLoader.DEFAULT_SYSLOG_DEVICE.exists():
if Log.DEFAULT_SYSLOG_DEVICE.exists():
return LogHandler.Syslog
return LogHandler.Console
@ -94,7 +94,7 @@ class LogLoader:
fileConfig(log_configuration, disable_existing_loggers=True)
logging.debug("using %s logger", default_handler)
except Exception:
logging.basicConfig(filename=None, format=LogLoader.DEFAULT_LOG_FORMAT, level=LogLoader.DEFAULT_LOG_LEVEL)
logging.basicConfig(filename=None, format=Log.DEFAULT_LOG_FORMAT, level=Log.DEFAULT_LOG_LEVEL)
logging.exception("could not load logging from configuration, fallback to stderr")
HttpLogHandler.load(configuration, report=report)

2
src/ahriman/core/upload/s3.py
View File

@ -132,7 +132,7 @@ class S3(Upload):
continue
local_path = path / local_file
remote_path = self.remote_root / local_file
remote_path = self.remote_root / local_file.name
(mime, _) = mimetypes.guess_type(local_path)
extra_args = {"ContentType": mime} if mime is not None else None

2
src/ahriman/models/repository_id.py
View File

@ -62,4 +62,4 @@ class RepositoryId:
if not isinstance(other, RepositoryId):
raise ValueError(f"'<' not supported between instances of '{type(self)}' and '{type(other)}'")
return self.name <= other.name and self.architecture < other.architecture
return (self.name, self.architecture) < (other.name, other.architecture)

28
src/ahriman/models/repository_paths.py
View File

@ -26,12 +26,11 @@ from functools import cached_property
from pathlib import Path
from ahriman.core.exceptions import PathError
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
@dataclass(frozen=True)
class RepositoryPaths(LazyLogging):
class RepositoryPaths:
"""
repository paths holder. For the most operations with paths you want to use this object
@ -57,6 +56,16 @@ class RepositoryPaths(LazyLogging):
root: Path
repository_id: RepositoryId
@property
def _repository_root(self) -> Path:
"""
repository root which can be used for invalid (not fully loaded instances)
Returns:
Path: root path to repositories
"""
return self.root / "repository"
@cached_property
def _suffix(self) -> Path:
"""
@ -66,8 +75,7 @@ class RepositoryPaths(LazyLogging):
Path: relative path which contains only architecture segment in case if legacy tree is used and repository
name and architecture otherwise
"""
if (self.root / "repository" / self.repository_id.architecture).is_dir():
self.logger.warning("legacy single repository tree has been found")
if (self._repository_root / self.repository_id.architecture).is_dir():
return Path(self.repository_id.architecture)
return Path(self.repository_id.name) / self.repository_id.architecture
@ -120,7 +128,7 @@ class RepositoryPaths(LazyLogging):
Returns:
Path: full path to the repository directory
"""
return self.root / "repository" / self._suffix
return self._repository_root / self._suffix
@property
def root_owner(self) -> tuple[int, int]:
@ -148,13 +156,15 @@ class RepositoryPaths(LazyLogging):
for architecture in filter(lambda path: path.is_dir(), repository_path.iterdir()):
yield RepositoryId(architecture.name, repository_name)
def walk_with_name(paths: RepositoryPaths) -> Generator[RepositoryId, None, None]:
for repository in filter(lambda path: path.is_dir(), paths.repository.iterdir()):
def walk_root(paths: RepositoryPaths) -> Generator[RepositoryId, None, None]:
# pylint: disable=protected-access
for repository in filter(lambda path: path.is_dir(), paths._repository_root.iterdir()):
print(repository)
yield from walk(repository, repository.name)
instance = cls(root, RepositoryId("", ""))
instance = cls(root, RepositoryId("", root.name)) # suppress initialization error
# try to get list per repository first and then fallback to old schema if nothing found
return set(walk_with_name(instance)) or set(walk(instance.repository, name))
return set(walk_root(instance)) or set(walk(instance._repository_root, name))
@staticmethod
def owner(path: Path) -> tuple[int, int]: