arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 15:27:17 +00:00
Code Issues Projects Releases Wiki Activity

use repostory id instead

Browse Source
This commit is contained in:
Evgenii Alekseev 2023-09-04 16:04:05 +03:00
parent 1b5bfe5088
commit e514740032
10 changed files with 82 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/setup.sh vendored
View File

@ -38,13 +38,13 @@ systemd-machine-id-setup
[[ -z $MINIMAL_INSTALL ]] && WEB_ARGS=("--web-port" "8080") [[ -z $MINIMAL_INSTALL ]] && WEB_ARGS=("--web-port" "8080")
ahriman -a x86_64 -r "github" service-setup --packager "ahriman bot <ahriman@example.com>" "${WEB_ARGS[@]}" ahriman -a x86_64 -r "github" service-setup --packager "ahriman bot <ahriman@example.com>" "${WEB_ARGS[@]}"
# validate configuration # validate configuration
ahriman -a x86_64 -r "github" service-config-validate --exit-code ahriman service-config-validate --exit-code
# enable services # enable services
systemctl enable ahriman-web@x86_64 systemctl enable ahriman-web@x86_64-github
systemctl enable ahriman@x86_64.timer systemctl enable ahriman@x86_64-github.timer
if [[ -z $MINIMAL_INSTALL ]]; then if [[ -z $MINIMAL_INSTALL ]]; then
# run web service (detached) # run web service (detached)
sudo -u ahriman -- ahriman -a x86_64 web & sudo -u ahriman -- ahriman web &
WEB_PID=$! WEB_PID=$!
fi fi
# add the first package # add the first package

6
docs/ahriman.core.log.rst
View File

@ -36,10 +36,10 @@ ahriman.core.log.lazy\_logging module
:no-undoc-members: :no-undoc-members:
:show-inheritance: :show-inheritance:
ahriman.core.log.log module ahriman.core.log.log\_loader module
--------------------------- -----------------------------------
.. automodule:: ahriman.core.log.log .. automodule:: ahriman.core.log.log_loader
:members: :members:
:no-undoc-members: :no-undoc-members:
:show-inheritance: :show-inheritance:

43
src/ahriman/core/database/migrations/m011_repository_name.py
View File

@ -51,9 +51,6 @@ steps = [
alter table build_queue add column repository text not null default '' alter table build_queue add column repository text not null default ''
""", """,
""" """
alter table build_queue add column architecture text not null default ''
""",
"""
alter table build_queue rename to build_queue_ alter table build_queue rename to build_queue_
""", """,
""" """
@ -61,8 +58,7 @@ steps = [
package_base text not null, package_base text not null,
properties json not null, properties json not null,
repository text not null, repository text not null,
architecture text not null, unique (package_base, repository)
unique (package_base, architecture, repository)
) )
""", """,
""" """
@ -76,9 +72,6 @@ steps = [
alter table package_bases add column repository text not null default '' alter table package_bases add column repository text not null default ''
""", """,
""" """
alter table package_bases add column architecture text not null default ''
""",
"""
alter table package_bases rename to package_bases_ alter table package_bases rename to package_bases_
""", """,
""" """
@ -92,8 +85,7 @@ steps = [
source text, source text,
packager text, packager text,
repository text not null, repository text not null,
architecture text not null, unique (package_base, repository)
unique (package_base, architecture, repository)
) )
""", """,
""" """
@ -107,9 +99,6 @@ steps = [
alter table package_statuses add column repository text not null default '' alter table package_statuses add column repository text not null default ''
""", """,
""" """
alter table package_statuses add column architecture text not null default ''
""",
"""
alter table package_statuses rename to package_statuses_ alter table package_statuses rename to package_statuses_
""", """,
""" """
@ -118,8 +107,7 @@ steps = [
status text not null, status text not null,
last_updated integer, last_updated integer,
repository text not null, repository text not null,
architecture text not null, unique (package_base, repository)
unique (package_base, architecture, repository)
) )
""", """,
""" """
@ -168,9 +156,6 @@ steps = [
alter table logs add column repository text not null default '' alter table logs add column repository text not null default ''
""", """,
""" """
alter table logs add column architecture text not null default ''
""",
"""
drop index logs_package_base_version drop index logs_package_base_version
""", """,
""" """
@ -182,16 +167,15 @@ steps = [
created real not null, created real not null,
record text, record text,
version text not null, version text not null,
repository text not null, repository text not null
architecture text not null
) )
""", """,
""" """
insert into logs select * from logs_ insert into logs select * from logs_
""", """,
""" """
create index logs_package_base_version_architecture_repository create index logs_package_base_version_repository
on logs (package_base, version, architecture, repository) on logs (package_base, version, repository)
""", """,
""" """
drop table logs_ drop table logs_
@ -220,13 +204,8 @@ def migrate_package_repository(connection: Connection, configuration: Configurat
""" """
_, repository_id = configuration.check_loaded() _, repository_id = configuration.check_loaded()
connection.execute("""update build_queue set repository = :repository, architecture = :architecture""", connection.execute("""update build_queue set repository = :repository""", {"repository": repository_id.id})
{"repository": repository_id.name, "architecture": repository_id.architecture}) connection.execute("""update package_bases set repository = :repository""", {"repository": repository_id.id})
connection.execute("""update package_bases set repository = :repository, architecture = :architecture""", connection.execute("""update package_statuses set repository = :repository""", {"repository": repository_id.id})
{"repository": repository_id.name, "architecture": repository_id.architecture}) connection.execute("""update packages set repository = :repository""", {"repository": repository_id.id})
connection.execute("""update package_statuses set repository = :repository, architecture = :architecture""", connection.execute("""update logs set repository = :repository""", {"repository": repository_id.id})
{"repository": repository_id.name, "architecture": repository_id.architecture})
connection.execute("""update packages set repository = :repository""",
{"repository": repository_id.name})
connection.execute("""update logs set repository = :repository, architecture = :architecture""",
{"repository": repository_id.name, "architecture": repository_id.architecture})

22
src/ahriman/core/database/operations/build_operations.py
View File

@ -39,13 +39,11 @@ class BuildOperations(Operations):
connection.execute( connection.execute(
""" """
delete from build_queue delete from build_queue
where (:package_base is null or package_base = :package_base) where (:package_base is null or package_base = :package_base) and repository = :repository
and repository = :repository and architecture = :architecture
""", """,
{ {
"package_base": package_base, "package_base": package_base,
"repository": self.repository_id.name, "repository": self.repository_id.id,
"architecture": self.repository_id.architecture,
}) })
return self.with_connection(run, commit=True) return self.with_connection(run, commit=True)
@ -61,11 +59,8 @@ class BuildOperations(Operations):
return [ return [
Package.from_json(row["properties"]) Package.from_json(row["properties"])
for row in connection.execute( for row in connection.execute(
""" """select properties from build_queue where repository = :repository""",
select properties from build_queue {"repository": self.repository_id.id}
where repository = :repository and architecture = :architecture
""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture}
) )
] ]
@ -82,17 +77,16 @@ class BuildOperations(Operations):
connection.execute( connection.execute(
""" """
insert into build_queue insert into build_queue
(package_base, properties, repository, architecture) (package_base, properties, repository)
values values
(:package_base, :properties, :repository, :architecture) (:package_base, :properties, :repository)
on conflict (package_base, architecture, repository) do update set on conflict (package_base, repository) do update set
properties = :properties properties = :properties
""", """,
{ {
"package_base": package.base, "package_base": package.base,
"properties": package.view(), "properties": package.view(),
"repository": self.repository_id.name, "repository": self.repository_id.id,
"architecture": self.repository_id.architecture,
}) })
return self.with_connection(run, commit=True) return self.with_connection(run, commit=True)

18
src/ahriman/core/database/operations/logs_operations.py
View File

@ -46,13 +46,12 @@ class LogsOperations(Operations):
for row in connection.execute( for row in connection.execute(
""" """
select created, record from logs select created, record from logs
where package_base = :package_base and repository = :repository and architecture = :architecture where package_base = :package_base and repository = :repository
order by created limit :limit offset :offset order by created limit :limit offset :offset
""", """,
{ {
"package_base": package_base, "package_base": package_base,
"repository": self.repository_id.name, "repository": self.repository_id.id,
"architecture": self.repository_id.architecture,
"limit": limit, "limit": limit,
"offset": offset, "offset": offset,
}) })
@ -73,17 +72,16 @@ class LogsOperations(Operations):
connection.execute( connection.execute(
""" """
insert into logs insert into logs
(package_base, version, created, record, repository, architecture) (package_base, version, created, record, repository)
values values
(:package_base, :version, :created, :record, :repository, :architecture) (:package_base, :version, :created, :record, :repository)
""", """,
{ {
"package_base": log_record_id.package_base, "package_base": log_record_id.package_base,
"version": log_record_id.version, "version": log_record_id.version,
"created": created, "created": created,
"record": record, "record": record,
"repository": self.repository_id.name, "repository": self.repository_id.id,
"architecture": self.repository_id.architecture,
} }
) )
@ -102,14 +100,14 @@ class LogsOperations(Operations):
connection.execute( connection.execute(
""" """
delete from logs delete from logs
where package_base = :package_base and repository = :repository and architecture = :architecture where package_base = :package_base
and repository = :repository
and (:version is null or version <> :version) and (:version is null or version <> :version)
""", """,
{ {
"package_base": package_base, "package_base": package_base,
"version": version, "version": version,
"repository": self.repository_id.name, "repository": self.repository_id.id,
"architecture": self.repository_id.architecture,
} }
) )

71
src/ahriman/core/database/operations/package_operations.py
View File

@ -41,25 +41,11 @@ class PackageOperations(Operations):
package_base(str): package base name package_base(str): package base name
""" """
connection.execute( connection.execute(
""" """delete from package_statuses where package_base = :package_base and repository = :repository""",
delete from package_statuses {"package_base": package_base, "repository": self.repository_id.id})
where package_base = :package_base and repository = :repository and architecture = :architecture
""",
{
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
connection.execute( connection.execute(
""" """delete from package_bases where package_base = :package_base and repository = :repository""",
delete from package_bases {"package_base": package_base, "repository": self.repository_id.id})
where package_base = :package_base and repository = :repository and architecture = :architecture
""",
{
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
def _package_remove_packages(self, connection: Connection, package_base: str, def _package_remove_packages(self, connection: Connection, package_base: str,
current_packages: Iterable[str]) -> None: current_packages: Iterable[str]) -> None:
@ -75,20 +61,13 @@ class PackageOperations(Operations):
package package
for package in connection.execute( for package in connection.execute(
""" """
select package, repository, architecture from packages select package, repository from packages
where package_base = :package_base and repository = :repository and architecture = :architecture""", where package_base = :package_base and repository = :repository""",
{ {"package_base": package_base, "repository": self.repository_id.id})
"package_base": package_base,
"repository": self.repository_id.name,
"architecture": self.repository_id.architecture,
})
if package["package"] not in current_packages if package["package"] not in current_packages
] ]
connection.executemany( connection.executemany(
""" """delete from packages where package = :package and repository = :repository""",
delete from packages
where package = :package and repository = :repository and architecture = :architecture
""",
packages) packages)
def _package_update_insert_base(self, connection: Connection, package: Package) -> None: def _package_update_insert_base(self, connection: Connection, package: Package) -> None:
@ -102,12 +81,10 @@ class PackageOperations(Operations):
connection.execute( connection.execute(
""" """
insert into package_bases insert into package_bases
(package_base, version, source, branch, git_url, path, web_url, packager, (package_base, version, source, branch, git_url, path, web_url, packager, repository)
repository, architecture)
values values
(:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager, (:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager, :repository)
:repository, :architecture) on conflict (package_base, repository) do update set
on conflict (package_base, architecture, repository) do update set
version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url, version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url,
source = :source, packager = :packager source = :source, packager = :packager
""", """,
@ -120,8 +97,7 @@ class PackageOperations(Operations):
"web_url": package.remote.web_url, "web_url": package.remote.web_url,
"source": package.remote.source.value, "source": package.remote.source.value,
"packager": package.packager, "packager": package.packager,
"repository": self.repository_id.name, "repository": self.repository_id.id,
"architecture": self.repository_id.architecture,
} }
) )
@ -140,7 +116,7 @@ class PackageOperations(Operations):
package_list.append({ package_list.append({
"package": name, "package": name,
"package_base": package.base, "package_base": package.base,
"repository": self.repository_id.name, "repository": self.repository_id.id,
**description.view(), **description.view(),
}) })
connection.executemany( connection.executemany(
@ -177,18 +153,17 @@ class PackageOperations(Operations):
connection.execute( connection.execute(
""" """
insert into package_statuses insert into package_statuses
(package_base, status, last_updated, repository, architecture) (package_base, status, last_updated, repository)
values values
(:package_base, :status, :last_updated, :repository, :architecture) (:package_base, :status, :last_updated, :repository)
on conflict (package_base, architecture, repository) do update set on conflict (package_base, repository) do update set
status = :status, last_updated = :last_updated status = :status, last_updated = :last_updated
""", """,
{ {
"package_base": package_base, "package_base": package_base,
"status": status.status.value, "status": status.status.value,
"last_updated": status.timestamp, "last_updated": status.timestamp,
"repository": self.repository_id.name, "repository": self.repository_id.id,
"architecture": self.repository_id.architecture,
}) })
def _packages_get_select_package_bases(self, connection: Connection) -> dict[str, Package]: def _packages_get_select_package_bases(self, connection: Connection) -> dict[str, Package]:
@ -209,8 +184,8 @@ class PackageOperations(Operations):
packages={}, packages={},
packager=row["packager"] or None, packager=row["packager"] or None,
) for row in connection.execute( ) for row in connection.execute(
"""select * from package_bases where repository = :repository and architecture = :architecture""", """select * from package_bases where repository = :repository""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture} {"repository": self.repository_id.id}
) )
} }
@ -226,8 +201,8 @@ class PackageOperations(Operations):
dict[str, Package]: map of the package base to its descriptor including individual packages dict[str, Package]: map of the package base to its descriptor including individual packages
""" """
for row in connection.execute( for row in connection.execute(
"""select * from packages where repository = :repository and architecture = :architecture""", """select * from packages where repository = :repository""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture} {"repository": self.repository_id.id}
): ):
if row["package_base"] not in packages: if row["package_base"] not in packages:
continue # normally must never happen though continue # normally must never happen though
@ -247,8 +222,8 @@ class PackageOperations(Operations):
return { return {
row["package_base"]: BuildStatus.from_json({"status": row["status"], "timestamp": row["last_updated"]}) row["package_base"]: BuildStatus.from_json({"status": row["status"], "timestamp": row["last_updated"]})
for row in connection.execute( for row in connection.execute(
"""select * from package_statuses where repository = :repository and architecture = :architecture""", """select * from package_statuses where repository = :repository""",
{"repository": self.repository_id.name, "architecture": self.repository_id.architecture} {"repository": self.repository_id.id}
) )
} }

10
src/ahriman/models/repository_id.py
View File

@ -44,6 +44,16 @@ class RepositoryId:
""" """
return not self.architecture or not self.name return not self.architecture or not self.name
@property
def id(self) -> str:
"""
get repository id to be used for databases
Returns:
str: unique id for this repository
"""
return f"{self.architecture}-{self.name}" # basically the same as used for command line
def __lt__(self, other: Any) -> bool: def __lt__(self, other: Any) -> bool:
""" """
comparison operator for sorting comparison operator for sorting

14
tests/ahriman/core/database/migrations/test_m011_repository_name.py
View File

@ -31,13 +31,9 @@ def test_migrate_package_repository(connection: Connection, configuration: Confi
migrate_package_repository(connection, configuration) migrate_package_repository(connection, configuration)
connection.execute.assert_has_calls([ connection.execute.assert_has_calls([
MockCall(pytest.helpers.anyvar(str, strict=True), MockCall(pytest.helpers.anyvar(str, strict=True), {"repository": configuration.repository_id.id}),
{"repository": configuration.repository_name, "architecture": configuration.architecture}), MockCall(pytest.helpers.anyvar(str, strict=True), {"repository": configuration.repository_id.id}),
MockCall(pytest.helpers.anyvar(str, strict=True), MockCall(pytest.helpers.anyvar(str, strict=True), {"repository": configuration.repository_id.id}),
{"repository": configuration.repository_name, "architecture": configuration.architecture}), MockCall(pytest.helpers.anyvar(str, strict=True), {"repository": configuration.repository_id.id}),
MockCall(pytest.helpers.anyvar(str, strict=True), MockCall(pytest.helpers.anyvar(str, strict=True), {"repository": configuration.repository_id.id}),
{"repository": configuration.repository_name, "architecture": configuration.architecture}),
MockCall(pytest.helpers.anyvar(str, strict=True), {"repository": configuration.repository_name}),
MockCall(pytest.helpers.anyvar(str, strict=True),
{"repository": configuration.repository_name, "architecture": configuration.architecture}),
]) ])

6
tests/ahriman/core/database/operations/test_package_operations.py
View File

@ -18,8 +18,7 @@ def test_package_remove_package_base(database: SQLite, connection: Connection) -
database._package_remove_package_base(connection, "package") database._package_remove_package_base(connection, "package")
args = { args = {
"package_base": "package", "package_base": "package",
"repository": database.repository_id.name, "repository": database.repository_id.id,
"architecture": database.repository_id.architecture,
} }
connection.execute.assert_has_calls([ connection.execute.assert_has_calls([
MockCall(pytest.helpers.anyvar(str, strict=True), args), MockCall(pytest.helpers.anyvar(str, strict=True), args),
@ -35,8 +34,7 @@ def test_package_remove_packages(database: SQLite, connection: Connection, packa
connection.execute.assert_called_once_with( connection.execute.assert_called_once_with(
pytest.helpers.anyvar(str, strict=True), { pytest.helpers.anyvar(str, strict=True), {
"package_base": package_ahriman.base, "package_base": package_ahriman.base,
"repository": database.repository_id.name, "repository": database.repository_id.id,
"architecture": database.repository_id.architecture,
}) })
connection.executemany.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), []) connection.executemany.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), [])

8
tests/ahriman/models/test_repository_id.py
View File

@ -13,6 +13,14 @@ def test_is_empty() -> None:
assert not RepositoryId("arch", "repo").is_empty assert not RepositoryId("arch", "repo").is_empty
def test_id() -> None:
"""
must correctly generate id
"""
assert RepositoryId("", "").id == "-"
assert RepositoryId("arch", "repo").id == "arch-repo"
def test_lt() -> None: def test_lt() -> None:
""" """
must correctly compare instances must correctly compare instances