arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-11-18 22:43:42 +00:00
Code Issues Projects Releases Wiki Activity

ping based workers

Browse Source
This commit is contained in:
Evgenii Alekseev
2024-01-02 16:48:22 +02:00
parent ee806f6d44
commit ac1ed209d4
47 changed files with 306 additions and 953 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
tests/ahriman/application/test_ahriman.py
View File

@ -1320,80 +1320,6 @@ def test_subparsers_service_tree_migrate(parser: argparse.ArgumentParser) -> Non
assert not args.report
def test_subparsers_service_worker_register(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register command must imply trigger
"""
args = parser.parse_args(["service-worker-register"])
assert args.trigger == ["ahriman.core.distributed.WorkerRegisterTrigger"]
def test_subparsers_service_worker_register_option_architecture(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register command must correctly parse architecture list
"""
args = parser.parse_args(["service-worker-register"])
assert args.architecture is None
args = parser.parse_args(["-a", "x86_64", "service-worker-register"])
assert args.architecture == "x86_64"
def test_subparsers_service_worker_register_option_repository(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register command must correctly parse repository list
"""
args = parser.parse_args(["service-worker-register"])
assert args.repository is None
args = parser.parse_args(["-r", "repo", "service-worker-register"])
assert args.repository == "repo"
def test_subparsers_service_worker_register_repo_triggers(parser: argparse.ArgumentParser) -> None:
"""
service-worker-register must have same keys as repo-triggers
"""
args = parser.parse_args(["service-worker-register"])
reference_args = parser.parse_args(["repo-triggers"])
assert dir(args) == dir(reference_args)
def test_subparsers_service_worker_unregister(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister command must imply trigger
"""
args = parser.parse_args(["service-worker-unregister"])
assert args.trigger == ["ahriman.core.distributed.WorkerUnregisterTrigger"]
def test_subparsers_service_worker_unregister_option_architecture(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister command must correctly parse architecture list
"""
args = parser.parse_args(["service-worker-unregister"])
assert args.architecture is None
args = parser.parse_args(["-a", "x86_64", "service-worker-unregister"])
assert args.architecture == "x86_64"
def test_subparsers_service_worker_unregister_option_repository(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister command must correctly parse repository list
"""
args = parser.parse_args(["service-worker-unregister"])
assert args.repository is None
args = parser.parse_args(["-r", "repo", "service-worker-unregister"])
assert args.repository == "repo"
def test_subparsers_service_worker_unregister_repo_triggers(parser: argparse.ArgumentParser) -> None:
"""
service-worker-unregister must have same keys as repo-triggers
"""
args = parser.parse_args(["service-worker-unregister"])
reference_args = parser.parse_args(["repo-triggers"])
assert dir(args) == dir(reference_args)
def test_subparsers_user_add(parser: argparse.ArgumentParser) -> None:
"""
user-add command must imply action, architecture, exit code, lock, quiet, report and repository

8
tests/ahriman/core/database/migrations/test_m013_workers.py
View File

@ -1,8 +0,0 @@
from ahriman.core.database.migrations.m013_workers import steps
def test_migration_workers() -> None:
"""
migration must not be empty
"""
assert steps

46
tests/ahriman/core/database/operations/test_workers_operations.py
View File

@ -1,46 +0,0 @@
from ahriman.core.database import SQLite
from ahriman.models.worker import Worker
def test_workers_get_insert(database: SQLite) -> None:
"""
must insert workers to database
"""
database.workers_insert(Worker("address1", identifier="1"))
database.workers_insert(Worker("address2", identifier="2"))
assert database.workers_get() == [
Worker("address1", identifier="1"), Worker("address2", identifier="2")
]
def test_workers_insert_remove(database: SQLite) -> None:
"""
must remove worker from database
"""
database.workers_insert(Worker("address1", identifier="1"))
database.workers_insert(Worker("address2", identifier="2"))
database.workers_remove("1")
assert database.workers_get() == [Worker("address2", identifier="2")]
def test_workers_insert_remove_all(database: SQLite) -> None:
"""
must remove all workers
"""
database.workers_insert(Worker("address1", identifier="1"))
database.workers_insert(Worker("address2", identifier="2"))
database.workers_remove()
assert database.workers_get() == []
def test_workers_insert_insert(database: SQLite) -> None:
"""
must update worker in database
"""
database.workers_insert(Worker("address1", identifier="1"))
assert database.workers_get() == [Worker("address1", identifier="1")]
database.workers_insert(Worker("address2", identifier="1"))
assert database.workers_get() == [Worker("address2", identifier="1")]

15
tests/ahriman/core/distributed/conftest.py
View File

@ -1,6 +1,7 @@
import pytest
from ahriman.core.configuration import Configuration
from ahriman.core.distributed import WorkersCache
from ahriman.core.distributed.distributed_system import DistributedSystem
@ -18,3 +19,17 @@ def distributed_system(configuration: Configuration) -> DistributedSystem:
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
return DistributedSystem(repository_id, configuration)
@pytest.fixture
def workers_cache(configuration: Configuration) -> WorkersCache:
"""
workers cache fixture
Args:
configuration(Configuration): configuration fixture
Returns:
WorkersCache: workers cache test instance
"""
return WorkersCache(configuration)

130
tests/ahriman/core/distributed/test_distributed_system.py
View File

@ -8,15 +8,6 @@ from ahriman.core.distributed.distributed_system import DistributedSystem
from ahriman.models.worker import Worker
def test_identifier_path(configuration: Configuration) -> None:
"""
must correctly set default identifier path
"""
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
assert DistributedSystem(repository_id, configuration).identifier_path
def test_configuration_sections(configuration: Configuration) -> None:
"""
must correctly parse target list
@ -31,138 +22,31 @@ def test_workers_url(distributed_system: DistributedSystem) -> None:
assert distributed_system._workers_url().startswith(distributed_system.address)
assert distributed_system._workers_url().endswith("/api/v1/distributed")
assert distributed_system._workers_url("id").startswith(distributed_system.address)
assert distributed_system._workers_url("id").endswith("/api/v1/distributed/id")
def test_load_identifier(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must generate identifier
"""
mocker.patch("pathlib.Path.is_file", return_value=False)
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
system = DistributedSystem(repository_id, configuration)
assert system.load_identifier(configuration, "worker")
def test_load_identifier_configuration(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load identifier from configuration
"""
identifier = "id"
mocker.patch("pathlib.Path.is_file", return_value=False)
configuration.set_option("worker", "identifier", identifier)
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
system = DistributedSystem(repository_id, configuration)
assert system.worker.identifier == identifier
def test_load_identifier_filesystem(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load identifier from filesystem
"""
identifier = "id"
mocker.patch("pathlib.Path.is_file", return_value=True)
read_mock = mocker.patch("pathlib.Path.read_text", return_value=identifier)
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
system = DistributedSystem(repository_id, configuration)
assert system.worker.identifier == identifier
read_mock.assert_called_once_with(encoding="utf8")
def test_register(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must register service
"""
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
mocker.patch("pathlib.Path.is_file", return_value=False)
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
write_mock = mocker.patch("pathlib.Path.write_text")
distributed_system.register()
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
json=distributed_system.worker.view())
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
assert distributed_system._owe_identifier
def test_register_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
def test_register_failed(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must skip service registration if it doesn't owe the identifier
must suppress any exception happened during worker registration
"""
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
mocker.patch("pathlib.Path.is_file", return_value=True)
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
write_mock = mocker.patch("pathlib.Path.write_text")
mocker.patch("requests.Session.request", side_effect=Exception())
distributed_system.register()
run_mock.assert_not_called()
write_mock.assert_not_called()
assert not distributed_system._owe_identifier
def test_register_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
def test_register_failed_http_error(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must register service even if it doesn't owe the identifier if force is supplied
must suppress HTTP exception happened during worker registration
"""
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
mocker.patch("pathlib.Path.is_file", return_value=True)
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
write_mock = mocker.patch("pathlib.Path.write_text")
distributed_system.register(force=True)
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
json=distributed_system.worker.view())
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
assert distributed_system._owe_identifier
def test_unregister(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must unregister service
"""
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
remove_mock = mocker.patch("pathlib.Path.unlink")
distributed_system._owe_identifier = True
distributed_system.unregister()
run_mock.assert_called_once_with(
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
remove_mock.assert_called_once_with(missing_ok=True)
def test_unregister_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must skip service removal if it doesn't owe the identifier
"""
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
remove_mock = mocker.patch("pathlib.Path.unlink")
distributed_system.unregister()
run_mock.assert_not_called()
remove_mock.assert_not_called()
def test_unregister_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
"""
must remove service even if it doesn't owe the identifier if force is supplied
"""
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
remove_mock = mocker.patch("pathlib.Path.unlink")
distributed_system.unregister(force=True)
run_mock.assert_called_once_with(
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
remove_mock.assert_called_once_with(missing_ok=True)
mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
distributed_system.register()
def test_workers(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:

17
tests/ahriman/core/distributed/test_worker_register_trigger.py
View File

@ -1,17 +0,0 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.distributed import WorkerRegisterTrigger
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must register itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerRegisterTrigger.register")
_, repository_id = configuration.check_loaded()
trigger = WorkerRegisterTrigger(repository_id, configuration)
trigger.on_start()
run_mock.assert_called_once_with(force=True)

34
tests/ahriman/core/distributed/test_worker_trigger.py
View File

@ -9,11 +9,10 @@ def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
must register itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register")
run_mock = mocker.patch("threading.Timer.start")
_, repository_id = configuration.check_loaded()
trigger = WorkerTrigger(repository_id, configuration)
trigger.on_start()
WorkerTrigger(repository_id, configuration).on_start()
run_mock.assert_called_once_with()
@ -22,9 +21,32 @@ def test_on_stop(configuration: Configuration, mocker: MockerFixture) -> None:
must unregister itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.unregister")
run_mock = mocker.patch("threading.Timer.cancel")
_, repository_id = configuration.check_loaded()
trigger = WorkerTrigger(repository_id, configuration)
trigger.on_stop()
WorkerTrigger(repository_id, configuration).on_stop()
run_mock.assert_called_once_with()
def test_on_stop_empty_timer(configuration: Configuration) -> None:
"""
must do not fail if no timer was started
"""
configuration.set_option("status", "address", "http://localhost:8081")
_, repository_id = configuration.check_loaded()
WorkerTrigger(repository_id, configuration).on_stop()
def test_ping(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must correctly process timer action
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register")
timer_mock = mocker.patch("threading.Timer.start")
_, repository_id = configuration.check_loaded()
WorkerTrigger(repository_id, configuration).ping()
run_mock.assert_called_once_with()
timer_mock.assert_called_once_with()

17
tests/ahriman/core/distributed/test_worker_unregister_trigger.py
View File

@ -1,17 +0,0 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.distributed import WorkerUnregisterTrigger
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must unregister itself as worker
"""
configuration.set_option("status", "address", "http://localhost:8081")
run_mock = mocker.patch("ahriman.core.distributed.WorkerUnregisterTrigger.unregister")
_, repository_id = configuration.check_loaded()
trigger = WorkerUnregisterTrigger(repository_id, configuration)
trigger.on_start()
run_mock.assert_called_once_with(force=True)

43
tests/ahriman/core/distributed/test_workers_cache.py Normal file
View File

@ -0,0 +1,43 @@
import time
from ahriman.core.distributed import WorkersCache
from ahriman.models.worker import Worker
def test_workers(workers_cache: WorkersCache) -> None:
"""
must return alive workers
"""
workers_cache._workers = {
str(index): (Worker(f"address{index}"), index)
for index in range(2)
}
workers_cache.time_to_live = time.monotonic()
assert workers_cache.workers == [Worker("address1")]
def test_workers_remove(workers_cache: WorkersCache) -> None:
"""
must remove all workers
"""
workers_cache.workers_update(Worker("address"))
assert workers_cache.workers
workers_cache.workers_remove()
assert not workers_cache.workers
def test_workers_update(workers_cache: WorkersCache) -> None:
"""
must update worker
"""
worker = Worker("address")
workers_cache.workers_update(worker)
assert workers_cache.workers == [worker]
_, first_last_seen = workers_cache._workers[worker.identifier]
workers_cache.workers_update(worker)
_, second_last_seen = workers_cache._workers[worker.identifier]
assert first_last_seen < second_last_seen

21
tests/ahriman/core/log/test_filtered_access_logger.py
View File

@ -4,7 +4,26 @@ from unittest.mock import MagicMock
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
def is_info_get() -> None:
def test_is_distributed_post() -> None:
"""
must correctly define distributed services ping request
"""
request = MagicMock()
request.method = "POST"
request.path = "/api/v1/distributed"
assert FilteredAccessLogger.is_distributed_post(request)
request.method = "GET"
request.path = "/api/v1/distributed"
assert not FilteredAccessLogger.is_distributed_post(request)
request.method = "POST"
request.path = "/api/v1/distributed/path"
assert not FilteredAccessLogger.is_distributed_post(request)
def test_is_info_get() -> None:
"""
must correctly define health check request
"""

38
tests/ahriman/core/status/test_watcher.py
View File

@ -10,7 +10,6 @@ from ahriman.models.changes import Changes
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.worker import Worker
def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
@ -228,40 +227,3 @@ def test_status_update(watcher: Watcher) -> None:
"""
watcher.status_update(BuildStatusEnum.Success)
assert watcher.status.status == BuildStatusEnum.Success
def test_workers_get(watcher: Watcher, mocker: MockerFixture) -> None:
"""
must retrieve workers
"""
worker = Worker("remote")
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_get", return_value=[worker])
assert watcher.workers_get() == [worker]
worker_mock.assert_called_once_with()
def test_workers_remove(watcher: Watcher, mocker: MockerFixture) -> None:
"""
must remove workers
"""
identifier = "identifier"
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_remove")
watcher.workers_remove(identifier)
watcher.workers_remove()
worker_mock.assert_has_calls([
MockCall(identifier),
MockCall(None),
])
def test_workers_update(watcher: Watcher, mocker: MockerFixture) -> None:
"""
must update workers
"""
worker = Worker("remote")
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_insert")
watcher.workers_update(worker)
worker_mock.assert_called_once_with(worker)

1
tests/ahriman/web/schemas/test_worker_id_schema.py
View File

@ -1 +0,0 @@
# schema testing goes in view class tests

9
tests/ahriman/web/views/test_view_base.py
View File

@ -50,11 +50,18 @@ def test_spawn(base: BaseView) -> None:
def test_validator(base: BaseView) -> None:
"""
must return service
must return validator service
"""
assert base.validator
def test_workers(base: BaseView) -> None:
"""
must return worker service
"""
assert base.workers
async def test_get_permission(base: BaseView) -> None:
"""
must search for permission attribute in class

70
tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_worker.py
View File

@ -1,70 +0,0 @@
import pytest
from aiohttp.test_utils import TestClient
from ahriman.models.user_access import UserAccess
from ahriman.models.worker import Worker
from ahriman.web.views.v1.distributed.worker import WorkerView
async def test_get_permission() -> None:
"""
must return correct permission for the request
"""
for method in ("DELETE", "GET"):
request = pytest.helpers.request("", "", method)
assert await WorkerView.get_permission(request) == UserAccess.Full
def test_routes() -> None:
"""
must return correct routes
"""
assert WorkerView.ROUTES == ["/api/v1/distributed/{identifier}"]
async def test_delete(client: TestClient) -> None:
"""
must delete single worker
"""
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
response = await client.delete("/api/v1/distributed/1")
assert response.status == 204
response = await client.get("/api/v1/distributed/1")
assert response.status == 404
response = await client.get("/api/v1/distributed/2")
assert response.ok
async def test_get(client: TestClient) -> None:
"""
must return specific worker
"""
worker = Worker("address1", identifier="1")
await client.post("/api/v1/distributed", json=worker.view())
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
response_schema = pytest.helpers.schema_response(WorkerView.get)
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
assert response.ok
json = await response.json()
assert not response_schema.validate(json, many=True)
workers = [Worker(item["address"], identifier=item["identifier"]) for item in json]
assert workers == [worker]
async def test_get_not_found(client: TestClient) -> None:
"""
must return Not Found for unknown package
"""
response_schema = pytest.helpers.schema_response(WorkerView.get, code=404)
response = await client.get("/api/v1/distributed/1")
assert response.status == 404
assert not response_schema.validate(await response.json())

3
tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_workers.py
View File

@ -68,9 +68,6 @@ async def test_post(client: TestClient) -> None:
response = await client.post("/api/v1/distributed", json=payload)
assert response.status == 204
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
assert response.ok
async def test_post_exception(client: TestClient) -> None:
"""