make _call method of handlers public and also simplify process spawn

This commit is contained in:
Evgenii Alekseev 2021-09-09 00:35:07 +03:00
parent ea421712a6
commit cafa76f680
4 changed files with 33 additions and 42 deletions

View File

@ -40,7 +40,7 @@ class Handler:
ALLOW_MULTI_ARCHITECTURE_RUN = True ALLOW_MULTI_ARCHITECTURE_RUN = True
@classmethod @classmethod
def _call(cls: Type[Handler], args: argparse.Namespace, architecture: str) -> bool: def call(cls: Type[Handler], args: argparse.Namespace, architecture: str) -> bool:
""" """
additional function to wrap all calls for multiprocessing library additional function to wrap all calls for multiprocessing library
:param args: command line args :param args: command line args
@ -72,9 +72,9 @@ class Handler:
with Pool(len(architectures)) as pool: with Pool(len(architectures)) as pool:
result = pool.starmap( result = pool.starmap(
cls._call, [(args, architecture) for architecture in architectures]) cls.call, [(args, architecture) for architecture in architectures])
else: else:
result = [cls._call(args, architectures.pop())] result = [cls.call(args, architectures.pop())]
return 0 if all(result) else 1 return 0 if all(result) else 1

View File

@ -25,7 +25,7 @@ import uuid
from multiprocessing import Process, Queue from multiprocessing import Process, Queue
from threading import Lock, Thread from threading import Lock, Thread
from typing import Callable, Dict, Iterable, Optional, Tuple from typing import Callable, Dict, Iterable, Tuple
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
@ -57,27 +57,21 @@ class Spawn(Thread):
self.lock = Lock() self.lock = Lock()
self.active: Dict[str, Process] = {} self.active: Dict[str, Process] = {}
# stupid pylint does not know that it is possible # stupid pylint does not know that it is possible
self.queue: Queue[Tuple[str, Optional[Exception]]] = Queue() # pylint: disable=unsubscriptable-object self.queue: Queue[Tuple[str, bool]] = Queue() # pylint: disable=unsubscriptable-object
@staticmethod @staticmethod
def process(callback: Callable[[argparse.Namespace, str, Configuration, bool], None], def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
args: argparse.Namespace, architecture: str, configuration: Configuration, process_id: str, queue: Queue[Tuple[str, bool]]) -> None: # pylint: disable=unsubscriptable-object
process_id: str, queue: Queue[Tuple[str, Optional[Exception]]]) -> None: # pylint: disable=unsubscriptable-object
""" """
helper to run external process helper to run external process
:param callback: application run function (i.e. Handler.run method) :param callback: application run function (i.e. Handler.run method)
:param args: command line arguments :param args: command line arguments
:param architecture: repository architecture :param architecture: repository architecture
:param configuration: configuration instance
:param process_id: process unique identifier :param process_id: process unique identifier
:param queue: output queue :param queue: output queue
""" """
try: result = callback(args, architecture)
callback(args, architecture, configuration, args.no_report) queue.put((process_id, result))
error = None
except Exception as e:
error = e
queue.put((process_id, error))
def packages_add(self, packages: Iterable[str], now: bool) -> None: def packages_add(self, packages: Iterable[str], now: bool) -> None:
""" """
@ -121,12 +115,14 @@ class Spawn(Thread):
arguments.append(f"--{argument}") arguments.append(f"--{argument}")
if value: if value:
arguments.append(value) arguments.append(value)
process_id = str(uuid.uuid4())
self.logger.info("full command line arguments of %s are %s", process_id, arguments)
parsed = self.args_parser.parse_args(arguments) parsed = self.args_parser.parse_args(arguments)
callback = parsed.handler.run callback = parsed.handler.call
process_id = str(uuid.uuid4())
process = Process(target=self.process, process = Process(target=self.process,
args=(callback, parsed, self.architecture, self.configuration, process_id, self.queue), args=(callback, parsed, self.architecture, process_id, self.queue),
daemon=True) daemon=True)
process.start() process.start()
@ -137,11 +133,8 @@ class Spawn(Thread):
""" """
thread run method thread run method
""" """
for process_id, error in iter(self.queue.get, None): for process_id, status in iter(self.queue.get, None):
if error is None: self.logger.info("process %s has been terminated with status %s", process_id, status)
self.logger.info("process %s has been terminated successfully", process_id)
else:
self.logger.exception("process %s has been terminated with exception %s", process_id, error)
with self.lock: with self.lock:
process = self.active.pop(process_id, None) process = self.active.pop(process_id, None)

View File

@ -20,7 +20,7 @@ def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None:
enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__") enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__")
exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__") exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__")
assert Handler._call(args, "x86_64") assert Handler.call(args, "x86_64")
enter_mock.assert_called_once() enter_mock.assert_called_once()
exit_mock.assert_called_once() exit_mock.assert_called_once()
@ -30,7 +30,7 @@ def test_call_exception(args: argparse.Namespace, mocker: MockerFixture) -> None
must process exception must process exception
""" """
mocker.patch("ahriman.application.lock.Lock.__enter__", side_effect=Exception()) mocker.patch("ahriman.application.lock.Lock.__enter__", side_effect=Exception())
assert not Handler._call(args, "x86_64") assert not Handler.call(args, "x86_64")
def test_execute(args: argparse.Namespace, mocker: MockerFixture) -> None: def test_execute(args: argparse.Namespace, mocker: MockerFixture) -> None:

View File

@ -9,15 +9,15 @@ def test_process(spawner: Spawn) -> None:
must process external process run correctly must process external process run correctly
""" """
args = MagicMock() args = MagicMock()
args.no_report = False
callback = MagicMock() callback = MagicMock()
callback.return_value = True
spawner.process(callback, args, spawner.architecture, spawner.configuration, "id", spawner.queue) spawner.process(callback, args, spawner.architecture, "id", spawner.queue)
callback.assert_called_with(args, spawner.architecture, spawner.configuration, False) callback.assert_called_with(args, spawner.architecture)
(uuid, error) = spawner.queue.get() (uuid, status) = spawner.queue.get()
assert uuid == "id" assert uuid == "id"
assert error is None assert status
assert spawner.queue.empty() assert spawner.queue.empty()
@ -26,13 +26,13 @@ def test_process_error(spawner: Spawn) -> None:
must process external run with error correctly must process external run with error correctly
""" """
callback = MagicMock() callback = MagicMock()
callback.side_effect = Exception() callback.return_value = False
spawner.process(callback, MagicMock(), spawner.architecture, spawner.configuration, "id", spawner.queue) spawner.process(callback, MagicMock(), spawner.architecture, "id", spawner.queue)
(uuid, error) = spawner.queue.get() (uuid, status) = spawner.queue.get()
assert uuid == "id" assert uuid == "id"
assert isinstance(error, Exception) assert not status
assert spawner.queue.empty() assert spawner.queue.empty()
@ -90,16 +90,14 @@ def test_run(spawner: Spawn, mocker: MockerFixture) -> None:
""" """
must implement run method must implement run method
""" """
logging_exception_mock = mocker.patch("logging.Logger.exception") logging_mock = mocker.patch("logging.Logger.info")
logging_info_mock = mocker.patch("logging.Logger.info")
spawner.queue.put(("1", None)) spawner.queue.put(("1", False))
spawner.queue.put(("2", Exception())) spawner.queue.put(("2", True))
spawner.queue.put(None) # terminate spawner.queue.put(None) # terminate
spawner.run() spawner.run()
logging_exception_mock.assert_called_once() logging_mock.assert_called()
logging_info_mock.assert_called_once()
def test_run_pop(spawner: Spawn) -> None: def test_run_pop(spawner: Spawn) -> None:
@ -109,8 +107,8 @@ def test_run_pop(spawner: Spawn) -> None:
first = spawner.active["1"] = MagicMock() first = spawner.active["1"] = MagicMock()
second = spawner.active["2"] = MagicMock() second = spawner.active["2"] = MagicMock()
spawner.queue.put(("1", None)) spawner.queue.put(("1", False))
spawner.queue.put(("2", Exception())) spawner.queue.put(("2", True))
spawner.queue.put(None) # terminate spawner.queue.put(None) # terminate
spawner.run() spawner.run()