Release 0.15.0

* load and save web service state to cache file
* disable web reporting to self
* restore console handler settings
* allow to redirect logs to stderr
* verbose http error logging
* update package status by group, not by single package
* split Repository class to several traits
* move json generators/readers to dataclasses

.pylintrc

@@ -0,0 +1,608 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Specify a score threshold to be exceeded before program exits with error.
+fail-under=10.0
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python module names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        missing-module-docstring,
+        line-too-long,
+        no-name-in-module,
+        import-outside-toplevel,
+        invalid-name,
+        raise-missing-from,
+        wrong-import-order,
+        too-few-public-methods,
+        too-many-instance-attributes,
+        broad-except,
+        logging-fstring-interpolation,
+        too-many-ancestors,
+        fixme,
+        too-many-arguments,
+        duplicate-code,
+        cyclic-import
+
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a score less than or equal to 10. You
+# have access to the variables 'error', 'warning', 'refactor', and 'convention'
+# which contain the number of messages in each category, as well as 'statement'
+# which is the total number of statements analyzed. This score is used by the
+# global evaluation report (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Bad variable names regexes, separated by a comma. If names match any regex,
+# they will always be refused
+bad-names-rgxs=
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _
+
+# Good variable names regexes, separated by a comma. If names match any regex,
+# they will always be accepted
+good-names-rgxs=
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis). It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+# List of decorators that change the signature of a decorated function.
+signature-mutators=
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[LOGGING]
+
+# The type of string formatting that logging methods do. `old` means using %
+# formatting, `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+# Regular expression of note tags to take in consideration.
+#notes-rgx=
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it work,
+# install the python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains the private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to the private dictionary (see the
+# --spelling-private-dict-file option) instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=no
+
+# This flag controls whether the implicit-str-concat should generate a warning
+# on implicit string concatenation in sequences defined over several lines.
+check-str-concat-over-line-jumps=no
+
+
+[IMPORTS]
+
+# List of modules that can be imported at any level, not just the top level
+# one.
+allow-any-import-level=
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+# Couples of modules and preferred modules, separated by a comma.
+preferred-modules=
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement (see R0916).
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# Warn about protected attribute access inside special methods
+check-protected-access-in-special-methods=no
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp,
+                      __post_init__
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "BaseException, Exception".
+overgeneral-exceptions=BaseException,
+                       Exception

Makefile

@@ -1,4 +1,4 @@
-.PHONY: archive archive_directory archlinux clean directory push version
+.PHONY: archive archive_directory archlinux check clean directory push version
 .DEFAULT_GOAL := archlinux
 
 PROJECT := ahriman
@@ -7,10 +7,6 @@ FILES := COPYING CONFIGURING.md README.md package src setup.py
 TARGET_FILES := $(addprefix $(PROJECT)/, $(FILES))
 IGNORE_FILES := package/archlinux src/.mypy_cache
 
-ifndef VERSION
-$(error VERSION is not set)
-endif
-
 $(TARGET_FILES) : $(addprefix $(PROJECT), %) : $(addprefix ., %) directory version
 	@cp -rp $< $@
 
@@ -28,6 +24,11 @@ archlinux: archive
 	sed -i "/sha512sums=('[0-9A-Fa-f]*/s/[^'][^)]*/sha512sums=('$$(sha512sum $(PROJECT)-$(VERSION)-src.tar.xz | awk '{print $$1}')'/" package/archlinux/PKGBUILD
 	sed -i "s/pkgver=[0-9.]*/pkgver=$(VERSION)/" package/archlinux/PKGBUILD
 
+check:
+	cd src && mypy --strict -p $(PROJECT)
+	cd src && find $(PROJECT) -name '*.py' -execdir autopep8 --max-line-length 120 -aa -i {} +
+	cd src && pylint --rcfile=../.pylintrc $(PROJECT)
+
 clean:
 	find . -type f -name '$(PROJECT)-*-src.tar.xz' -delete
 	rm -rf "$(PROJECT)"
@@ -43,4 +44,7 @@ push: archlinux
 	git push --tags
 
 version:
+ifndef VERSION
+	$(error VERSION is required, but not set)
+endif
 	sed -i "/__version__ = '[0-9.]*/s/[^'][^)]*/__version__ = '$(VERSION)'/" src/ahriman/version.py

package/archlinux/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=0.14.1
+pkgver=0.15.0
 pkgrel=1
 pkgdesc="ArcHlinux ReposItory MANager"
 arch=('any')
@@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
         'ahriman.sysusers'
         'ahriman.tmpfiles')
-sha512sums=('54286cfd1c9b03e7adfa639b976ace233e4e3ea8d2a2cbd11c22fc43eda60906e1d3b795e1505b40e41171948ba95d6591a4f7c328146200f4622a8ed657e8a5'
+sha512sums=('a1db44390ce1785da3d535e3cfd2242d8d56070228eb9b3c1d5629163b65941d60753c481c0fdc69e475e534a828ceea39568dc6711abeee092616dac08e31a9'
             '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
             '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
 backup=('etc/ahriman.ini'

package/etc/ahriman.ini.d/logging.ini

@@ -2,11 +2,17 @@
 keys = root,builder,build_details,http
 
 [handlers]
-keys = build_file_handler,file_handler,http_handler
+keys = console_handler,build_file_handler,file_handler,http_handler
 
 [formatters]
 keys = generic_format
 
+[handler_console_handler]
+class = StreamHandler
+level = DEBUG
+formatter = generic_format
+args = (sys.stderr,)
+
 [handler_file_handler]
 class = logging.handlers.RotatingFileHandler
 level = DEBUG
@@ -26,7 +32,7 @@ formatter = generic_format
 args = ('/var/log/ahriman/http.log', 'a', 20971520, 20)
 
 [formatter_generic_format]
-format = %(asctime)s : %(levelname)s : %(funcName)s : %(message)s
+format = [%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s
 datefmt =
 
 [logger_root]

src/ahriman/application/ahriman.py

@@ -39,7 +39,7 @@ def _call(args: argparse.Namespace, architecture: str, config: Configuration) ->
     :return: True on success, False otherwise
     '''
     try:
-        with Lock(args.lock, architecture, args.force, args.unsafe, config):
+        with Lock(args, architecture, config):
             args.fn(args, architecture, config)
         return True
     except Exception:
@@ -75,8 +75,9 @@ def dump_config(args: argparse.Namespace, architecture: str, config: Configurati
     :param architecture: repository architecture
     :param config: configuration instance
     '''
-    result = config.dump(architecture)
-    for section, values in sorted(result.items()):
+    del args
+    config_dump = config.dump(architecture)
+    for section, values in sorted(config_dump.items()):
         print(f'[{section}]')
         for key, value in sorted(values.items()):
             print(f'{key} = {value}')
@@ -90,6 +91,7 @@ def rebuild(args: argparse.Namespace, architecture: str, config: Configuration)
     :param architecture: repository architecture
     :param config: configuration instance
     '''
+    del args
     app = Application(architecture, config)
     packages = app.repository.packages()
     app.update(packages)
@@ -151,6 +153,7 @@ def web(args: argparse.Namespace, architecture: str, config: Configuration) -> N
     :param architecture: repository architecture
     :param config: configuration instance
     '''
+    del args
     from ahriman.web.web import run_server, setup_service
     application = setup_service(architecture, config)
     run_server(application, architecture)
@@ -166,6 +169,8 @@ if __name__ == '__main__':
     parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini')
     parser.add_argument('--force', help='force run, remove file lock', action='store_true')
     parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
+    parser.add_argument('--no-log', help='redirect all log messages to stderr', action='store_true')
+    parser.add_argument('--no-report', help='force disable reporting to web service', action='store_true')
     parser.add_argument('--unsafe', help='allow to run ahriman as non-ahriman user', action='store_true')
     parser.add_argument('-v', '--version', action='version', version=version.__version__)
     subparsers = parser.add_subparsers(title='command')
@@ -175,7 +180,7 @@ if __name__ == '__main__':
     add_parser.add_argument('--without-dependencies', help='do not add dependencies', action='store_true')
     add_parser.set_defaults(fn=add)
 
-    check_parser = subparsers.add_parser('check', description='check for updates')
+    check_parser = subparsers.add_parser('check', description='check for updates. Same as update --dry-run --no-manual')
     check_parser.add_argument('package', help='filter check by packages', nargs='*')
     check_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
     check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=True)
@@ -213,21 +218,22 @@ if __name__ == '__main__':
     update_parser.add_argument('package', help='filter check by packages', nargs='*')
     update_parser.add_argument(
         '--dry-run', help='just perform check for updates, same as check command', action='store_true')
-    update_parser.add_argument('--no-aur', help='do not check for AUR updates', action='store_true')
+    update_parser.add_argument('--no-aur', help='do not check for AUR updates. Implies --no-vcs', action='store_true')
     update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true')
     update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
     update_parser.set_defaults(fn=update)
 
     web_parser = subparsers.add_parser('web', description='start web server')
-    web_parser.set_defaults(fn=web, lock=None)
+    web_parser.set_defaults(fn=web, lock=None, no_report=True)
 
-    args = parser.parse_args()
-    if 'fn' not in args:
+    cmd_args = parser.parse_args()
+    if 'fn' not in cmd_args:
         parser.print_help()
-        exit(1)
+        sys.exit(1)
 
-    config = Configuration.from_path(args.config)
-    with Pool(len(args.architecture)) as pool:
-        result = pool.starmap(_call, [(args, architecture, config) for architecture in args.architecture])
+    configuration = Configuration.from_path(cmd_args.config, not cmd_args.no_log)
+    with Pool(len(cmd_args.architecture)) as pool:
+        result = pool.starmap(
+            _call, [(cmd_args, architecture, configuration) for architecture in cmd_args.architecture])
 
     sys.exit(0 if all(result) else 1)

src/ahriman/application/application.py

@@ -25,7 +25,7 @@ from typing import Callable, Iterable, List, Optional, Set
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
-from ahriman.core.repository import Repository
+from ahriman.repository.repository import Repository
 from ahriman.core.tree import Tree
 from ahriman.models.package import Package
 
@@ -136,15 +136,15 @@ class Application:
         :param no_packages: do not clear directory with built packages
         '''
         if not no_build:
-            self.repository._clear_build()
+            self.repository.clear_build()
         if not no_cache:
-            self.repository._clear_cache()
+            self.repository.clear_cache()
         if not no_chroot:
-            self.repository._clear_chroot()
+            self.repository.clear_chroot()
         if not no_manual:
-            self.repository._clear_manual()
+            self.repository.clear_manual()
         if not no_packages:
-            self.repository._clear_packages()
+            self.repository.clear_packages()
 
     def remove(self, names: Iterable[str]) -> None:
         '''

src/ahriman/application/lock.py

@@ -19,6 +19,7 @@
 #
 from __future__ import annotations
 
+import argparse
 import os
 
 from types import TracebackType
@@ -40,22 +41,19 @@ class Lock:
     :ivar unsafe: skip user check
     '''
 
-    def __init__(self, path: Optional[str], architecture: str, force: bool, unsafe: bool,
-                 config: Configuration) -> None:
+    def __init__(self, args: argparse.Namespace, architecture: str, config: Configuration) -> None:
         '''
         default constructor
-        :param path: optional path to lock file, if empty no file lock will be used
+        :param args: command line args
         :param architecture: repository architecture
-        :param force: remove lock file on start if any
-        :param unsafe: skip user check
         :param config: configuration instance
         '''
-        self.path = f'{path}_{architecture}' if path is not None else None
-        self.force = force
-        self.unsafe = unsafe
+        self.path = f'{args.lock}_{architecture}' if args.lock is not None else None
+        self.force = args.force
+        self.unsafe = args.unsafe
 
         self.root = config.get('repository', 'root')
-        self.reporter = Client.load(architecture, config)
+        self.reporter = Client() if args.no_report else Client.load(architecture, config)
 
     def __enter__(self) -> Lock:
         '''

src/ahriman/core/configuration.py

@@ -37,7 +37,7 @@ class Configuration(configparser.RawConfigParser):
     :cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump)
     '''
 
-    DEFAULT_LOG_FORMAT = '%(asctime)s : %(levelname)s : %(funcName)s : %(message)s'
+    DEFAULT_LOG_FORMAT = '[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s'
     DEFAULT_LOG_LEVEL = logging.DEBUG
 
     STATIC_SECTIONS = ['alpm', 'report', 'repository', 'settings', 'upload']
@@ -45,7 +45,7 @@ class Configuration(configparser.RawConfigParser):
 
     def __init__(self) -> None:
         '''
-        default constructor
+        default constructor. In the most cases must not be called directly
         '''
         configparser.RawConfigParser.__init__(self, allow_no_value=True)
         self.path: Optional[str] = None
@@ -58,15 +58,16 @@ class Configuration(configparser.RawConfigParser):
         return self.get('settings', 'include')
 
     @classmethod
-    def from_path(cls: Type[Configuration], path: str) -> Configuration:
+    def from_path(cls: Type[Configuration], path: str, logfile: bool) -> Configuration:
         '''
         constructor with full object initialization
         :param path: path to root configuration file
+        :param logfile: use log file to output messages
         :return: configuration instance
         '''
         config = cls()
         config.load(path)
-        config.load_logging()
+        config.load_logging(logfile)
         return config
 
     def dump(self, architecture: str) -> Dict[str, Dict[str, str]]:
@@ -129,13 +130,23 @@ class Configuration(configparser.RawConfigParser):
         except (FileNotFoundError, configparser.NoOptionError):
             pass
 
-    def load_logging(self) -> None:
+    def load_logging(self, logfile: bool) -> None:
         '''
         setup logging settings from configuration
+        :param logfile: use log file to output messages
         '''
-        try:
-            fileConfig(self.get('settings', 'logging'))
-        except PermissionError:
+        def file_logger() -> None:
+            try:
+                fileConfig(self.get('settings', 'logging'))
+            except PermissionError:
+                console_logger()
+                logging.error('could not create logfile, fallback to stderr', exc_info=True)
+
+        def console_logger() -> None:
             logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT,
                                 level=Configuration.DEFAULT_LOG_LEVEL)
-            logging.error('could not create logfile, fallback to stderr', exc_info=True)
+
+        if logfile:
+            file_logger()
+        else:
+            console_logger()

src/ahriman/core/report/report.py

@@ -72,4 +72,3 @@ class Report:
         generate report for the specified packages
         :param packages: list of packages to generate report
         '''
-        pass

src/ahriman/core/repository.py

@@ -1,281 +0,0 @@
-#
-# Copyright (c) 2021 Evgenii Alekseev.
-#
-# This file is part of ahriman
-# (see https://github.com/arcan1s/ahriman).
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import logging
-import os
-import shutil
-
-from typing import Dict, Iterable, List, Optional
-
-from ahriman.core.alpm.pacman import Pacman
-from ahriman.core.alpm.repo import Repo
-from ahriman.core.build_tools.task import Task
-from ahriman.core.configuration import Configuration
-from ahriman.core.report.report import Report
-from ahriman.core.sign.gpg import GPG
-from ahriman.core.upload.uploader import Uploader
-from ahriman.core.util import package_like
-from ahriman.core.watcher.client import Client
-from ahriman.models.package import Package
-from ahriman.models.repository_paths import RepositoryPaths
-
-
-class Repository:
-    '''
-    base repository control class
-    :ivar architecture: repository architecture
-    :ivar aur_url: base AUR url
-    :ivar config: configuration instance
-    :ivar logger: class logger
-    :ivar name: repository name
-    :ivar pacman: alpm wrapper instance
-    :ivar paths: repository paths instance
-    :ivar repo: repo commands wrapper instance
-    :ivar reporter: build status reporter instance
-    :ivar sign: GPG wrapper instance
-    '''
-
-    def __init__(self, architecture: str, config: Configuration) -> None:
-        '''
-        default constructor
-        :param architecture: repository architecture
-        :param config: configuration instance
-        '''
-        self.logger = logging.getLogger('builder')
-        self.architecture = architecture
-        self.config = config
-
-        self.aur_url = config.get('alpm', 'aur_url')
-        self.name = config.get('repository', 'name')
-
-        self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
-        self.paths.create_tree()
-
-        self.pacman = Pacman(config)
-        self.sign = GPG(architecture, config)
-        self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
-        self.reporter = Client.load(architecture, config)
-
-    def _clear_build(self) -> None:
-        '''
-        clear sources directory
-        '''
-        for package in os.listdir(self.paths.sources):
-            shutil.rmtree(os.path.join(self.paths.sources, package))
-
-    def _clear_cache(self) -> None:
-        '''
-        clear cache directory
-        '''
-        for package in os.listdir(self.paths.cache):
-            shutil.rmtree(os.path.join(self.paths.cache, package))
-
-    def _clear_chroot(self) -> None:
-        '''
-        clear cache directory. Warning: this method is architecture independent and will clear every chroot
-        '''
-        for chroot in os.listdir(self.paths.chroot):
-            shutil.rmtree(os.path.join(self.paths.chroot, chroot))
-
-    def _clear_manual(self) -> None:
-        '''
-        clear directory with manual package updates
-        '''
-        for package in os.listdir(self.paths.manual):
-            shutil.rmtree(os.path.join(self.paths.manual, package))
-
-    def _clear_packages(self) -> None:
-        '''
-        clear directory with built packages (NOT repository itself)
-        '''
-        for package in self.packages_built():
-            os.remove(package)
-
-    def packages(self) -> List[Package]:
-        '''
-        generate list of repository packages
-        :return: list of packages properties
-        '''
-        result: Dict[str, Package] = {}
-        for fn in os.listdir(self.paths.repository):
-            if not package_like(fn):
-                continue
-            full_path = os.path.join(self.paths.repository, fn)
-            try:
-                local = Package.load(full_path, self.pacman, self.aur_url)
-                result.setdefault(local.base, local).packages.update(local.packages)
-            except Exception:
-                self.logger.exception(f'could not load package from {fn}', exc_info=True)
-                continue
-        return list(result.values())
-
-    def packages_built(self) -> List[str]:
-        '''
-        get list of files in built packages directory
-        :return: list of filenames from the directory
-        '''
-        return [
-            os.path.join(self.paths.packages, fn)
-            for fn in os.listdir(self.paths.packages)
-        ]
-
-    def process_build(self, updates: Iterable[Package]) -> List[str]:
-        '''
-        build packages
-        :param updates: list of packages properties to build
-        :return: `packages_built`
-        '''
-        def build_single(package: Package) -> None:
-            self.reporter.set_building(package.base)
-            task = Task(package, self.architecture, self.config, self.paths)
-            task.init()
-            built = task.build()
-            for src in built:
-                dst = os.path.join(self.paths.packages, os.path.basename(src))
-                shutil.move(src, dst)
-
-        for package in updates:
-            try:
-                build_single(package)
-            except Exception:
-                self.reporter.set_failed(package.base)
-                self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
-                continue
-        self._clear_build()
-
-        return self.packages_built()
-
-    def process_remove(self, packages: Iterable[str]) -> str:
-        '''
-        remove packages from list
-        :param packages: list of package names or bases to rmeove
-        :return: path to repository database
-        '''
-        def remove_single(package: str) -> None:
-            try:
-                self.repo.remove(package)
-            except Exception:
-                self.logger.exception(f'could not remove {package}', exc_info=True)
-
-        requested = set(packages)
-        for local in self.packages():
-            if local.base in packages:
-                to_remove = set(local.packages.keys())
-                self.reporter.remove(local.base)  # we only update status page in case of base removal
-            elif requested.intersection(local.packages.keys()):
-                to_remove = requested.intersection(local.packages.keys())
-            else:
-                to_remove = set()
-            for package in to_remove:
-                remove_single(package)
-
-        return self.repo.repo_path
-
-    def process_report(self, targets: Optional[Iterable[str]]) -> None:
-        '''
-        generate reports
-        :param targets: list of targets to generate reports. Configuration option will be used if it is not set
-        '''
-        if targets is None:
-            targets = self.config.getlist('report', 'target')
-        for target in targets:
-            Report.run(self.architecture, self.config, target, self.packages())
-
-    def process_sync(self, targets: Optional[Iterable[str]]) -> None:
-        '''
-        process synchronization to remote servers
-        :param targets: list of targets to sync. Configuration option will be used if it is not set
-        '''
-        if targets is None:
-            targets = self.config.getlist('upload', 'target')
-        for target in targets:
-            Uploader.run(self.architecture, self.config, target, self.paths.repository)
-
-    def process_update(self, packages: Iterable[str]) -> str:
-        '''
-        sign packages, add them to repository and update repository database
-        :param packages: list of filenames to run
-        :return: path to repository database
-        '''
-        for package in packages:
-            local = Package.load(package, self.pacman, self.aur_url)  # we will use it for status reports
-            try:
-                files = self.sign.sign_package(package, local.base)
-                for src in files:
-                    dst = os.path.join(self.paths.repository, os.path.basename(src))
-                    shutil.move(src, dst)
-                package_fn = os.path.join(self.paths.repository, os.path.basename(package))
-                self.repo.add(package_fn)
-                self.reporter.set_success(local)
-            except Exception:
-                self.logger.exception(f'could not process {package}', exc_info=True)
-                self.reporter.set_failed(local.base)
-        self._clear_packages()
-
-        return self.repo.repo_path
-
-    def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
-        '''
-        check AUR for updates
-        :param filter_packages: do not check every package just specified in the list
-        :param no_vcs: do not check VCS packages
-        :return: list of packages which are out-of-dated
-        '''
-        result: List[Package] = []
-
-        build_section = self.config.get_section_name('build', self.architecture)
-        ignore_list = self.config.getlist(build_section, 'ignore_packages')
-
-        for local in self.packages():
-            if local.base in ignore_list:
-                continue
-            if local.is_vcs and no_vcs:
-                continue
-            if filter_packages and local.base not in filter_packages:
-                continue
-
-            try:
-                remote = Package.load(local.base, self.pacman, self.aur_url)
-                if local.is_outdated(remote, self.paths):
-                    result.append(remote)
-                    self.reporter.set_pending(local.base)
-            except Exception:
-                self.reporter.set_failed(local.base)
-                self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
-                continue
-
-        return result
-
-    def updates_manual(self) -> List[Package]:
-        '''
-        check for packages for which manual update has been requested
-        :return: list of packages which are out-of-dated
-        '''
-        result: List[Package] = []
-
-        for fn in os.listdir(self.paths.manual):
-            try:
-                local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
-                result.append(local)
-                self.reporter.set_unknown(local)
-            except Exception:
-                self.logger.exception(f'could not add package from {fn}', exc_info=True)
-        self._clear_manual()
-
-        return result

src/ahriman/core/sign/gpg.py

@@ -59,6 +59,16 @@ class GPG:
             return []
         return ['--sign', '--key', self.default_key]
 
+    @staticmethod
+    def sign_cmd(path: str, key: str) -> List[str]:
+        '''
+        gpg command to run
+        :param path: path to file to sign
+        :param key: PGP key ID
+        :return: gpg command with all required arguments
+        '''
+        return ['gpg', '-u', key, '-b', path]
+
     def process(self, path: str, key: str) -> List[str]:
         '''
         gpg command wrapper
@@ -67,21 +77,12 @@ class GPG:
         :return: list of generated files including original file
         '''
         check_output(
-            *self.sign_cmd(path, key),
+            *GPG.sign_cmd(path, key),
             exception=BuildFailed(path),
             cwd=os.path.dirname(path),
             logger=self.logger)
         return [path, f'{path}.sig']
 
-    def sign_cmd(self, path: str, key: str) -> List[str]:
-        '''
-        gpg command to run
-        :param path: path to file to sign
-        :param key: PGP key ID
-        :return: gpg command with all required arguments
-        '''
-        return ['gpg', '-u', key, '-b', path]
-
     def sign_package(self, path: str, base: str) -> List[str]:
         '''
         sign package if required by configuration

src/ahriman/core/tree.py

@@ -92,7 +92,7 @@ class Tree:
         '''
         result: List[List[Package]] = []
 
-        unprocessed = [leaf for leaf in self.leaves]
+        unprocessed = self.leaves[:]
         while unprocessed:
             result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)])
             unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)]

src/ahriman/core/upload/rsync.py

@@ -43,6 +43,6 @@ class Rsync(Uploader):
         sync data to remote server
         :param path: local path to sync
         '''
-        check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote,
+        check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--delete', path, self.remote,
                      exception=None,
                      logger=self.logger)

src/ahriman/core/upload/s3.py

@@ -44,6 +44,6 @@ class S3(Uploader):
         :param path: local path to sync
         '''
         # TODO rewrite to boto, but it is bullshit
-        check_output('aws', 's3', 'sync', '--delete', path, self.bucket,
+        check_output('aws', 's3', 'sync', '--quiet', '--delete', path, self.bucket,
                      exception=None,
                      logger=self.logger)

src/ahriman/core/upload/uploader.py

@@ -72,4 +72,3 @@ class Uploader:
         sync data to remote server
         :param path: local path to sync
         '''
-        pass

src/ahriman/core/util.py

@@ -79,16 +79,16 @@ def pretty_size(size: Optional[float], level: int = 0) -> str:
     def str_level() -> str:
         if level == 0:
             return 'B'
-        elif level == 1:
+        if level == 1:
             return 'KiB'
-        elif level == 2:
+        if level == 2:
             return 'MiB'
-        elif level == 3:
+        if level == 3:
             return 'GiB'
         raise InvalidOption(level)  # I hope it will not be more than 1024 GiB
 
     if size is None:
         return ''
-    elif size < 1024:
+    if size < 1024:
         return f'{round(size, 2)} {str_level()}'
     return pretty_size(size / 1024, level + 1)

src/ahriman/core/watcher/client.py

@@ -35,14 +35,12 @@ class Client:
         :param package: package properties
         :param status: current package build status
         '''
-        pass
 
     def remove(self, base: str) -> None:
         '''
         remove packages from watcher
         :param base: basename to remove
         '''
-        pass
 
     def update(self, base: str, status: BuildStatusEnum) -> None:
         '''
@@ -50,14 +48,12 @@ class Client:
         :param base: package base to update
         :param status: current package build status
         '''
-        pass
 
     def update_self(self, status: BuildStatusEnum) -> None:
         '''
         update ahriman status itself
         :param status: current ahriman status
         '''
-        pass
 
     def set_building(self, base: str) -> None:
         '''

src/ahriman/core/watcher/watcher.py

@@ -17,10 +17,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Dict, List, Optional, Tuple
+import json
+import logging
+import os
+
+from typing import Any, Dict, List, Optional, Tuple
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.repository import Repository
+from ahriman.repository.repository import Repository
 from ahriman.models.build_status import BuildStatus, BuildStatusEnum
 from ahriman.models.package import Package
 
@@ -30,7 +34,9 @@ class Watcher:
     package status watcher
     :ivar architecture: repository architecture
     :ivar known: list of known packages. For the most cases `packages` should be used instead
+    :ivar logger: class logger
     :ivar repository: repository object
+    :ivar status: daemon status
     '''
 
     def __init__(self, architecture: str, config: Configuration) -> None:
@@ -39,18 +45,72 @@ class Watcher:
         :param architecture: repository architecture
         :param config: configuration instance
         '''
+        self.logger = logging.getLogger('http')
+
         self.architecture = architecture
         self.repository = Repository(architecture, config)
 
         self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
         self.status = BuildStatus()
 
+    @property
+    def cache_path(self) -> str:
+        '''
+        :return: path to dump with json cache
+        '''
+        return os.path.join(self.repository.paths.root, 'status_cache.json')
+
     @property
     def packages(self) -> List[Tuple[Package, BuildStatus]]:
         '''
         :return: list of packages together with their statuses
         '''
-        return [pair for pair in self.known.values()]
+        return list(self.known.values())
+
+    def _cache_load(self) -> None:
+        '''
+        update current state from cache
+        '''
+        def parse_single(properties: Dict[str, Any]) -> None:
+            package = Package.from_json(properties['package'])
+            status = BuildStatus(**properties['status'])
+            if package.base in self.known:
+                self.known[package.base] = (package, status)
+
+        if not os.path.isfile(self.cache_path):
+            return
+        with open(self.cache_path) as cache:
+            dump = json.load(cache)
+        for item in dump['packages']:
+            try:
+                parse_single(item)
+            except Exception:
+                self.logger.exception(f'cannot parse item f{item} to package', exc_info=True)
+
+    def _cache_save(self) -> None:
+        '''
+        dump current cache to filesystem
+        '''
+        dump = {
+            'packages': [
+                {
+                    'package': package.view(),
+                    'status': status.view()
+                } for package, status in self.packages
+            ]
+        }
+        try:
+            with open(self.cache_path, 'w') as cache:
+                json.dump(dump, cache)
+        except Exception:
+            self.logger.exception('cannot dump cache', exc_info=True)
+
+    def get(self, base: str) -> Tuple[Package, BuildStatus]:
+        '''
+        get current package base build status
+        :return: package and its status
+        '''
+        return self.known[base]
 
     def load(self) -> None:
         '''
@@ -64,6 +124,7 @@ class Watcher:
             else:
                 _, status = current
             self.known[package.base] = (package, status)
+        self._cache_load()
 
     def remove(self, base: str) -> None:
         '''
@@ -71,6 +132,7 @@ class Watcher:
         :param base: package base
         '''
         self.known.pop(base, None)
+        self._cache_save()
 
     def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
         '''
@@ -83,6 +145,7 @@ class Watcher:
             package, _ = self.known[base]
         full_status = BuildStatus(status)
         self.known[base] = (package, full_status)
+        self._cache_save()
 
     def update_self(self, status: BuildStatusEnum) -> None:
         '''

src/ahriman/core/watcher/web_client.py

@@ -20,9 +20,6 @@
 import logging
 import requests
 
-from dataclasses import asdict
-from typing import Any, Dict
-
 from ahriman.core.watcher.client import Client
 from ahriman.models.build_status import BuildStatusEnum
 from ahriman.models.package import Package
@@ -67,14 +64,16 @@ class WebClient(Client):
         :param package: package properties
         :param status: current package build status
         '''
-        payload: Dict[str, Any] = {
+        payload = {
             'status': status.value,
-            'package': asdict(package)
+            'package': package.view()
         }
 
         try:
             response = requests.post(self._package_url(package.base), json=payload)
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not add {package.base}: {e.response.text}', exc_info=True)
         except Exception:
             self.logger.exception(f'could not add {package.base}', exc_info=True)
 
@@ -86,6 +85,8 @@ class WebClient(Client):
         try:
             response = requests.delete(self._package_url(base))
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not delete {base}: {e.response.text}', exc_info=True)
         except Exception:
             self.logger.exception(f'could not delete {base}', exc_info=True)
 
@@ -95,11 +96,13 @@ class WebClient(Client):
         :param base: package base to update
         :param status: current package build status
         '''
-        payload: Dict[str, Any] = {'status': status.value}
+        payload = {'status': status.value}
 
         try:
             response = requests.post(self._package_url(base), json=payload)
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not update {base}: {e.response.text}', exc_info=True)
         except Exception:
             self.logger.exception(f'could not update {base}', exc_info=True)
 
@@ -108,10 +111,12 @@ class WebClient(Client):
         update ahriman status itself
         :param status: current ahriman status
         '''
-        payload: Dict[str, Any] = {'status': status.value}
+        payload = {'status': status.value}
 
         try:
             response = requests.post(self._ahriman_url(), json=payload)
             response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception(f'could not update service status: {e.response.text}', exc_info=True)
         except Exception:
-            self.logger.exception(f'could not update service status', exc_info=True)
+            self.logger.exception('could not update service status', exc_info=True)

src/ahriman/models/build_status.py

@@ -20,7 +20,7 @@
 import datetime
 
 from enum import Enum
-from typing import Optional, Union
+from typing import Any, Dict, Optional, Union
 
 
 class BuildStatusEnum(Enum):
@@ -46,11 +46,11 @@ class BuildStatusEnum(Enum):
         '''
         if self == BuildStatusEnum.Pending:
             return 'yellow'
-        elif self == BuildStatusEnum.Building:
+        if self == BuildStatusEnum.Building:
             return 'yellow'
-        elif self == BuildStatusEnum.Failed:
+        if self == BuildStatusEnum.Failed:
             return 'critical'
-        elif self == BuildStatusEnum.Success:
+        if self == BuildStatusEnum.Success:
             return 'success'
         return 'inactive'
 
@@ -71,3 +71,13 @@ class BuildStatus:
         '''
         self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown
         self.timestamp = timestamp or int(datetime.datetime.utcnow().timestamp())
+
+    def view(self) -> Dict[str, Any]:
+        '''
+        generate json status view
+        :return: json-friendly dictionary
+        '''
+        return {
+            'status': self.status.value,
+            'timestamp': self.timestamp
+        }

src/ahriman/models/package.py

@@ -19,15 +19,14 @@
 #
 from __future__ import annotations
 
-import logging
-
 import aur  # type: ignore
+import logging
 import os
 
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from pyalpm import vercmp  # type: ignore
 from srcinfo.parse import parse_srcinfo  # type: ignore
-from typing import Dict, List, Optional, Set, Type
+from typing import Any, Dict, List, Optional, Set, Type
 
 from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.exceptions import InvalidPackageInfo
@@ -77,31 +76,6 @@ class Package:
         '''
         return f'{self.aur_url}/packages/{self.base}'
 
-    def actual_version(self, paths: RepositoryPaths) -> str:
-        '''
-        additional method to handle VCS package versions
-        :param paths: repository paths instance
-        :return: package version if package is not VCS and current version according to VCS otherwise
-        '''
-        if not self.is_vcs:
-            return self.version
-
-        from ahriman.core.build_tools.task import Task
-
-        clone_dir = os.path.join(paths.cache, self.base)
-        logger = logging.getLogger('build_details')
-        Task.fetch(clone_dir, self.git_url)
-
-        # update pkgver first
-        check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger)
-        # generate new .SRCINFO and put it to parser
-        src_info_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger)
-        src_info, errors = parse_srcinfo(src_info_source)
-        if errors:
-            raise InvalidPackageInfo(errors)
-
-        return self.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
-
     @classmethod
     def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
         '''
@@ -134,14 +108,31 @@ class Package:
         :param aur_url: AUR root url
         :return: package properties
         '''
-        with open(os.path.join(path, '.SRCINFO')) as fn:
-            src_info, errors = parse_srcinfo(fn.read())
+        with open(os.path.join(path, '.SRCINFO')) as srcinfo_file:
+            srcinfo, errors = parse_srcinfo(srcinfo_file.read())
         if errors:
             raise InvalidPackageInfo(errors)
-        packages = {key: PackageDescription() for key in src_info['packages'].keys()}
-        version = cls.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
+        packages = {key: PackageDescription() for key in srcinfo['packages']}
+        version = cls.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel'])
 
-        return cls(src_info['pkgbase'], version, aur_url, packages)
+        return cls(srcinfo['pkgbase'], version, aur_url, packages)
+
+    @classmethod
+    def from_json(cls: Type[Package], dump: Dict[str, Any]) -> Package:
+        '''
+        construct package properties from json dump
+        :param dump: json dump body
+        :return: package properties
+        '''
+        packages = {
+            key: PackageDescription(**value)
+            for key, value in dump.get('packages', {}).items()
+        }
+        return Package(
+            base=dump['base'],
+            version=dump['version'],
+            aur_url=dump['aur_url'],
+            packages=packages)
 
     @staticmethod
     def dependencies(path: str) -> Set[str]:
@@ -150,17 +141,17 @@ class Package:
         :param path: path to package sources directory
         :return: list of package dependencies including makedepends array, but excluding packages from this base
         '''
-        with open(os.path.join(path, '.SRCINFO')) as fn:
-            src_info, errors = parse_srcinfo(fn.read())
+        with open(os.path.join(path, '.SRCINFO')) as srcinfo_file:
+            srcinfo, errors = parse_srcinfo(srcinfo_file.read())
         if errors:
             raise InvalidPackageInfo(errors)
-        makedepends = src_info.get('makedepends', [])
+        makedepends = srcinfo.get('makedepends', [])
         # sum over each package
-        depends: List[str] = src_info.get('depends', [])
-        for package in src_info['packages'].values():
+        depends: List[str] = srcinfo.get('depends', [])
+        for package in srcinfo['packages'].values():
             depends.extend(package.get('depends', []))
         # we are not interested in dependencies inside pkgbase
-        packages = set(src_info['packages'].keys())
+        packages = set(srcinfo['packages'].keys())
         return set(depends + makedepends) - packages
 
     @staticmethod
@@ -197,6 +188,31 @@ class Package:
         except Exception as e:
             raise InvalidPackageInfo(str(e))
 
+    def actual_version(self, paths: RepositoryPaths) -> str:
+        '''
+        additional method to handle VCS package versions
+        :param paths: repository paths instance
+        :return: package version if package is not VCS and current version according to VCS otherwise
+        '''
+        if not self.is_vcs:
+            return self.version
+
+        from ahriman.core.build_tools.task import Task
+
+        clone_dir = os.path.join(paths.cache, self.base)
+        logger = logging.getLogger('build_details')
+        Task.fetch(clone_dir, self.git_url)
+
+        # update pkgver first
+        check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger)
+        # generate new .SRCINFO and put it to parser
+        srcinfo_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger)
+        srcinfo, errors = parse_srcinfo(srcinfo_source)
+        if errors:
+            raise InvalidPackageInfo(errors)
+
+        return self.full_version(srcinfo.get('epoch'), srcinfo['pkgver'], srcinfo['pkgrel'])
+
     def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool:
         '''
         check if package is out-of-dated
@@ -207,3 +223,10 @@ class Package:
         remote_version = remote.actual_version(paths)  # either normal version or updated VCS
         result: int = vercmp(self.version, remote_version)
         return result < 0
+
+    def view(self) -> Dict[str, Any]:
+        '''
+        generate json package view
+        :return: json-friendly dictionary
+        '''
+        return asdict(self)

src/ahriman/models/sign_settings.py

@@ -43,6 +43,6 @@ class SignSettings(Enum):
         '''
         if value.lower() in ('package', 'packages', 'sign-package'):
             return SignSettings.SignPackages
-        elif value.lower() in ('repository', 'sign-repository'):
+        if value.lower() in ('repository', 'sign-repository'):
             return SignSettings.SignRepository
         raise InvalidOption(value)

src/ahriman/models/upload_settings.py

@@ -43,6 +43,6 @@ class UploadSettings(Enum):
         '''
         if value.lower() in ('rsync',):
             return UploadSettings.Rsync
-        elif value.lower() in ('s3',):
+        if value.lower() in ('s3',):
             return UploadSettings.S3
         raise InvalidOption(value)

src/ahriman/repository/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

src/ahriman/repository/cleaner.py

@@ -0,0 +1,78 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+import shutil
+
+from typing import List
+
+from ahriman.repository.properties import Properties
+
+
+class Cleaner(Properties):
+    '''
+    trait to clean common repository objects
+    '''
+
+    def packages_built(self) -> List[str]:
+        '''
+        get list of files in built packages directory
+        :return: list of filenames from the directory
+        '''
+        raise NotImplementedError
+
+    def clear_build(self) -> None:
+        '''
+        clear sources directory
+        '''
+        self.logger.info('clear package sources directory')
+        for package in os.listdir(self.paths.sources):
+            shutil.rmtree(os.path.join(self.paths.sources, package))
+
+    def clear_cache(self) -> None:
+        '''
+        clear cache directory
+        '''
+        self.logger.info('clear packages sources cache directory')
+        for package in os.listdir(self.paths.cache):
+            shutil.rmtree(os.path.join(self.paths.cache, package))
+
+    def clear_chroot(self) -> None:
+        '''
+        clear cache directory. Warning: this method is architecture independent and will clear every chroot
+        '''
+        self.logger.info('clear build chroot directory')
+        for chroot in os.listdir(self.paths.chroot):
+            shutil.rmtree(os.path.join(self.paths.chroot, chroot))
+
+    def clear_manual(self) -> None:
+        '''
+        clear directory with manual package updates
+        '''
+        self.logger.info('clear manual packages')
+        for package in os.listdir(self.paths.manual):
+            shutil.rmtree(os.path.join(self.paths.manual, package))
+
+    def clear_packages(self) -> None:
+        '''
+        clear directory with built packages (NOT repository itself)
+        '''
+        self.logger.info('clear built packages directory')
+        for package in self.packages_built():
+            os.remove(package)

src/ahriman/repository/executor.py

@@ -0,0 +1,151 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+import shutil
+
+from typing import Dict, Iterable, List, Optional
+
+from ahriman.core.build_tools.task import Task
+from ahriman.core.report.report import Report
+from ahriman.core.upload.uploader import Uploader
+from ahriman.models.package import Package
+from ahriman.repository.cleaner import Cleaner
+
+
+class Executor(Cleaner):
+    '''
+    trait for common repository update processes
+    '''
+
+    def packages(self) -> List[Package]:
+        '''
+        generate list of repository packages
+        :return: list of packages properties
+        '''
+        raise NotImplementedError
+
+    def process_build(self, updates: Iterable[Package]) -> List[str]:
+        '''
+        build packages
+        :param updates: list of packages properties to build
+        :return: `packages_built`
+        '''
+        def build_single(package: Package) -> None:
+            self.reporter.set_building(package.base)
+            task = Task(package, self.architecture, self.config, self.paths)
+            task.init()
+            built = task.build()
+            for src in built:
+                dst = os.path.join(self.paths.packages, os.path.basename(src))
+                shutil.move(src, dst)
+
+        for package in updates:
+            try:
+                build_single(package)
+            except Exception:
+                self.reporter.set_failed(package.base)
+                self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
+                continue
+        self.clear_build()
+
+        return self.packages_built()
+
+    def process_remove(self, packages: Iterable[str]) -> str:
+        '''
+        remove packages from list
+        :param packages: list of package names or bases to rmeove
+        :return: path to repository database
+        '''
+        def remove_single(package: str) -> None:
+            try:
+                self.repo.remove(package)
+            except Exception:
+                self.logger.exception(f'could not remove {package}', exc_info=True)
+
+        requested = set(packages)
+        for local in self.packages():
+            if local.base in packages:
+                to_remove = set(local.packages.keys())
+                self.reporter.remove(local.base)  # we only update status page in case of base removal
+            elif requested.intersection(local.packages.keys()):
+                to_remove = requested.intersection(local.packages.keys())
+            else:
+                to_remove = set()
+            for package in to_remove:
+                remove_single(package)
+
+        return self.repo.repo_path
+
+    def process_report(self, targets: Optional[Iterable[str]]) -> None:
+        '''
+        generate reports
+        :param targets: list of targets to generate reports. Configuration option will be used if it is not set
+        '''
+        if targets is None:
+            targets = self.config.getlist('report', 'target')
+        for target in targets:
+            Report.run(self.architecture, self.config, target, self.packages())
+
+    def process_sync(self, targets: Optional[Iterable[str]]) -> None:
+        '''
+        process synchronization to remote servers
+        :param targets: list of targets to sync. Configuration option will be used if it is not set
+        '''
+        if targets is None:
+            targets = self.config.getlist('upload', 'target')
+        for target in targets:
+            Uploader.run(self.architecture, self.config, target, self.paths.repository)
+
+    def process_update(self, packages: Iterable[str]) -> str:
+        '''
+        sign packages, add them to repository and update repository database
+        :param packages: list of filenames to run
+        :return: path to repository database
+        '''
+        def update_single(fn: Optional[str], base: str) -> None:
+            if fn is None:
+                self.logger.warning(f'received empty package name for base {base}')
+                return  # suppress type checking, it never can be none actually
+            # in theory it might be NOT packages directory, but we suppose it is
+            full_path = os.path.join(self.paths.packages, fn)
+            files = self.sign.sign_package(full_path, base)
+            for src in files:
+                dst = os.path.join(self.paths.repository, os.path.basename(src))
+                shutil.move(src, dst)
+            package_path = os.path.join(self.paths.repository, fn)
+            self.repo.add(package_path)
+
+        # we are iterating over bases, not single packages
+        updates: Dict[str, Package] = {}
+        for fn in packages:
+            local = Package.load(fn, self.pacman, self.aur_url)
+            updates.setdefault(local.base, local).packages.update(local.packages)
+
+        for local in updates.values():
+            try:
+                for description in local.packages.values():
+                    update_single(description.filename, local.base)
+                self.reporter.set_success(local)
+            except Exception:
+                self.reporter.set_failed(local.base)
+                self.logger.exception(f'could not process {local.base}', exc_info=True)
+        self.clear_packages()
+
+        return self.repo.repo_path

src/ahriman/repository/properties.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+
+from ahriman.core.alpm.pacman import Pacman
+from ahriman.core.alpm.repo import Repo
+from ahriman.core.configuration import Configuration
+from ahriman.core.sign.gpg import GPG
+from ahriman.core.watcher.client import Client
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+class Properties:
+    '''
+    repository internal objects holder
+    :ivar architecture: repository architecture
+    :ivar aur_url: base AUR url
+    :ivar config: configuration instance
+    :ivar logger: class logger
+    :ivar name: repository name
+    :ivar pacman: alpm wrapper instance
+    :ivar paths: repository paths instance
+    :ivar repo: repo commands wrapper instance
+    :ivar reporter: build status reporter instance
+    :ivar sign: GPG wrapper instance
+    '''
+
+    def __init__(self, architecture: str, config: Configuration) -> None:
+        self.logger = logging.getLogger('builder')
+        self.architecture = architecture
+        self.config = config
+
+        self.aur_url = config.get('alpm', 'aur_url')
+        self.name = config.get('repository', 'name')
+
+        self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
+        self.paths.create_tree()
+
+        self.pacman = Pacman(config)
+        self.sign = GPG(architecture, config)
+        self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
+        self.reporter = Client.load(architecture, config)

src/ahriman/repository/repository.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+
+from typing import Dict, List
+
+from ahriman.core.util import package_like
+from ahriman.models.package import Package
+from ahriman.repository.executor import Executor
+from ahriman.repository.update_handler import UpdateHandler
+
+
+class Repository(Executor, UpdateHandler):
+    '''
+    base repository control class
+    '''
+
+    def packages(self) -> List[Package]:
+        '''
+        generate list of repository packages
+        :return: list of packages properties
+        '''
+        result: Dict[str, Package] = {}
+        for fn in os.listdir(self.paths.repository):
+            if not package_like(fn):
+                continue
+            full_path = os.path.join(self.paths.repository, fn)
+            try:
+                local = Package.load(full_path, self.pacman, self.aur_url)
+                result.setdefault(local.base, local).packages.update(local.packages)
+            except Exception:
+                self.logger.exception(f'could not load package from {fn}', exc_info=True)
+                continue
+        return list(result.values())
+
+    def packages_built(self) -> List[str]:
+        '''
+        get list of files in built packages directory
+        :return: list of filenames from the directory
+        '''
+        return [
+            os.path.join(self.paths.packages, fn)
+            for fn in os.listdir(self.paths.packages)
+        ]

src/ahriman/repository/update_handler.py

@@ -0,0 +1,92 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+
+from typing import Iterable, List
+
+from ahriman.models.package import Package
+from ahriman.repository.cleaner import Cleaner
+
+
+class UpdateHandler(Cleaner):
+    '''
+    trait to get package update list
+    '''
+
+    def packages(self) -> List[Package]:
+        '''
+        generate list of repository packages
+        :return: list of packages properties
+        '''
+        raise NotImplementedError
+
+    def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
+        '''
+        check AUR for updates
+        :param filter_packages: do not check every package just specified in the list
+        :param no_vcs: do not check VCS packages
+        :return: list of packages which are out-of-dated
+        '''
+        result: List[Package] = []
+
+        build_section = self.config.get_section_name('build', self.architecture)
+        ignore_list = self.config.getlist(build_section, 'ignore_packages')
+
+        for local in self.packages():
+            if local.base in ignore_list:
+                continue
+            if local.is_vcs and no_vcs:
+                continue
+            if filter_packages and local.base not in filter_packages:
+                continue
+
+            try:
+                remote = Package.load(local.base, self.pacman, self.aur_url)
+                if local.is_outdated(remote, self.paths):
+                    self.reporter.set_pending(local.base)
+                    result.append(remote)
+            except Exception:
+                self.reporter.set_failed(local.base)
+                self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
+                continue
+
+        return result
+
+    def updates_manual(self) -> List[Package]:
+        '''
+        check for packages for which manual update has been requested
+        :return: list of packages which are out-of-dated
+        '''
+        result: List[Package] = []
+        known_bases = {package.base for package in self.packages()}
+
+        for fn in os.listdir(self.paths.manual):
+            try:
+                local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
+                result.append(local)
+                if local.base not in known_bases:
+                    self.reporter.set_unknown(local)
+                else:
+                    self.reporter.set_pending(local.base)
+            except Exception:
+                self.logger.exception(f'could not add package from {fn}', exc_info=True)
+        self.clear_manual()
+
+        return result

src/ahriman/version.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__version__ = '0.14.1'
+__version__ = '0.15.0'

src/ahriman/web/routes.py

@@ -34,21 +34,27 @@ def setup_routes(application: Application) -> None:
         GET /                           get build status page
         GET /index.html                 same as above
 
+        GET /api/v1/ahriman             get current service status
         POST /api/v1/ahriman            update service status
 
+        GET /api/v1/packages            get all known packages
         POST /api/v1/packages           force update every package from repository
 
-        POST /api/v1/package/:base      update package base status
         DELETE /api/v1/package/:base    delete package base from status page
+        GET /api/v1/package/:base       get package base status
+        POST /api/v1/package/:base      update package base status
 
     :param application: web application instance
     '''
     application.router.add_get('/', IndexView)
     application.router.add_get('/index.html', IndexView)
 
+    application.router.add_get('/api/v1/ahriman', AhrimanView)
     application.router.add_post('/api/v1/ahriman', AhrimanView)
 
+    application.router.add_get('/api/v1/packages', PackagesView)
     application.router.add_post('/api/v1/packages', PackagesView)
 
     application.router.add_delete('/api/v1/packages/{package}', PackageView)
+    application.router.add_get('/api/v1/packages/{package}', PackageView)
     application.router.add_post('/api/v1/packages/{package}', PackageView)

src/ahriman/web/views/ahriman.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from aiohttp.web import HTTPBadRequest, HTTPOk, Response
+from aiohttp.web import HTTPBadRequest, HTTPNoContent, Response, json_response
 
 from ahriman.models.build_status import BuildStatusEnum
 from ahriman.web.views.base import BaseView
@@ -28,6 +28,13 @@ class AhrimanView(BaseView):
     service status web view
     '''
 
+    async def get(self) -> Response:
+        '''
+        get current service status
+        :return: 200 with service status object
+        '''
+        return json_response(self.service.status.view())
+
     async def post(self) -> Response:
         '''
         update service status
@@ -37,7 +44,7 @@ class AhrimanView(BaseView):
             "status": "unknown",   # service status string, must be valid `BuildStatusEnum`
         }
 
-        :return: 200 on success
+        :return: 204 on success
         '''
         data = await self.request.json()
 
@@ -48,4 +55,4 @@ class AhrimanView(BaseView):
 
         self.service.update_self(status)
 
-        return HTTPOk()
+        return HTTPNoContent()

src/ahriman/web/views/index.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import aiohttp_jinja2  # type: ignore
+import aiohttp_jinja2
 
 from typing import Any, Dict
 
@@ -41,7 +41,7 @@ class IndexView(BaseView):
         version - ahriman version, string, required
     '''
 
-    @aiohttp_jinja2.template("build-status.jinja2")  # type: ignore
+    @aiohttp_jinja2.template("build-status.jinja2")
     async def get(self) -> Dict[str, Any]:
         '''
         process get request. No parameters supported here
@@ -51,7 +51,7 @@ class IndexView(BaseView):
         packages = [
             {
                 'base': package.base,
-                'packages': [p for p in sorted(package.packages)],
+                'packages': list(sorted(package.packages)),
                 'status': status.status.value,
                 'timestamp': pretty_datetime(status.timestamp),
                 'version': package.version,

src/ahriman/web/views/package.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from aiohttp.web import HTTPBadRequest, HTTPOk, Response
+from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
 
 from ahriman.models.build_status import BuildStatusEnum
 from ahriman.models.package import Package
@@ -29,15 +29,33 @@ class PackageView(BaseView):
     package base specific web view
     '''
 
+    async def get(self) -> Response:
+        '''
+        get current package base status
+        :return: 200 with package description on success
+        '''
+        base = self.request.match_info['package']
+
+        try:
+            package, status = self.service.get(base)
+        except KeyError:
+            raise HTTPNotFound()
+
+        response = {
+            'package': package.view(),
+            'status': status.view()
+        }
+        return json_response(response)
+
     async def delete(self) -> Response:
         '''
         delete package base from status page
-        :return: 200 on success
+        :return: 204 on success
         '''
         base = self.request.match_info['package']
         self.service.remove(base)
 
-        return HTTPOk()
+        return HTTPNoContent()
 
     async def post(self) -> Response:
         '''
@@ -50,13 +68,13 @@ class PackageView(BaseView):
                            # Must be supplied in case if package base is unknown
         }
 
-        :return: 200 on success
+        :return: 204 on success
         '''
         base = self.request.match_info['package']
         data = await self.request.json()
 
         try:
-            package = Package(**data['package']) if 'package' in data else None
+            package = Package.from_json(data['package']) if 'package' in data else None
             status = BuildStatusEnum(data['status'])
         except Exception as e:
             raise HTTPBadRequest(text=str(e))
@@ -66,4 +84,4 @@ class PackageView(BaseView):
         except KeyError:
             raise HTTPBadRequest(text=f'Package {base} is unknown, but no package body set')
 
-        return HTTPOk()
+        return HTTPNoContent()

src/ahriman/web/views/packages.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from aiohttp.web import HTTPOk, Response
+from aiohttp.web import HTTPNoContent, Response, json_response
 
 from ahriman.web.views.base import BaseView
 
@@ -27,11 +27,24 @@ class PackagesView(BaseView):
     global watcher view
     '''
 
+    async def get(self) -> Response:
+        '''
+        get current packages status
+        :return: 200 with package description on success
+        '''
+        response = [
+            {
+                'package': package.view(),
+                'status': status.view()
+            } for package, status in self.service.packages
+        ]
+        return json_response(response)
+
     async def post(self) -> Response:
         '''
         reload all packages from repository. No parameters supported here
-        :return: 200 on success
+        :return: 204 on success
         '''
         self.service.load()
 
-        return HTTPOk()
+        return HTTPNoContent()

src/ahriman/web/web.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import aiohttp_jinja2  # type: ignore
+import aiohttp_jinja2
 import jinja2
 import logging