implement support of unix socket for server

This feature can be used for unauthorized access to apis - e.g. for
reporting service if it is run on the same machine. Since now it becomes
recommended way for the interprocess communication, thus some options
(e.g. creating user with as-service flag) are no longer available now
This commit is contained in:
Evgenii Alekseev 2022-11-29 01:18:01 +02:00
parent bba58352e0
commit fe66c6c45c
24 changed files with 247 additions and 134 deletions

View File

@ -10,6 +10,7 @@ ENV AHRIMAN_PACKAGER="ahriman bot <ahriman@example.com>"
ENV AHRIMAN_PORT="" ENV AHRIMAN_PORT=""
ENV AHRIMAN_REPOSITORY="aur-clone" ENV AHRIMAN_REPOSITORY="aur-clone"
ENV AHRIMAN_REPOSITORY_ROOT="/var/lib/ahriman/ahriman" ENV AHRIMAN_REPOSITORY_ROOT="/var/lib/ahriman/ahriman"
ENV AHRIMAN_UNIX_SOCKET=""
ENV AHRIMAN_USER="ahriman" ENV AHRIMAN_USER="ahriman"
# install environment # install environment
@ -26,7 +27,7 @@ COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package"
## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size ## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size
RUN pacman --noconfirm -Sy devtools git pyalpm python-inflection python-passlib python-requests python-setuptools python-srcinfo && \ RUN pacman --noconfirm -Sy devtools git pyalpm python-inflection python-passlib python-requests python-setuptools python-srcinfo && \
pacman --noconfirm -Sy python-build python-installer python-wheel && \ pacman --noconfirm -Sy python-build python-installer python-wheel && \
pacman --noconfirm -Sy breezy mercurial python-aiohttp python-boto3 python-cryptography python-jinja rsync subversion && \ pacman --noconfirm -Sy breezy mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-requests-unixsocket rsync subversion && \
runuser -u build -- install-aur-package python-aioauth-client python-aiohttp-jinja2 python-aiohttp-debugtoolbar \ runuser -u build -- install-aur-package python-aioauth-client python-aiohttp-jinja2 python-aiohttp-debugtoolbar \
python-aiohttp-session python-aiohttp-security python-aiohttp-session python-aiohttp-security

View File

@ -4,9 +4,17 @@ set -e
[ -n "$AHRIMAN_DEBUG" ] && set -x [ -n "$AHRIMAN_DEBUG" ] && set -x
# configuration tune # configuration tune
sed -i "s|root = /var/lib/ahriman|root = $AHRIMAN_REPOSITORY_ROOT|g" "/etc/ahriman.ini" cat <<EOF > "/etc/ahriman.ini.d/00-docker.ini"
sed -i "s|database = /var/lib/ahriman/ahriman.db|database = $AHRIMAN_REPOSITORY_ROOT/ahriman.db|g" "/etc/ahriman.ini" [repository]
sed -i "s|host = 127.0.0.1|host = $AHRIMAN_HOST|g" "/etc/ahriman.ini" root = $AHRIMAN_REPOSITORY_ROOT
[settings]
database = $AHRIMAN_REPOSITORY_ROOT/ahriman.db
[web]
host = $AHRIMAN_HOST
EOF
sed -i "s|handlers = syslog_handler|handlers = ${AHRIMAN_OUTPUT}_handler|g" "/etc/ahriman.ini.d/logging.ini" sed -i "s|handlers = syslog_handler|handlers = ${AHRIMAN_OUTPUT}_handler|g" "/etc/ahriman.ini.d/logging.ini"
AHRIMAN_DEFAULT_ARGS=("--architecture" "$AHRIMAN_ARCHITECTURE") AHRIMAN_DEFAULT_ARGS=("--architecture" "$AHRIMAN_ARCHITECTURE")
@ -32,9 +40,11 @@ AHRIMAN_SETUP_ARGS=("--build-as-user" "$AHRIMAN_USER")
AHRIMAN_SETUP_ARGS+=("--packager" "$AHRIMAN_PACKAGER") AHRIMAN_SETUP_ARGS+=("--packager" "$AHRIMAN_PACKAGER")
AHRIMAN_SETUP_ARGS+=("--repository" "$AHRIMAN_REPOSITORY") AHRIMAN_SETUP_ARGS+=("--repository" "$AHRIMAN_REPOSITORY")
if [ -n "$AHRIMAN_PORT" ]; then if [ -n "$AHRIMAN_PORT" ]; then
# in addition it must be handled in docker run command
AHRIMAN_SETUP_ARGS+=("--web-port" "$AHRIMAN_PORT") AHRIMAN_SETUP_ARGS+=("--web-port" "$AHRIMAN_PORT")
fi fi
if [ -n "$AHRIMAN_UNIX_SOCKET" ]; then
AHRIMAN_SETUP_ARGS+=("--web-unix-socket" "$AHRIMAN_UNIX_SOCKET")
fi
ahriman "${AHRIMAN_DEFAULT_ARGS[@]}" repo-setup "${AHRIMAN_SETUP_ARGS[@]}" ahriman "${AHRIMAN_DEFAULT_ARGS[@]}" repo-setup "${AHRIMAN_SETUP_ARGS[@]}"
# refresh database # refresh database

View File

@ -1,4 +1,4 @@
.TH AHRIMAN "1" "2022\-11\-16" "ahriman" "Generated Python Manual" .TH AHRIMAN "1" "2022\-11\-29" "ahriman" "Generated Python Manual"
.SH NAME .SH NAME
ahriman ahriman
.SH SYNOPSIS .SH SYNOPSIS
@ -128,7 +128,7 @@ run triggers
update packages update packages
.TP .TP
\fBahriman\fR \fI\,shell\/\fR \fBahriman\fR \fI\,shell\/\fR
envoke python shell invoke python shell
.TP .TP
\fBahriman\fR \fI\,user\-add\/\fR \fBahriman\fR \fI\,user\-add\/\fR
create or update user create or update user
@ -509,7 +509,7 @@ root path of the extracted files
usage: ahriman repo\-setup [\-h] [\-\-build\-as\-user BUILD_AS_USER] [\-\-build\-command BUILD_COMMAND] usage: ahriman repo\-setup [\-h] [\-\-build\-as\-user BUILD_AS_USER] [\-\-build\-command BUILD_COMMAND]
[\-\-from\-configuration FROM_CONFIGURATION] [\-\-multilib | \-\-no\-multilib] \-\-packager PACKAGER [\-\-from\-configuration FROM_CONFIGURATION] [\-\-multilib | \-\-no\-multilib] \-\-packager PACKAGER
\-\-repository REPOSITORY [\-\-sign\-key SIGN_KEY] [\-\-sign\-target {disabled,pacakges,repository}] \-\-repository REPOSITORY [\-\-sign\-key SIGN_KEY] [\-\-sign\-target {disabled,pacakges,repository}]
[\-\-web\-port WEB_PORT] [\-\-web\-port WEB_PORT] [\-\-web\-unix\-socket WEB_UNIX_SOCKET]
create initial service configuration, requires root create initial service configuration, requires root
@ -550,6 +550,10 @@ sign options
\fB\-\-web\-port\fR \fI\,WEB_PORT\/\fR \fB\-\-web\-port\fR \fI\,WEB_PORT\/\fR
port of the web service port of the web service
.TP
\fB\-\-web\-unix\-socket\fR \fI\,WEB_UNIX_SOCKET\/\fR
path to unix socket used for interprocess communications
.SH COMMAND \fI\,'ahriman repo\-sign'\/\fR .SH COMMAND \fI\,'ahriman repo\-sign'\/\fR
usage: ahriman repo\-sign [\-h] [package ...] usage: ahriman repo\-sign [\-h] [package ...]
@ -633,7 +637,7 @@ drop into python shell while having created application
instead of dropping into shell, just execute the specified code instead of dropping into shell, just execute the specified code
.SH COMMAND \fI\,'ahriman user\-add'\/\fR .SH COMMAND \fI\,'ahriman user\-add'\/\fR
usage: ahriman user\-add [\-h] [\-\-as\-service] [\-p PASSWORD] [\-r {unauthorized,read,reporter,full}] [\-s] username usage: ahriman user\-add [\-h] [\-p PASSWORD] [\-r {unauthorized,read,reporter,full}] [\-s] username
update user for web services with the given password and role. In case if password was not entered it will be asked interactively update user for web services with the given password and role. In case if password was not entered it will be asked interactively
@ -642,10 +646,6 @@ update user for web services with the given password and role. In case if passwo
username for web service username for web service
.SH OPTIONS \fI\,'ahriman user\-add'\/\fR .SH OPTIONS \fI\,'ahriman user\-add'\/\fR
.TP
\fB\-\-as\-service\fR
add user as service user
.TP .TP
\fB\-p\fR \fI\,PASSWORD\/\fR, \fB\-\-password\fR \fI\,PASSWORD\/\fR \fB\-p\fR \fI\,PASSWORD\/\fR, \fB\-\-password\fR \fI\,PASSWORD\/\fR
user password. Blank password will be treated as empty password, which is in particular must be used for OAuth2 user password. Blank password will be treated as empty password, which is in particular must be used for OAuth2
@ -678,7 +678,7 @@ return non\-zero exit status if result is empty
filter users by role filter users by role
.SH COMMAND \fI\,'ahriman user\-remove'\/\fR .SH COMMAND \fI\,'ahriman user\-remove'\/\fR
usage: ahriman user\-remove [\-h] [\-s] username usage: ahriman user\-remove [\-h] username
remove user from the user mapping and update the configuration remove user from the user mapping and update the configuration
@ -686,11 +686,6 @@ remove user from the user mapping and update the configuration
\fBusername\fR \fBusername\fR
username for web service username for web service
.SH OPTIONS \fI\,'ahriman user\-remove'\/\fR
.TP
\fB\-s\fR, \fB\-\-secure\fR
set file permissions to user\-only
.SH COMMAND \fI\,'ahriman version'\/\fR .SH COMMAND \fI\,'ahriman version'\/\fR
usage: ahriman version [\-h] usage: ahriman version [\-h]

View File

@ -36,6 +36,7 @@ This package contains everything which is required for any time of application r
* ``ahriman.core.database`` is everything including data and schema migrations for database. * ``ahriman.core.database`` is everything including data and schema migrations for database.
* ``ahriman.core.formatters`` package provides ``Printer`` sub-classes for printing data (e.g. package properties) to stdout which are used by some handlers. * ``ahriman.core.formatters`` package provides ``Printer`` sub-classes for printing data (e.g. package properties) to stdout which are used by some handlers.
* ``ahriman.core.gitremote`` is a package with remote PKGBUILD triggers. Should not be called directly. * ``ahriman.core.gitremote`` is a package with remote PKGBUILD triggers. Should not be called directly.
* ``ahriman.core.log`` is a log utils package. It includes logger loader class, custom HTTP based logger and access logger for HTTP services with additional filters.
* ``ahriman.core.report`` is a package with reporting triggers. Should not be called directly. * ``ahriman.core.report`` is a package with reporting triggers. Should not be called directly.
* ``ahriman.core.repository`` contains several traits and base repository (``ahriman.core.repository.Repository`` class) implementation. * ``ahriman.core.repository`` contains several traits and base repository (``ahriman.core.repository.Repository`` class) implementation.
* ``ahriman.core.sign`` package provides sign feature (only gpg calls are available). * ``ahriman.core.sign`` package provides sign feature (only gpg calls are available).
@ -196,7 +197,9 @@ means that there is user ``username`` with ``read`` access and password ``passwo
OAuth provider uses library definitions (``aioauth-client``) in order *authenticate* users. It still requires user permission to be set in database, thus it inherits mapping provider without any changes. Whereas we could override ``check_credentials`` (authentication method) by something custom, OAuth flow is a bit more complex than just forward request, thus we have to implement the flow in login form. OAuth provider uses library definitions (``aioauth-client``) in order *authenticate* users. It still requires user permission to be set in database, thus it inherits mapping provider without any changes. Whereas we could override ``check_credentials`` (authentication method) by something custom, OAuth flow is a bit more complex than just forward request, thus we have to implement the flow in login form.
OAuth's implementation also allows authenticating users via username + password (in the same way as mapping does) though it is not recommended for end-users and password must be left blank. In particular this feature is used by service reporting (aka robots). OAuth's implementation also allows authenticating users via username + password (in the same way as mapping does) though it is not recommended for end-users and password must be left blank. In particular this feature can be used by service reporting (aka robots).
In addition, web service checks the source socket used. In case if it belongs to ``socket.AF_UNIX`` family, it will skip any furher checks considering the request to be performed in safe environment (e.g. on the same physical machine). This feature, in particular is being used by the reporter instances in case if socket address is set in configuration.
In order to configure users there are special commands. In order to configure users there are special commands.
@ -244,6 +247,7 @@ Web application requires the following python packages to be installed:
* In addition, ``aiohttp_debugtoolbar`` is required for debug panel. Please note that this option does not work together with authorization and basically must not be used in production. * In addition, ``aiohttp_debugtoolbar`` is required for debug panel. Please note that this option does not work together with authorization and basically must not be used in production.
* In addition, authorization feature requires ``aiohttp_security``, ``aiohttp_session`` and ``cryptography``. * In addition, authorization feature requires ``aiohttp_security``, ``aiohttp_session`` and ``cryptography``.
* In addition to base authorization dependencies, OAuth2 also requires ``aioauth-client`` library. * In addition to base authorization dependencies, OAuth2 also requires ``aioauth-client`` library.
* In addition if you would like to disable authorization for local access (recommended way in order to run the application itself with reporting support), the ``requests-unixsocket`` library is required.
Middlewares Middlewares
^^^^^^^^^^^ ^^^^^^^^^^^

View File

@ -240,4 +240,5 @@ Web server settings. If any of ``host``/``port`` is not set, web integration wil
* ``port`` - port to bind, int, optional. * ``port`` - port to bind, int, optional.
* ``static_path`` - path to directory with static files, string, required. * ``static_path`` - path to directory with static files, string, required.
* ``templates`` - path to templates directory, string, required. * ``templates`` - path to templates directory, string, required.
* ``unix_socket`` - path to the listening unix socket, string, optional. If set, server will create the socket on the specified address which can (and will) be used by application. Note, that unlike usual host/port configuration, unix socket allows to perform requests without authorization.
* ``username`` - username to authorize in web service in order to update service status, string, required in case if authorization enabled. * ``username`` - username to authorize in web service in order to update service status, string, required in case if authorization enabled.

View File

@ -350,13 +350,13 @@ The default action (in case if no arguments provided) is ``repo-update``. Basica
.. code-block:: shell .. code-block:: shell
docker run -v /path/to/local/repo:/var/lib/ahriman -v /path/to/overrides/overrides.ini:/etc/ahriman.ini.d/10-overrides.ini arcan1s/ahriman:latest docker run --privileged -v /path/to/local/repo:/var/lib/ahriman -v /path/to/overrides/overrides.ini:/etc/ahriman.ini.d/10-overrides.ini arcan1s/ahriman:latest
The action can be specified during run, e.g.: The action can be specified during run, e.g.:
.. code-block:: shell .. code-block:: shell
docker run arcan1s/ahriman:latest package-add ahriman --now docker run --privileged -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest package-add ahriman --now
For more details please refer to docker FAQ. For more details please refer to docker FAQ.
@ -374,13 +374,25 @@ The following environment variables are supported:
* ``AHRIMAN_PORT`` - HTTP server port if any, default is empty. * ``AHRIMAN_PORT`` - HTTP server port if any, default is empty.
* ``AHRIMAN_REPOSITORY`` - repository name, default is ``aur-clone``. * ``AHRIMAN_REPOSITORY`` - repository name, default is ``aur-clone``.
* ``AHRIMAN_REPOSITORY_ROOT`` - repository root. Because of filesystem rights it is required to override default repository root. By default, it uses ``ahriman`` directory inside ahriman's home, which can be passed as mount volume. * ``AHRIMAN_REPOSITORY_ROOT`` - repository root. Because of filesystem rights it is required to override default repository root. By default, it uses ``ahriman`` directory inside ahriman's home, which can be passed as mount volume.
* ``AHRIMAN_UNIX_SOCKET`` - full path to unix socket which is used by web server, default is empty. Note that more likely you would like to put it inside ``AHRIMAN_REPOSITORY_ROOT`` directory (e.g. ``/var/lib/ahriman/ahriman/ahriman-web.sock``) or to ``/tmp``.
* ``AHRIMAN_USER`` - ahriman user, usually must not be overwritten, default is ``ahriman``. * ``AHRIMAN_USER`` - ahriman user, usually must not be overwritten, default is ``ahriman``.
You can pass any of these variables by using ``-e`` argument, e.g.: You can pass any of these variables by using ``-e`` argument, e.g.:
.. code-block:: shell .. code-block:: shell
docker run -e AHRIMAN_PORT=8080 -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest docker run --privileged -e AHRIMAN_PORT=8080 -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest
Daemon service
^^^^^^^^^^^^^^
There is special ``daemon`` subcommand which emulates systemd timer and will perform repository update periodically:
.. code-block:: shell
docker run --privileged -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest daemon
This command uses same rules as ``repo-update``, thus, e.g. requires ``--privileged`` flag.
Web service setup Web service setup
^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^
@ -389,26 +401,23 @@ Well for that you would need to have web container instance running forever; it
.. code-block:: shell .. code-block:: shell
docker run -p 8080:8080 -e AHRIMAN_PORT=8080 -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest docker run --privileged -p 8080:8080 -e AHRIMAN_PORT=8080 -e AHRIMAN_UNIX_SOCKET=/var/lib/ahriman/ahriman/ahriman-web.sock -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest
Note about ``AHRIMAN_PORT`` environment variable which is required in order to enable web service. An additional port bind by ``-p 8080:8080`` is required to pass docker port outside of container. Note about ``AHRIMAN_PORT`` environment variable which is required in order to enable web service. An additional port bind by ``-p 8080:8080`` is required to pass docker port outside of container.
For every next container run use arguments ``-e AHRIMAN_PORT=8080 --net=host``, e.g.: The ``AHRIMAN_UNIX_SOCKET`` variable is not required, however, highly recommended as it can be used for interprocess communications. If you set this variable you would like to be sure that this path is available outside of container if you are going to use multiple docker instances.
If you are using ``AHRIMAN_UNIX_SOCKET`` variable, for every next container run it has to be passed also, e.g.:
.. code-block:: shell .. code-block:: shell
docker run --privileged -e AHRIMAN_PORT=8080 --net=host -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest docker run --privileged -e AHRIMAN_UNIX_SOCKET=/var/lib/ahriman/ahriman/ahriman-web.sock -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest
Daemon service Otherwise, you would need to pass ``AHRIMAN_PORT`` and mount container network to the host system (``--net=host``), e.g.:
^^^^^^^^^^^^^^
There is special subcommand which emulates systemd timer and will perform repository update periodically:
.. code-block:: shell .. code-block:: shell
docker run --privileged -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest daemon docker run --privileged --net=host -e AHRIMAN_PORT=8080 -v /path/to/local/repo:/var/lib/ahriman arcan1s/ahriman:latest
This command uses same rules as ``repo-update``, thus, e.g. requires ``--privileged`` flag.
Remote synchronization Remote synchronization
---------------------- ----------------------
@ -667,18 +676,36 @@ How to enable basic authorization
target = configuration target = configuration
#. #.
Create user for the service: In order to provide access for reporting from application instances you can (recommended way) use unix sockets by configuring the following (note, that it requires ``python-requests-unixsocket`` package to be installed):
.. code-block:: ini
[web]
unix_socket = /var/lib/ahriman/ahriman-web.sock
This socket path must be available for web service instance and must be available for application instances (e.g. in case if you are using docker container, see above, you need to be sure that the socket is passed to the root filesystem).
By the way, unix socket variable will be automatically set in case if ``--web-unix-socket`` argument is supplied to the ``setup`` subcommand.
Alternatively, you need to create user for the service:
.. code-block:: shell .. code-block:: shell
sudo -u ahriman ahriman user-add --as-service -r write api sudo -u ahriman ahriman user-add -r write api
This command will ask for the password, just type it in stdin; *do not* leave the field blank, user will not be able to authorize. This command will ask for the password, just type it in stdin; *do not* leave the field blank, user will not be able to authorize, and finally configure the application:
.. code-block:: ini
[web]
username = api
password = pa55w0rd
#. #.
Create end-user ``sudo -u ahriman ahriman user-add -r write my-first-user`` with password. Create end-user ``sudo -u ahriman ahriman user-add -r write my-first-user`` with password.
#. Restart web service ``systemctl restart ahriman-web@x86_64``. #.
Restart web service ``systemctl restart ahriman-web@x86_64``.
How to enable OAuth authorization How to enable OAuth authorization
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -717,7 +744,8 @@ How to enable OAuth authorization
#. #.
Create end-user ``sudo -u ahriman ahriman user-add -r write my-first-user``. When it will ask for the password leave it blank. Create end-user ``sudo -u ahriman ahriman user-add -r write my-first-user``. When it will ask for the password leave it blank.
#. Restart web service ``systemctl restart ahriman-web@x86_64``. #.
Restart web service ``systemctl restart ahriman-web@x86_64``.
Backup and restore Backup and restore
------------------ ------------------

View File

@ -20,6 +20,7 @@ optdepends=('breezy: -bzr packages support'
'python-aiohttp-session: web server with authorization' 'python-aiohttp-session: web server with authorization'
'python-boto3: sync to s3' 'python-boto3: sync to s3'
'python-cryptography: web server with authorization' 'python-cryptography: web server with authorization'
'python-requests-unixsocket: client report to web server by unix socket'
'python-jinja: html report generation' 'python-jinja: html report generation'
'rsync: sync by using rsync' 'rsync: sync by using rsync'
'subversion: -svn packages support') 'subversion: -svn packages support')

View File

@ -134,6 +134,7 @@ setup(
"aiohttp_session", "aiohttp_session",
"aiohttp_security", "aiohttp_security",
"cryptography", "cryptography",
"requests-unixsocket", # required by unix socket support
], ],
}, },
) )

View File

@ -640,6 +640,7 @@ def _set_repo_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("--sign-target", help="sign options", action="append", parser.add_argument("--sign-target", help="sign options", action="append",
type=SignSettings.from_option, choices=enum_values(SignSettings)) type=SignSettings.from_option, choices=enum_values(SignSettings))
parser.add_argument("--web-port", help="port of the web service", type=int) parser.add_argument("--web-port", help="port of the web service", type=int)
parser.add_argument("--web-unix-socket", help="path to unix socket used for interprocess communications", type=Path)
parser.set_defaults(handler=handlers.Setup, lock=None, report=False, quiet=True, unsafe=True) parser.set_defaults(handler=handlers.Setup, lock=None, report=False, quiet=True, unsafe=True)
return parser return parser
@ -782,12 +783,10 @@ def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("user-add", help="create or update user", parser = root.add_parser("user-add", help="create or update user",
description="update user for web services with the given password and role. " description="update user for web services with the given password and role. "
"In case if password was not entered it will be asked interactively", "In case if password was not entered it will be asked interactively",
epilog="In case of first run (i.e. if password salt is not set yet) or if ``as-service`` " epilog="In case of first run (i.e. if password salt is not set yet) this action requires "
"flag is supplied, this action requires root privileges because it performs write " "root privileges because it performs write to filesystem configuration.",
"to filesystem configuration.",
formatter_class=_formatter) formatter_class=_formatter)
parser.add_argument("username", help="username for web service") parser.add_argument("username", help="username for web service")
parser.add_argument("--as-service", help="add user as service user", action="store_true")
parser.add_argument("-p", "--password", help="user password. Blank password will be treated as empty password, " parser.add_argument("-p", "--password", help="user password. Blank password will be treated as empty password, "
"which is in particular must be used for OAuth2 authorization type.") "which is in particular must be used for OAuth2 authorization type.")
parser.add_argument("-r", "--role", help="user access level", parser.add_argument("-r", "--role", help="user access level",

View File

@ -118,6 +118,10 @@ class Setup(Handler):
section = Configuration.section_name("web", architecture) section = Configuration.section_name("web", architecture)
configuration.set_option(section, "port", str(args.web_port)) configuration.set_option(section, "port", str(args.web_port))
if args.web_unix_socket is not None:
section = Configuration.section_name("web", architecture)
configuration.set_option(section, "unix_socket", str(args.web_unix_socket))
target = include_path / "00-setup-overrides.ini" target = include_path / "00-setup-overrides.ini"
with target.open("w") as ahriman_configuration: with target.open("w") as ahriman_configuration:
configuration.write(ahriman_configuration) configuration.write(ahriman_configuration)

View File

@ -58,9 +58,9 @@ class Users(Handler):
old_salt, salt = Users.get_salt(configuration) old_salt, salt = Users.get_salt(configuration)
user = Users.user_create(args) user = Users.user_create(args)
if old_salt is None or args.as_service: if old_salt is None:
auth_configuration = Users.configuration_get(configuration.include) auth_configuration = Users.configuration_get(configuration.include)
Users.configuration_create(auth_configuration, user, salt, args.as_service, args.secure) Users.configuration_create(auth_configuration, salt, args.secure)
database.user_update(user.hash_password(salt)) database.user_update(user.hash_password(salt))
elif args.action == Action.List: elif args.action == Action.List:
@ -72,22 +72,16 @@ class Users(Handler):
database.user_remove(args.username) database.user_remove(args.username)
@staticmethod @staticmethod
def configuration_create(configuration: Configuration, user: User, salt: str, def configuration_create(configuration: Configuration, salt: str, secure: bool) -> None:
as_service_user: bool, secure: bool) -> None:
""" """
enable configuration if it has been disabled enable configuration if it has been disabled
Args: Args:
configuration(Configuration): configuration instance configuration(Configuration): configuration instance
user(User): user descriptor
salt(str): password hash salt salt(str): password hash salt
as_service_user(bool): add user as service user, also set password and user to configuration
secure(bool): if true then set file permissions to 0o600 secure(bool): if true then set file permissions to 0o600
""" """
configuration.set_option("auth", "salt", salt) configuration.set_option("auth", "salt", salt)
if as_service_user:
configuration.set_option("web", "username", user.username)
configuration.set_option("web", "password", user.password)
Users.configuration_write(configuration, secure) Users.configuration_write(configuration, secure)
@staticmethod @staticmethod

View File

@ -52,8 +52,12 @@ class Client:
address = configuration.get("web", "address", fallback=None) address = configuration.get("web", "address", fallback=None)
host = configuration.get("web", "host", fallback=None) host = configuration.get("web", "host", fallback=None)
port = configuration.getint("web", "port", fallback=None) port = configuration.getint("web", "port", fallback=None)
socket = configuration.get("web", "unix_socket", fallback=None)
if address or (host and port): # basically we just check if there is something we can use for interaction with remote server
# at the moment (end of 2022) I think it would be much better idea to introduce flag like `enabled`,
# but it will totally break used experience
if address or (host and port) or socket:
from ahriman.core.status.web_client import WebClient from ahriman.core.status.web_client import WebClient
return WebClient(configuration) return WebClient(configuration)
return cls() return cls()

View File

@ -21,6 +21,7 @@ import logging
import requests import requests
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
from urllib.parse import quote_plus as urlencode
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.log import LazyLogging from ahriman.core.log import LazyLogging
@ -48,13 +49,12 @@ class WebClient(Client, LazyLogging):
Args: Args:
configuration(Configuration): configuration instance configuration(Configuration): configuration instance
""" """
self.address = self.parse_address(configuration) self.address, use_unix_socket = self.parse_address(configuration)
self.user = User.from_option( self.user = User.from_option(
configuration.get("web", "username", fallback=None), configuration.get("web", "username", fallback=None),
configuration.get("web", "password", fallback=None)) configuration.get("web", "password", fallback=None))
self.__session = requests.session() self.__session = self._create_session(use_unix_socket=use_unix_socket)
self._login()
@property @property
def _login_url(self) -> str: def _login_url(self) -> str:
@ -77,7 +77,7 @@ class WebClient(Client, LazyLogging):
return f"{self.address}/api/v1/status" return f"{self.address}/api/v1/status"
@staticmethod @staticmethod
def parse_address(configuration: Configuration) -> str: def parse_address(configuration: Configuration) -> Tuple[str, bool]:
""" """
parse address from configuration parse address from configuration
@ -85,15 +85,38 @@ class WebClient(Client, LazyLogging):
configuration(Configuration): configuration instance configuration(Configuration): configuration instance
Returns: Returns:
str: valid http address Tuple[str, bool]: tuple of server address and socket flag (True in case if unix socket must be used)
""" """
if (unix_socket := configuration.get("web", "unix_socket", fallback=None)) is not None:
# special pseudo-protocol which is used for unix sockets
return f"http+unix://{urlencode(unix_socket)}", True
address = configuration.get("web", "address", fallback=None) address = configuration.get("web", "address", fallback=None)
if not address: if not address:
# build address from host and port directly # build address from host and port directly
host = configuration.get("web", "host") host = configuration.get("web", "host")
port = configuration.getint("web", "port") port = configuration.getint("web", "port")
address = f"http://{host}:{port}" address = f"http://{host}:{port}"
return address return address, False
def _create_session(self, *, use_unix_socket: bool) -> requests.Session:
"""
generate new request session
Args:
use_unix_socket(bool): if set to True then unix socket session will be generated instead of native requests
Returns:
requests.Session: generated session object
"""
if use_unix_socket:
import requests_unixsocket # type: ignore
session: requests.Session = requests_unixsocket.Session()
return session
session = requests.Session()
self._login()
return session
def _login(self) -> None: def _login(self) -> None:
""" """

View File

@ -19,6 +19,7 @@
# #
import aiohttp_security # type: ignore import aiohttp_security # type: ignore
import base64 import base64
import socket
import types import types
from aiohttp import web from aiohttp import web
@ -101,7 +102,11 @@ def auth_handler(allow_read_only: bool) -> MiddlewareType:
""" """
@middleware @middleware
async def handle(request: Request, handler: HandlerType) -> StreamResponse: async def handle(request: Request, handler: HandlerType) -> StreamResponse:
if (permission_method := getattr(handler, "get_permission", None)) is not None: if (unix_socket := request.get_extra_info("socket")) is not None and unix_socket.family == socket.AF_UNIX:
# special case for unix sockets. We need to extract socket which is used for the request
# and check its address family
permission = UserAccess.Unauthorized
elif (permission_method := getattr(handler, "get_permission", None)) is not None:
permission = await permission_method(request) permission = await permission_method(request)
elif isinstance(handler, types.MethodType): # additional wrapper for static resources elif isinstance(handler, types.MethodType): # additional wrapper for static resources
handler_instance = getattr(handler, "__self__", None) handler_instance = getattr(handler, "__self__", None)

View File

@ -78,8 +78,9 @@ def run_server(application: web.Application) -> None:
configuration: Configuration = application["configuration"] configuration: Configuration = application["configuration"]
host = configuration.get("web", "host") host = configuration.get("web", "host")
port = configuration.getint("web", "port") port = configuration.getint("web", "port")
unix_socket = configuration.get("web", "unix_socket", fallback=None)
web.run_app(application, host=host, port=port, handle_signals=False, web.run_app(application, host=host, port=port, path=unix_socket, handle_signals=False,
access_log=logging.getLogger("http"), access_log_class=FilteredAccessLogger) access_log=logging.getLogger("http"), access_log_class=FilteredAccessLogger)

View File

@ -1,17 +1,16 @@
import pytest import pytest
from collections import namedtuple from unittest.mock import MagicMock
_passwd = namedtuple("passwd", ["pw_dir"])
@pytest.fixture @pytest.fixture
def passwd() -> _passwd: def passwd() -> MagicMock:
""" """
get passwd structure for the user get passwd structure for the user
Returns: Returns:
_passwd: passwd structure test instance MagicMock: passwd structure test instance
""" """
return _passwd("home") passwd = MagicMock()
passwd.pw_dir = "home"
return passwd

View File

@ -31,6 +31,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
args.sign_key = "key" args.sign_key = "key"
args.sign_target = [SignSettings.Packages] args.sign_target = [SignSettings.Packages]
args.web_port = 8080 args.web_port = 8080
args.web_unix_socket = Path("/var/lib/ahriman/ahriman-web.sock")
return args return args
@ -91,6 +92,7 @@ def test_configuration_create_ahriman(args: argparse.Namespace, configuration: C
" ".join([target.name.lower() for target in args.sign_target])), " ".join([target.name.lower() for target in args.sign_target])),
MockCall(Configuration.section_name("sign", "x86_64"), "key", args.sign_key), MockCall(Configuration.section_name("sign", "x86_64"), "key", args.sign_key),
MockCall(Configuration.section_name("web", "x86_64"), "port", str(args.web_port)), MockCall(Configuration.section_name("web", "x86_64"), "port", str(args.web_port)),
MockCall(Configuration.section_name("web", "x86_64"), "unix_socket", str(args.web_unix_socket)),
]) ])
write_mock.assert_called_once_with(pytest.helpers.anyvar(int)) write_mock.assert_called_once_with(pytest.helpers.anyvar(int))

View File

@ -26,7 +26,6 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
""" """
args.username = "user" args.username = "user"
args.action = Action.Update args.action = Action.Update
args.as_service = False
args.exit_code = False args.exit_code = False
args.password = "pa55w0rd" args.password = "pa55w0rd"
args.role = UserAccess.Reporter args.role = UserAccess.Reporter
@ -73,33 +72,8 @@ def test_run_empty_salt(args: argparse.Namespace, configuration: Configuration,
Users.run(args, "x86_64", configuration, report=False, unsafe=False) Users.run(args, "x86_64", configuration, report=False, unsafe=False)
get_auth_configuration_mock.assert_called_once_with(configuration.include) get_auth_configuration_mock.assert_called_once_with(configuration.include)
create_configuration_mock.assert_called_once_with(pytest.helpers.anyvar(int), pytest.helpers.anyvar(int), create_configuration_mock.assert_called_once_with(
pytest.helpers.anyvar(int), args.as_service, args.secure) pytest.helpers.anyvar(int), pytest.helpers.anyvar(int), args.secure)
create_user_mock.assert_called_once_with(args)
get_salt_mock.assert_called_once_with(configuration)
update_mock.assert_called_once_with(user)
def test_run_service_user(args: argparse.Namespace, configuration: Configuration, database: SQLite,
mocker: MockerFixture) -> None:
"""
must create configuration if as service argument is provided
"""
args = _default_args(args)
args.as_service = True
user = User(username=args.username, password=args.password, access=args.role)
mocker.patch("ahriman.core.database.SQLite.load", return_value=database)
mocker.patch("ahriman.models.user.User.hash_password", return_value=user)
get_auth_configuration_mock = mocker.patch("ahriman.application.handlers.Users.configuration_get")
create_configuration_mock = mocker.patch("ahriman.application.handlers.Users.configuration_create")
create_user_mock = mocker.patch("ahriman.application.handlers.Users.user_create", return_value=user)
get_salt_mock = mocker.patch("ahriman.application.handlers.Users.get_salt", return_value=("salt", "salt"))
update_mock = mocker.patch("ahriman.core.database.SQLite.user_update")
Users.run(args, "x86_64", configuration, report=False, unsafe=False)
get_auth_configuration_mock.assert_called_once_with(configuration.include)
create_configuration_mock.assert_called_once_with(pytest.helpers.anyvar(int), pytest.helpers.anyvar(int),
pytest.helpers.anyvar(int), args.as_service, args.secure)
create_user_mock.assert_called_once_with(args) create_user_mock.assert_called_once_with(args)
get_salt_mock.assert_called_once_with(configuration) get_salt_mock.assert_called_once_with(configuration)
update_mock.assert_called_once_with(user) update_mock.assert_called_once_with(user)
@ -151,7 +125,7 @@ def test_run_remove(args: argparse.Namespace, configuration: Configuration, data
remove_mock.assert_called_once_with(args.username) remove_mock.assert_called_once_with(args.username)
def test_configuration_create(configuration: Configuration, user: User, mocker: MockerFixture) -> None: def test_configuration_create(configuration: Configuration, mocker: MockerFixture) -> None:
""" """
must correctly create configuration file must correctly create configuration file
""" """
@ -159,26 +133,11 @@ def test_configuration_create(configuration: Configuration, user: User, mocker:
set_mock = mocker.patch("ahriman.core.configuration.Configuration.set_option") set_mock = mocker.patch("ahriman.core.configuration.Configuration.set_option")
write_mock = mocker.patch("ahriman.application.handlers.Users.configuration_write") write_mock = mocker.patch("ahriman.application.handlers.Users.configuration_write")
Users.configuration_create(configuration, user, "salt", False, False) Users.configuration_create(configuration, "salt", False)
set_mock.assert_called_once_with("auth", "salt", pytest.helpers.anyvar(int)) set_mock.assert_called_once_with("auth", "salt", pytest.helpers.anyvar(int))
write_mock.assert_called_once_with(configuration, False) write_mock.assert_called_once_with(configuration, False)
def test_configuration_create_with_plain_password(configuration: Configuration, user: User,
mocker: MockerFixture) -> None:
"""
must set plain text password and user for the service
"""
mocker.patch("pathlib.Path.open")
Users.configuration_create(configuration, user, "salt", True, False)
generated = User.from_option(user.username, user.password).hash_password("salt")
service = User.from_option(configuration.get("web", "username"), configuration.get("web", "password"))
assert generated.username == service.username
assert generated.check_credentials(service.password, configuration.get("auth", "salt"))
def test_configuration_get(mocker: MockerFixture) -> None: def test_configuration_get(mocker: MockerFixture) -> None:
""" """
must load configuration from filesystem must load configuration from filesystem

View File

@ -43,6 +43,14 @@ def test_load_full_client_from_address(configuration: Configuration) -> None:
assert isinstance(Client.load(configuration, report=True), WebClient) assert isinstance(Client.load(configuration, report=True), WebClient)
def test_load_full_client_from_unix_socket(configuration: Configuration) -> None:
"""
must load full client by using unix socket
"""
configuration.set_option("web", "unix_socket", "/var/lib/ahriman/ahriman-web.sock")
assert isinstance(Client.load(configuration, report=True), WebClient)
def test_add(client: Client, package_ahriman: Package) -> None: def test_add(client: Client, package_ahriman: Package) -> None:
""" """
must process package addition without errors must process package addition without errors

View File

@ -2,6 +2,7 @@ import json
import logging import logging
import pytest import pytest
import requests import requests
import requests_unixsocket
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
from requests import Response from requests import Response
@ -36,10 +37,36 @@ def test_parse_address(configuration: Configuration) -> None:
""" """
configuration.set_option("web", "host", "localhost") configuration.set_option("web", "host", "localhost")
configuration.set_option("web", "port", "8080") configuration.set_option("web", "port", "8080")
assert WebClient.parse_address(configuration) == "http://localhost:8080" assert WebClient.parse_address(configuration) == ("http://localhost:8080", False)
configuration.set_option("web", "address", "http://localhost:8081") configuration.set_option("web", "address", "http://localhost:8081")
assert WebClient.parse_address(configuration) == "http://localhost:8081" assert WebClient.parse_address(configuration) == ("http://localhost:8081", False)
configuration.set_option("web", "unix_socket", "/run/ahriman.sock")
assert WebClient.parse_address(configuration) == ("http+unix://%2Frun%2Fahriman.sock", True)
def test_create_session(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must create normal requests session
"""
login_mock = mocker.patch("ahriman.core.status.web_client.WebClient._login")
session = web_client._create_session(use_unix_socket=False)
assert isinstance(session, requests.Session)
assert not isinstance(session, requests_unixsocket.Session)
login_mock.assert_called_once_with()
def test_create_session_unix_socket(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must create unix socket session
"""
login_mock = mocker.patch("ahriman.core.status.web_client.WebClient._login")
session = web_client._create_session(use_unix_socket=True)
assert isinstance(session, requests_unixsocket.Session)
login_mock.assert_not_called()
def test_login(web_client: WebClient, user: User, mocker: MockerFixture) -> None: def test_login(web_client: WebClient, user: User, mocker: MockerFixture) -> None:

View File

@ -1,6 +1,5 @@
import pytest import pytest
from collections import namedtuple
from typing import Any, Dict, List from typing import Any, Dict, List
from unittest.mock import MagicMock from unittest.mock import MagicMock
@ -10,9 +9,6 @@ from ahriman.core.upload.rsync import Rsync
from ahriman.core.upload.s3 import S3 from ahriman.core.upload.s3 import S3
_s3_object = namedtuple("s3_object", ["key", "e_tag", "delete"])
@pytest.fixture @pytest.fixture
def github(configuration: Configuration) -> Github: def github(configuration: Configuration) -> Github:
""" """
@ -78,12 +74,22 @@ def s3(configuration: Configuration) -> S3:
@pytest.fixture @pytest.fixture
def s3_remote_objects() -> List[_s3_object]: def s3_remote_objects() -> List[MagicMock]:
""" """
fixture for boto3 like S3 objects fixture for boto3 like S3 objects
Returns: Returns:
List[_s3_object]: boto3 like S3 objects test instance List[MagicMock]: boto3 like S3 objects test instance
""" """
delete_mock = MagicMock() delete_mock = MagicMock()
return list(map(lambda item: _s3_object(f"x86_64/{item}", f"\"{item}\"", delete_mock), ["a", "b", "c"]))
result = []
for item in ["a", "b", "c"]:
s3_object = MagicMock()
s3_object.key = f"x86_64/{item}"
s3_object.e_tag = f"\"{item}\""
s3_object.delete = delete_mock
result.append(s3_object)
return result

View File

@ -3,9 +3,8 @@ import pytest
from asyncio import BaseEventLoop from asyncio import BaseEventLoop
from aiohttp import web from aiohttp import web
from aiohttp.test_utils import TestClient from aiohttp.test_utils import TestClient
from collections import namedtuple
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
from typing import Any from typing import Any, Dict, Optional
from unittest.mock import MagicMock from unittest.mock import MagicMock
import ahriman.core.auth.helpers import ahriman.core.auth.helpers
@ -18,11 +17,9 @@ from ahriman.models.user import User
from ahriman.web.web import setup_service from ahriman.web.web import setup_service
_request = namedtuple("_request", ["app", "path", "method", "json", "post"])
@pytest.helpers.register @pytest.helpers.register
def request(app: web.Application, path: str, method: str, json: Any = None, data: Any = None) -> _request: def request(app: web.Application, path: str, method: str, json: Any = None, data: Any = None,
extra: Optional[Dict[str, Any]] = None) -> MagicMock:
""" """
request generator helper request generator helper
@ -32,11 +29,22 @@ def request(app: web.Application, path: str, method: str, json: Any = None, data
method(str): method for the request method(str): method for the request
json(Any, optional): json payload of the request (Default value = None) json(Any, optional): json payload of the request (Default value = None)
data(Any, optional): form data payload of the request (Default value = None) data(Any, optional): form data payload of the request (Default value = None)
extra(Optional[Dict[str, Any]], optional): extra info which will be injected for ``get_extra_info`` command
Returns: Returns:
_request: dummy request object MagicMock: dummy request mock
""" """
return _request(app, path, method, json, data) request_mock = MagicMock()
request_mock.app = app
request_mock.path = path
request_mock.method = method
request_mock.json = json
request_mock.post = data
extra = extra or {}
request_mock.get_extra_info.side_effect = lambda key: extra.get(key)
return request_mock
@pytest.fixture @pytest.fixture

View File

@ -1,4 +1,5 @@
import pytest import pytest
import socket
from aiohttp import web from aiohttp import web
from aiohttp.test_utils import TestClient from aiohttp.test_utils import TestClient
@ -55,6 +56,21 @@ async def test_permits(authorization_policy: AuthorizationPolicy, user: User) ->
assert not await authorization_policy.permits(user.username, user.access, "/endpoint") assert not await authorization_policy.permits(user.username, user.access, "/endpoint")
async def test_auth_handler_unix_socket(client_with_auth: TestClient, mocker: MockerFixture) -> None:
"""
must allow calls via unix sockets
"""
aiohttp_request = pytest.helpers.request(
"", "/api/v1/status", "GET", extra={"socket": socket.socket(socket.AF_UNIX)})
request_handler = AsyncMock()
request_handler.get_permission.return_value = UserAccess.Full
check_permission_mock = mocker.patch("aiohttp_security.check_permission")
handler = auth_handler(allow_read_only=False)
await handler(aiohttp_request, request_handler)
check_permission_mock.assert_not_called()
async def test_auth_handler_api(mocker: MockerFixture) -> None: async def test_auth_handler_api(mocker: MockerFixture) -> None:
""" """
must ask for status permission for api calls must ask for status permission for api calls

View File

@ -50,7 +50,7 @@ def test_run(application: web.Application, mocker: MockerFixture) -> None:
run_server(application) run_server(application)
run_application_mock.assert_called_once_with( run_application_mock.assert_called_once_with(
application, host="127.0.0.1", port=port, handle_signals=False, application, host="127.0.0.1", port=port, path=None, handle_signals=False,
access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger
) )
@ -65,7 +65,7 @@ def test_run_with_auth(application_with_auth: web.Application, mocker: MockerFix
run_server(application_with_auth) run_server(application_with_auth)
run_application_mock.assert_called_once_with( run_application_mock.assert_called_once_with(
application_with_auth, host="127.0.0.1", port=port, handle_signals=False, application_with_auth, host="127.0.0.1", port=port, path=None, handle_signals=False,
access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger
) )
@ -80,6 +80,23 @@ def test_run_with_debug(application_with_debug: web.Application, mocker: MockerF
run_server(application_with_debug) run_server(application_with_debug)
run_application_mock.assert_called_once_with( run_application_mock.assert_called_once_with(
application_with_debug, host="127.0.0.1", port=port, handle_signals=False, application_with_debug, host="127.0.0.1", port=port, path=None, handle_signals=False,
access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger
)
def test_run_with_socket(application: web.Application, mocker: MockerFixture) -> None:
"""
must run application
"""
port = 8080
socket = "/run/ahriman.sock"
application["configuration"].set_option("web", "port", str(port))
application["configuration"].set_option("web", "unix_socket", socket)
run_application_mock = mocker.patch("aiohttp.web.run_app")
run_server(application)
run_application_mock.assert_called_once_with(
application, host="127.0.0.1", port=port, path=socket, handle_signals=False,
access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger
) )