Compare commits

..

15 Commits
2.0.0 ... 2.2.1

Author SHA1 Message Date
4f6bd29ff4 Release 2.2.1 2022-09-14 04:49:08 +03:00
f6d9ea480a docs update 2022-09-14 04:48:11 +03:00
08f62842ba Release 2.2.0 2022-09-14 03:28:28 +03:00
1912790ccc Make optional dependencies trully optional (#67)
The issue appears when there is no boto, jinja and some other libraries
are not installed because the classes which use these libraries are
still being imported inside the package file. The fix removes those
imports from package root, because they should not be here, in fact,
content of report and upload packages must be imported only inside the
trigger class and only if they are actually required

This commit also adds setuptools as required dependency since it is used
for some parsers (previously it was provided dependency)
2022-09-11 01:44:06 +03:00
cf3c48ffeb patch architecture list in runtime (#66) 2022-08-09 15:18:20 +03:00
6633766cc3 frozen dataclasses 2022-07-26 14:40:28 +03:00
f73d1eb424 reduce docker image size a bit 2022-07-18 11:42:26 +03:00
87ab3683ad Release 2.1.0 2022-07-02 15:42:39 +03:00
9c2f73af8c simplify tmpdir method 2022-06-28 19:11:38 +03:00
31f551bdf2 review loggers once more
This commit makes loggers like java.util.logging with fully qualified
logger name which is created by LazyLogging trait
2022-06-28 11:00:45 +03:00
cbbe2b01e4 fix case with package name which cannot be downloaded
(without special settings)

The issue appears if file or its version contains one of special URI
characters, e.g. +. Theu will be interpreted as query parameters by
(some) servers (e.g. S3 works in this way). In this commit we rename
archive to the one with safe name.
2022-06-27 18:53:48 +03:00
cd361a483d review loggers once more
This commit makes loggers like java.util.logging with fully qualified
logger name which is created by LazyLogging trait
2022-06-27 01:41:49 +03:00
cee4fd4cce bump web libraries
also encode strings for the views
2022-06-24 12:03:38 +03:00
18daecaac7 review loggers 2022-06-21 11:48:42 +03:00
03c298c762 replace if with while for telelgram reporting 2022-06-10 12:02:00 +03:00
115 changed files with 3815 additions and 3401 deletions

View File

@ -1 +1 @@
skips: ['B101', 'B105', 'B404']
skips: ['B101', 'B105', 'B106', 'B404']

View File

@ -8,7 +8,7 @@ echo -e '[arcanisrepo]\nServer = http://repo.arcanis.me/$arch\nSigLevel = Never'
# refresh the image
pacman --noconfirm -Syu
# main dependencies
pacman --noconfirm -Sy base-devel devtools git pyalpm python-aur python-passlib python-srcinfo sudo
pacman --noconfirm -Sy base-devel devtools git pyalpm python-aur python-passlib python-setuptools python-srcinfo sudo
# make dependencies
pacman --noconfirm -Sy python-build python-installer python-wheel
# optional dependencies

View File

@ -4,7 +4,7 @@
set -ex
# install dependencies
pacman --noconfirm -Syu base-devel python-pip python-tox
pacman --noconfirm -Syu base-devel python-pip python-setuptools python-tox
# run test and check targets
make check tests

View File

@ -31,7 +31,7 @@ Again, the most checks can be performed by `make check` command, though some add
* In case if class load requires some actions, it is recommended to create class method which can be used for class instantiating.
* The code must follow the exception safety, unless it is explicitly asked by end user. It means that most exceptions must be handled and printed to log, no other actions must be done (e.g. raising another exception).
* For the external command `ahriman.core.util.check_output` function must be used.
* Every temporary file/directory must be removed at the end of processing, no matter what. The `ahriman.core.util.tmpdir` function provides wrapper for the directories.
* Every temporary file/directory must be removed at the end of processing, no matter what. The ``tempfile`` module provides good ways to do it.
* Import order must be the following:
```python
@ -60,6 +60,7 @@ Again, the most checks can be performed by `make check` command, though some add
* The file size mentioned above must be applicable in general. In case of big classes consider splitting them into traits.
* No global variable allowed outside of `ahriman.version` module.
* Single quotes are not allowed. The reason behind this restriction is the fact that docstrings must be written by using double quotes only, and we would like to make style consistent.
* If your class writes anything to log, the `ahriman.core.lazy_logging.LazyLogging` trait must be used.
### Other checks

View File

@ -1,4 +1,4 @@
FROM archlinux:base-devel
FROM archlinux:base
# image configuration
ENV AHRIMAN_ARCHITECTURE="x86_64"
@ -13,24 +13,22 @@ ENV AHRIMAN_REPOSITORY_ROOT="/var/lib/ahriman/ahriman"
ENV AHRIMAN_USER="ahriman"
# install environment
## install git which is required for AUR interaction and go for yay
RUN pacman --noconfirm -Syu git go
## install minimal required packages
RUN pacman --noconfirm -Syu binutils fakeroot git make sudo
## create build user
RUN useradd -m -d /home/build -s /usr/bin/nologin build && \
echo "build ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/build
## install AUR helper
RUN YAY_DIR="$(runuser -u build -- mktemp -d)" && \
git clone https://aur.archlinux.org/yay.git "$YAY_DIR" && \
cd "$YAY_DIR" && \
runuser -u build -- makepkg --noconfirm --install && \
cd - && rm -r "$YAY_DIR"
COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package"
## install package dependencies
RUN runuser -u build -- yay --noconfirm -Sy devtools git pyalpm python-inflection python-passlib python-requests python-srcinfo && \
runuser -u build -- yay --noconfirm -Sy python-build python-installer python-wheel && \
runuser -u build -- yay --noconfirm -Sy breezy darcs mercurial python-aioauth-client python-aiohttp \
python-aiohttp-debugtoolbar python-aiohttp-jinja2 python-aiohttp-security \
python-aiohttp-session python-boto3 python-cryptography python-jinja \
rsync subversion
## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size
RUN pacman --noconfirm -Sy devtools git pyalpm python-inflection python-passlib python-requests python-setuptools python-srcinfo && \
pacman --noconfirm -Sy python-build python-installer python-wheel && \
pacman --noconfirm -Sy breezy mercurial python-aiohttp python-boto3 python-cryptography python-jinja rsync subversion && \
runuser -u build -- install-aur-package python-aioauth-client python-aiohttp-jinja2 python-aiohttp-debugtoolbar \
python-aiohttp-session python-aiohttp-security
# cleanup unused
RUN find "/var/cache/pacman/pkg" -type f -delete
# install ahriman
## copy tree
@ -41,7 +39,7 @@ RUN cd "/home/build/ahriman" && \
cp ./*-src.tar.xz "package/archlinux" && \
cd "package/archlinux" && \
runuser -u build -- makepkg --noconfirm --install --skipchecksums && \
cd - && rm -r "/home/build/ahriman"
cd / && rm -r "/home/build/ahriman"
VOLUME ["/var/lib/ahriman"]

View File

@ -2,7 +2,7 @@
[![tests status](https://github.com/arcan1s/ahriman/actions/workflows/run-tests.yml/badge.svg)](https://github.com/arcan1s/ahriman/actions/workflows/run-tests.yml)
[![setup status](https://github.com/arcan1s/ahriman/actions/workflows/run-setup.yml/badge.svg)](https://github.com/arcan1s/ahriman/actions/workflows/run-setup.yml)
[![docker image](https://github.com/arcan1s/ahriman/actions/workflows/docker-image.yml/badge.svg)](https://github.com/arcan1s/ahriman/actions/workflows/docker-image.yml)
[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/arcan1s/ahriman?label=docker%20image)](https://hub.docker.com/r/arcan1s/ahriman)
[![CodeFactor](https://www.codefactor.io/repository/github/arcan1s/ahriman/badge)](https://www.codefactor.io/repository/github/arcan1s/ahriman)
[![Documentation Status](https://readthedocs.org/projects/ahriman/badge/?version=latest)](https://ahriman.readthedocs.io/?badge=latest)
@ -30,6 +30,6 @@ For installation details kindly refer to the [documentation](https://ahriman.rea
Every available option is described in the [documentation](https://ahriman.readthedocs.io/en/latest/configuration.html).
The application provides reasonable defaults which allow to use it out-of-box, though additional steps (like configuring build toolchain and sudoers) is recommended and can be easily achieved by following install instructions.
The application provides reasonable defaults which allow to use it out-of-box; however additional steps (like configuring build toolchain and sudoers) are recommended and can be easily achieved by following install instructions.
## [FAQ](https://ahriman.readthedocs.io/en/latest/faq.html)

View File

@ -33,7 +33,7 @@ fi
ahriman "${AHRIMAN_DEFAULT_ARGS[@]}" repo-setup "${AHRIMAN_SETUP_ARGS[@]}"
# refresh database
runuser -u build -- yay --noconfirm -Syy &> /dev/null
pacman -Syy &> /dev/null
# create machine-id which is required by build tools
systemd-machine-id-setup &> /dev/null

12
docker/install-aur-package.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash
set -e
for PACKAGE in "$@"; do
BUILD_DIR="$(mktemp -d)"
git clone https://aur.archlinux.org/"$PACKAGE".git "$BUILD_DIR"
cd "$BUILD_DIR"
makepkg --noconfirm --install --rmdeps --syncdeps
cd /
rm -r "$BUILD_DIR"
done

File diff suppressed because it is too large Load Diff

Before

Width:  |  Height:  |  Size: 522 KiB

After

Width:  |  Height:  |  Size: 537 KiB

View File

@ -38,6 +38,14 @@ ahriman.core.exceptions module
:no-undoc-members:
:show-inheritance:
ahriman.core.lazy\_logging module
---------------------------------
.. automodule:: ahriman.core.lazy_logging
:members:
:no-undoc-members:
:show-inheritance:
ahriman.core.spawn module
-------------------------

View File

@ -92,6 +92,14 @@ ahriman.models.package\_source module
:no-undoc-members:
:show-inheritance:
ahriman.models.pkgbuild\_patch module
-------------------------------------
.. automodule:: ahriman.models.pkgbuild_patch
:members:
:no-undoc-members:
:show-inheritance:
ahriman.models.property module
------------------------------

View File

@ -138,6 +138,7 @@ Section name must be either ``telegram`` (plus optional architecture name, e.g.
* ``link_path`` - prefix for HTML links, string, required.
* ``template_path`` - path to Jinja2 template, string, required.
* ``template_type`` - ``parse_mode`` to be passed to telegram API, one of ``MarkdownV2``, ``HTML``, ``Markdown``, string, optional, default ``HTML``.
* ``timeout`` - HTTP request timeout in seconds, int, optional, default is ``30``.
``upload`` group
----------------
@ -167,6 +168,7 @@ This feature requires Github key creation (see below). Section name must be eith
#. Generate new token. Required scope is ``public_repo`` (or ``repo`` for private repository support).
* ``repository`` - Github repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme).
* ``timeout`` - HTTP request timeout in seconds, int, optional, default is ``30``.
* ``username`` - Github authorization user, string, required. Basically the same as ``owner``.
``rsync`` type

View File

@ -240,7 +240,7 @@ The default action (in case if no arguments provided) is ``repo-update``. Basica
docker run -v /path/to/local/repo:/var/lib/ahriman -v /etc/ahriman.ini:/etc/ahriman.ini.d/10-overrides.ini arcan1s/ahriman:latest
By default, it runs ``repo-update``, but it can be overwritten to any other command you would like to, e.g.:
The action can be specified during run, e.g.:
.. code-block:: shell
@ -692,7 +692,7 @@ You can also edit configuration and forward logs to ``stderr``, just change ``ha
sed -i 's/handlers = syslog_handler/handlers = console_handler/g' /etc/ahriman.ini.d/logging.ini
You can even configure logging as you wish, but kindly refer to python ``logging`` module `configuration <https://docs.python.org/3/library/logging.config.html>`_.
You can even configure logging as you wish, but kindly refer to python ``logging`` module `configuration <https://docs.python.org/3/library/logging.config.html>`_. The application uses java concept to log messages, e.g. class ``Application`` imported from ``ahriman.application.application`` package will have logger called ``ahriman.application.application.Application``. In order to e.g. change logger name for whole application package it is possible to change values for ``ahriman.application`` package; thus editing ``ahriman`` logger configuration will change logging for whole application (unless there are overrides for another logger).
Html customization
^^^^^^^^^^^^^^^^^^

View File

@ -30,10 +30,3 @@ Contents
advanced-usage
triggers
modules
Indices and tables
------------------
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

View File

@ -1,13 +1,13 @@
# Maintainer: Evgeniy Alekseev
pkgname='ahriman'
pkgver=2.0.0
pkgver=2.2.1
pkgrel=1
pkgdesc="ArcH linux ReposItory MANager"
arch=('any')
url="https://github.com/arcan1s/ahriman"
license=('GPL3')
depends=('devtools' 'git' 'pyalpm' 'python-inflection' 'python-passlib' 'python-requests' 'python-srcinfo')
depends=('devtools' 'git' 'pyalpm' 'python-inflection' 'python-passlib' 'python-requests' 'python-setuptools' 'python-srcinfo')
makedepends=('python-build' 'python-installer' 'python-wheel')
optdepends=('breezy: -bzr packages support'
'darcs: -darcs packages support'

View File

@ -20,7 +20,7 @@ archbuild_flags =
build_command = extra-x86_64-build
ignore_packages =
makechrootpkg_flags =
makepkg_flags = --nocolor
makepkg_flags = --nocolor --ignorearch
triggers = ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger
[repository]

View File

@ -1,5 +1,5 @@
[loggers]
keys = root,build_details,database,http,stderr,boto3,botocore,nose,s3transfer
keys = root,http,stderr,boto3,botocore,nose,s3transfer
[handlers]
keys = console_handler,syslog_handler
@ -20,11 +20,11 @@ formatter = syslog_format
args = ("/dev/log",)
[formatter_generic_format]
format = [%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d %(funcName)s]: %(message)s
format = [%(levelname)s %(asctime)s] [%(threadName)s] [%(name)s]: %(message)s
datefmt =
[formatter_syslog_format]
format = [%(levelname)s] [%(name)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s
format = [%(levelname)s] [%(threadName)s] [%(name)s]: %(message)s
datefmt =
[logger_root]
@ -32,18 +32,6 @@ level = DEBUG
handlers = syslog_handler
qualname = root
[logger_build_details]
level = DEBUG
handlers = syslog_handler
qualname = build_details
propagate = 0
[logger_database]
level = DEBUG
handlers = syslog_handler
qualname = database
propagate = 0
[logger_http]
level = DEBUG
handlers = syslog_handler

View File

@ -50,12 +50,12 @@
});
function addPackages() {
const packages = [packageInput.val()]
const packages = [packageInput.val()];
doPackageAction("/api/v1/service/add", packages);
}
function requestPackages() {
const packages = [packageInput.val()]
const packages = [packageInput.val()];
doPackageAction("/api/v1/service/request", packages);
}
</script>

View File

@ -32,7 +32,7 @@
showSuccess(details);
},
error: (jqXHR, _, errorThrown) => { showFailure(errorThrown); },
})
});
}
function getSelection() {
@ -58,25 +58,30 @@
dataType: "json",
success: response => {
const extractListProperties = (description, property) => {
return Object.values(description.packages).map(pkg => {
return pkg[property];
}).reduce((left, right) => { return left.concat(right); }, []);
return Object.values(description.packages)
.map(pkg => { return pkg[property]; })
.reduce((left, right) => { return left.concat(right); }, []);
};
const listToTable = data => {
return Array.from(new Set(data))
.sort()
.map(entry => { return safe(entry); })
.join("<br>");
};
const listToTable = data => { return Array.from(new Set(data)).sort().join("<br>"); };
const payload = response.map(description => {
const package_base = description.package.base;
const web_url = description.package.remote?.web_url;
return {
id: description.package.base,
base: web_url ? `<a href="${web_url}" title="${package_base}">${package_base}</a>` : package_base,
version: description.package.version,
id: package_base,
base: web_url ? `<a href="${safe(web_url)}" title="${safe(package_base)}">${safe(package_base)}</a>` : safe(package_base),
version: safe(description.package.version),
packages: listToTable(Object.keys(description.package.packages)),
groups: listToTable(extractListProperties(description.package, "groups")),
licenses: listToTable(extractListProperties(description.package, "licenses")),
timestamp: new Date(1000 * description.status.timestamp).toISOString(),
status: description.status.status
}
status: description.status.status,
};
});
table.bootstrapTable("load", payload);
@ -85,17 +90,17 @@
hideControls(false);
},
error: (jqXHR, _, errorThrown) => {
hideControls(true);
if ((jqXHR.status === 401) || (jqXHR.status === 403)) {
// authorization error
const text = "In order to see statuses you must login first.";
table.find("tr.unauthorized").remove();
table.find("tbody").append(`<tr class="unauthorized"><td colspan="100%">${text}</td></tr>`);
table.find("tbody").append(`<tr class="unauthorized"><td colspan="100%">${safe(text)}</td></tr>`);
table.bootstrapTable("hideLoading");
} else {
// other errors
showFailure(errorThrown);
}
hideControls(true);
},
});
@ -129,6 +134,14 @@
});
}
function safe(string) {
return String(string)
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;");
}
function statusFormat(value) {
const cellClass = status => {
if (status === "pending") return "table-warning";
@ -143,5 +156,5 @@
$(() => {
table.bootstrapTable({});
reload();
})
});
</script>

View File

@ -5,19 +5,8 @@
<script src="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-ka7Sk0Gln4gmtz2MlQnikT1wXgYsOg+OMhuP+IlRH9sENBO0LRn5q+8nbTov4+1p" crossorigin="anonymous"></script>
<script src="https://unpkg.com/bootstrap-table@1.19.1/dist/bootstrap-table.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.20.2/dist/bootstrap-table.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.19.1/dist/extensions/export/bootstrap-table-export.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.20.2/dist/extensions/export/bootstrap-table-export.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.19.1/dist/extensions/resizable/bootstrap-table-resizable.js"></script>
<script>
$("#packages").bootstrapTable({
formatClearSearch: function () {
return "Clear search";
},
formatSearch: function () {
return "search";
}
})
</script>
<script src="https://unpkg.com/bootstrap-table@1.20.2/dist/extensions/resizable/bootstrap-table-resizable.js"></script>

View File

@ -1,7 +1,7 @@
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous" type="text/css">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.7.2/font/bootstrap-icons.css" type="text/css">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.8.3/font/bootstrap-icons.css" type="text/css">
<link rel="stylesheet" href="https://unpkg.com/bootstrap-table@1.19.1/dist/bootstrap-table.min.css" type="text/css">
<link rel="stylesheet" href="https://unpkg.com/bootstrap-table@1.20.2/dist/bootstrap-table.min.css" type="text/css">
<link href="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.css" rel="stylesheet">

View File

@ -32,6 +32,7 @@ setup(
"inflection",
"passlib",
"requests",
"setuptools",
"srcinfo",
],
setup_requires=[

View File

@ -21,11 +21,12 @@ import requests
import shutil
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Iterable, Set
from ahriman.application.application.application_properties import ApplicationProperties
from ahriman.core.build_tools.sources import Sources
from ahriman.core.util import package_like, tmpdir
from ahriman.core.util import package_like
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
from ahriman.models.result import Result
@ -85,7 +86,8 @@ class ApplicationPackages(ApplicationProperties):
self.database.build_queue_insert(package)
self.database.remote_update(package)
with tmpdir() as local_dir:
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, \
(local_dir := Path(dir_name)): # pylint: disable=confusing-with-statement
Sources.load(local_dir, package, self.database.patches_get(package.base), self.repository.paths)
self._process_dependencies(local_dir, known_packages, without_dependencies)
@ -127,7 +129,7 @@ class ApplicationPackages(ApplicationProperties):
source(str): remote URL of the package archive
"""
dst = self.repository.paths.packages / Path(source).name # URL is path, is not it?
response = requests.get(source, stream=True)
response = requests.get(source, stream=True, timeout=None) # timeout=None to suppress pylint warns
response.raise_for_status()
with dst.open("wb") as local_file:

View File

@ -17,14 +17,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.repository import Repository
class ApplicationProperties:
class ApplicationProperties(LazyLogging):
"""
application base properties class
@ -32,7 +31,6 @@ class ApplicationProperties:
architecture(str): repository architecture
configuration(Configuration): configuration instance
database(SQLite): database instance
logger(logging.Logger): application logger
repository(Repository): repository instance
"""
@ -44,9 +42,8 @@ class ApplicationProperties:
architecture(str): repository architecture
configuration(Configuration): configuration instance
no_report(bool): force disable reporting
unsafe(bool): if set no user check will be performed before path creation
unsafe(bool): if set no user check will be performed before path creation
"""
self.logger = logging.getLogger("root")
self.configuration = configuration
self.architecture = architecture
self.database = SQLite.load(configuration)

View File

@ -52,7 +52,7 @@ class Rebuild(Handler):
if args.from_database:
updates = Rebuild.extract_packages(application)
else:
updates = application.repository.packages_depends_on(depends_on)
updates = application.repository.packages_depend_on(depends_on)
Rebuild.check_if_empty(args.exit_code, not updates)
if args.dry_run:

View File

@ -149,7 +149,7 @@ class Users(Handler):
Returns:
User: built user descriptor
"""
user = User(args.username, args.password, args.role)
if user.password is None:
user.password = getpass.getpass()
return user
password = args.password
if password is None:
password = getpass.getpass()
return User(username=args.username, password=password, access=args.role)

View File

@ -20,7 +20,6 @@
from __future__ import annotations
import argparse
import logging
from pathlib import Path
from types import TracebackType
@ -29,12 +28,13 @@ from typing import Literal, Optional, Type
from ahriman import version
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import DuplicateRun
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum
class Lock:
class Lock(LazyLogging):
"""
wrapper for application lock file
@ -115,10 +115,8 @@ class Lock:
"""
status = self.reporter.get_internal()
if status.version is not None and status.version != version.__version__:
logging.getLogger("root").warning(
"status watcher version mismatch, our %s, their %s",
version.__version__,
status.version)
self.logger.warning("status watcher version mismatch, our %s, their %s",
version.__version__, status.version)
def check_user(self) -> None:
"""

View File

@ -36,11 +36,13 @@ class AUR(Remote):
DEFAULT_AUR_URL(str): (class attribute) default AUR url
DEFAULT_RPC_URL(str): (class attribute) default AUR RPC url
DEFAULT_RPC_VERSION(str): (class attribute) default AUR RPC version
DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds
"""
DEFAULT_AUR_URL = "https://aur.archlinux.org"
DEFAULT_RPC_URL = f"{DEFAULT_AUR_URL}/rpc"
DEFAULT_RPC_VERSION = "5"
DEFAULT_TIMEOUT = 30
@staticmethod
def parse_response(response: Dict[str, Any]) -> List[AURPackage]:
@ -113,7 +115,7 @@ class AUR(Remote):
query[key] = value
try:
response = requests.get(self.DEFAULT_RPC_URL, params=query)
response = requests.get(self.DEFAULT_RPC_URL, params=query, timeout=self.DEFAULT_TIMEOUT)
response.raise_for_status()
return self.parse_response(response.json())
except requests.HTTPError as e:

View File

@ -36,11 +36,13 @@ class Official(Remote):
DEFAULT_ARCHLINUX_URL(str): (class attribute) default archlinux url
DEFAULT_SEARCH_REPOSITORIES(List[str]): (class attribute) default list of repositories to search
DEFAULT_RPC_URL(str): (class attribute) default archlinux repositories RPC url
DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds
"""
DEFAULT_ARCHLINUX_URL = "https://archlinux.org"
DEFAULT_SEARCH_REPOSITORIES = ["Core", "Extra", "Multilib", "Community"]
DEFAULT_RPC_URL = "https://archlinux.org/packages/search/json"
DEFAULT_TIMEOUT = 30
@staticmethod
def parse_response(response: Dict[str, Any]) -> List[AURPackage]:
@ -101,7 +103,10 @@ class Official(Remote):
List[AURPackage]: response parsed to package list
"""
try:
response = requests.get(self.DEFAULT_RPC_URL, params={by: args, "repo": self.DEFAULT_SEARCH_REPOSITORIES})
response = requests.get(
self.DEFAULT_RPC_URL,
params={by: args, "repo": self.DEFAULT_SEARCH_REPOSITORIES},
timeout=self.DEFAULT_TIMEOUT)
response.raise_for_status()
return self.parse_response(response.json())
except requests.HTTPError as e:

View File

@ -19,21 +19,17 @@
#
from __future__ import annotations
import logging
from typing import Dict, List, Type
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.lazy_logging import LazyLogging
from ahriman.models.aur_package import AURPackage
class Remote:
class Remote(LazyLogging):
"""
base class for remote package search
Attributes:
logger(logging.Logger): class logger
Examples:
These classes are designed to be used without instancing. In order to achieve it several class methods are
provided: ``info``, ``multisearch`` and ``search``. Thus, the basic flow is the following::
@ -47,12 +43,6 @@ class Remote:
directly, whereas ``multisearch`` splits search one by one and finds intersection between search results.
"""
def __init__(self) -> None:
"""
default constructor
"""
self.logger = logging.getLogger("build_details")
@classmethod
def info(cls: Type[Remote], package_name: str, *, pacman: Pacman) -> AURPackage:
"""

View File

@ -17,22 +17,20 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from pathlib import Path
from typing import List
from ahriman.core.exceptions import BuildFailed
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.util import check_output
from ahriman.models.repository_paths import RepositoryPaths
class Repo:
class Repo(LazyLogging):
"""
repo-add and repo-remove wrapper
Attributes:
logger(logging.Logger): class logger
name(str): repository name
paths(RepositoryPaths): repository paths instance
sign_args(List[str]): additional args which have to be used to sign repository archive
@ -50,7 +48,6 @@ class Repo:
paths(RepositoryPaths): repository paths instance
sign_args(List[str]): additional args which have to be used to sign repository archive
"""
self.logger = logging.getLogger("build_details")
self.name = name
self.paths = paths
self.uid, _ = paths.root_owner

View File

@ -19,23 +19,21 @@
#
from __future__ import annotations
import logging
from typing import Optional, Type
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.lazy_logging import LazyLogging
from ahriman.models.auth_settings import AuthSettings
from ahriman.models.user_access import UserAccess
class Auth:
class Auth(LazyLogging):
"""
helper to deal with user authorization
Attributes:
enabled(bool): indicates if authorization is enabled
logger(logging.Logger): class logger
max_age(int): session age in seconds. It will be used for both client side and server side checks
allow_read_only(bool): allow read only access to APIs
"""
@ -48,8 +46,6 @@ class Auth:
configuration(Configuration): configuration instance
provider(AuthSettings, optional): authorization type definition (Default value = AuthSettings.Disabled)
"""
self.logger = logging.getLogger("http")
self.allow_read_only = configuration.getboolean("auth", "allow_read_only")
self.enabled = provider.is_enabled

View File

@ -17,81 +17,49 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import shutil
from pathlib import Path
from typing import List, Optional
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.util import check_output, walk
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
class Sources:
class Sources(LazyLogging):
"""
helper to download package sources (PKGBUILD etc)
Attributes:
DEFAULT_BRANCH(str): (class attribute) default branch to process git repositories.
Must be used only for local stored repositories, use RemoteSource descriptor instead for real packages
logger(logging.Logger): (class attribute) class logger
"""
DEFAULT_BRANCH = "master" # default fallback branch
logger = logging.getLogger("build_details")
_check_output = check_output
@staticmethod
def _add(sources_dir: Path, *pattern: str) -> None:
def extend_architectures(sources_dir: Path, architecture: str) -> None:
"""
track found files via git
extend existing PKGBUILD with repository architecture
Args:
sources_dir(Path): local path to git repository
*pattern(str): glob patterns
sources_dir(Path): local path to directory with source files
architecture(str): repository architecture
"""
# glob directory to find files which match the specified patterns
found_files: List[Path] = []
for glob in pattern:
found_files.extend(sources_dir.glob(glob))
if not found_files:
return # no additional files found
Sources.logger.info("found matching files %s", found_files)
# add them to index
Sources._check_output("git", "add", "--intent-to-add",
*[str(fn.relative_to(sources_dir)) for fn in found_files],
exception=None, cwd=sources_dir, logger=Sources.logger)
pkgbuild_path = sources_dir / "PKGBUILD"
if not pkgbuild_path.is_file():
return
@staticmethod
def _diff(sources_dir: Path) -> str:
"""
generate diff from the current version and write it to the output file
Args:
sources_dir(Path): local path to git repository
Returns:
str: patch as plain string
"""
return Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def _move(pkgbuild_dir: Path, sources_dir: Path) -> None:
"""
move content from pkgbuild_dir to sources_dir
Args:
pkgbuild_dir(Path): path to directory with pkgbuild from which need to move
sources_dir(Path): path to target directory
"""
if pkgbuild_dir == sources_dir:
return # directories are the same, no need to move
for src in walk(pkgbuild_dir):
dst = sources_dir / src.relative_to(pkgbuild_dir)
shutil.move(src, dst)
architectures = Package.supported_architectures(sources_dir)
architectures.add(architecture)
patch = PkgbuildPatch("arch", list(architectures))
patch.write(pkgbuild_path)
@staticmethod
def fetch(sources_dir: Path, remote: Optional[RemoteSource]) -> None:
@ -102,37 +70,38 @@ class Sources:
sources_dir(Path): local path to fetch
remote(Optional[RemoteSource]): remote target (from where to fetch)
"""
instance = Sources()
# local directory exists and there is .git directory
is_initialized_git = (sources_dir / ".git").is_dir()
if is_initialized_git and not Sources.has_remotes(sources_dir):
if is_initialized_git and not instance.has_remotes(sources_dir):
# there is git repository, but no remote configured so far
Sources.logger.info("skip update at %s because there are no branches configured", sources_dir)
instance.logger.info("skip update at %s because there are no branches configured", sources_dir)
return
branch = remote.branch if remote is not None else Sources.DEFAULT_BRANCH
branch = remote.branch if remote is not None else instance.DEFAULT_BRANCH
if is_initialized_git:
Sources.logger.info("update HEAD to remote at %s using branch %s", sources_dir, branch)
instance.logger.info("update HEAD to remote at %s using branch %s", sources_dir, branch)
Sources._check_output("git", "fetch", "origin", branch,
exception=None, cwd=sources_dir, logger=Sources.logger)
exception=None, cwd=sources_dir, logger=instance.logger)
elif remote is not None:
Sources.logger.info("clone remote %s to %s using branch %s", remote.git_url, sources_dir, branch)
instance.logger.info("clone remote %s to %s using branch %s", remote.git_url, sources_dir, branch)
Sources._check_output("git", "clone", "--branch", branch, "--single-branch",
remote.git_url, str(sources_dir),
exception=None, cwd=sources_dir, logger=Sources.logger)
exception=None, cwd=sources_dir, logger=instance.logger)
else:
# it will cause an exception later
Sources.logger.error("%s is not initialized, but no remote provided", sources_dir)
instance.logger.error("%s is not initialized, but no remote provided", sources_dir)
# and now force reset to our branch
Sources._check_output("git", "checkout", "--force", branch,
exception=None, cwd=sources_dir, logger=Sources.logger)
exception=None, cwd=sources_dir, logger=instance.logger)
Sources._check_output("git", "reset", "--hard", f"origin/{branch}",
exception=None, cwd=sources_dir, logger=Sources.logger)
exception=None, cwd=sources_dir, logger=instance.logger)
# move content if required
# we are using full path to source directory in order to make append possible
pkgbuild_dir = remote.pkgbuild_dir if remote is not None else sources_dir.resolve()
Sources._move((sources_dir / pkgbuild_dir).resolve(), sources_dir)
instance.move((sources_dir / pkgbuild_dir).resolve(), sources_dir)
@staticmethod
def has_remotes(sources_dir: Path) -> bool:
@ -145,7 +114,8 @@ class Sources:
Returns:
bool: True in case if there is any remote and false otherwise
"""
remotes = Sources._check_output("git", "remote", exception=None, cwd=sources_dir, logger=Sources.logger)
instance = Sources()
remotes = Sources._check_output("git", "remote", exception=None, cwd=sources_dir, logger=instance.logger)
return bool(remotes)
@staticmethod
@ -156,8 +126,9 @@ class Sources:
Args:
sources_dir(Path): local path to sources
"""
Sources._check_output("git", "init", "--initial-branch", Sources.DEFAULT_BRANCH,
exception=None, cwd=sources_dir, logger=Sources.logger)
instance = Sources()
Sources._check_output("git", "init", "--initial-branch", instance.DEFAULT_BRANCH,
exception=None, cwd=sources_dir, logger=instance.logger)
@staticmethod
def load(sources_dir: Path, package: Package, patch: Optional[str], paths: RepositoryPaths) -> None:
@ -170,29 +141,15 @@ class Sources:
patch(Optional[str]): optional patch to be applied
paths(RepositoryPaths): repository paths instance
"""
instance = Sources()
if (cache_dir := paths.cache_for(package.base)).is_dir() and cache_dir != sources_dir:
# no need to clone whole repository, just copy from cache first
shutil.copytree(cache_dir, sources_dir, dirs_exist_ok=True)
Sources.fetch(sources_dir, package.remote)
instance.fetch(sources_dir, package.remote)
if patch is None:
Sources.logger.info("no patches found")
return
Sources.patch_apply(sources_dir, patch)
@staticmethod
def patch_apply(sources_dir: Path, patch: str) -> None:
"""
apply patches if any
Args:
sources_dir(Path): local path to directory with git sources
patch(str): patch to be applied
"""
# create patch
Sources.logger.info("apply patch from database")
Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace",
exception=None, cwd=sources_dir, input_data=patch, logger=Sources.logger)
if patch is not None:
instance.patch_apply(sources_dir, patch)
instance.extend_architectures(sources_dir, paths.architecture)
@staticmethod
def patch_create(sources_dir: Path, *pattern: str) -> str:
@ -206,6 +163,67 @@ class Sources:
Returns:
str: patch as plain text
"""
Sources._add(sources_dir, *pattern)
diff = Sources._diff(sources_dir)
instance = Sources()
instance.add(sources_dir, *pattern)
diff = instance.diff(sources_dir)
return f"{diff}\n" # otherwise, patch will be broken
def add(self, sources_dir: Path, *pattern: str) -> None:
"""
track found files via git
Args:
sources_dir(Path): local path to git repository
*pattern(str): glob patterns
"""
# glob directory to find files which match the specified patterns
found_files: List[Path] = []
for glob in pattern:
found_files.extend(sources_dir.glob(glob))
if not found_files:
return # no additional files found
self.logger.info("found matching files %s", found_files)
# add them to index
Sources._check_output("git", "add", "--intent-to-add",
*[str(fn.relative_to(sources_dir)) for fn in found_files],
exception=None, cwd=sources_dir, logger=self.logger)
def diff(self, sources_dir: Path) -> str:
"""
generate diff from the current version and write it to the output file
Args:
sources_dir(Path): local path to git repository
Returns:
str: patch as plain string
"""
return Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=self.logger)
def move(self, pkgbuild_dir: Path, sources_dir: Path) -> None:
"""
move content from pkgbuild_dir to sources_dir
Args:
pkgbuild_dir(Path): path to directory with pkgbuild from which need to move
sources_dir(Path): path to target directory
"""
del self
if pkgbuild_dir == sources_dir:
return # directories are the same, no need to move
for src in walk(pkgbuild_dir):
dst = sources_dir / src.relative_to(pkgbuild_dir)
shutil.move(src, dst)
def patch_apply(self, sources_dir: Path, patch: str) -> None:
"""
apply patches if any
Args:
sources_dir(Path): local path to directory with git sources
patch(str): patch to be applied
"""
# create patch
self.logger.info("apply patch from database")
Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace",
exception=None, cwd=sources_dir, input_data=patch, logger=self.logger)

View File

@ -17,8 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from pathlib import Path
from typing import List
@ -26,18 +24,17 @@ from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import BuildFailed
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.util import check_output
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
class Task:
class Task(LazyLogging):
"""
base package build task
Attributes:
build_logger(logging.Logger): logger for build process
logger(logging.Logger): class logger
package(Package): package definitions
paths(RepositoryPaths): repository paths instance
uid(int): uid of the repository owner user
@ -54,8 +51,6 @@ class Task:
configuration(Configuration): configuration instance
paths(RepositoryPaths): repository paths instance
"""
self.logger = logging.getLogger("root")
self.build_logger = logging.getLogger("build_details")
self.package = package
self.paths = paths
self.uid, _ = paths.root_owner
@ -85,14 +80,14 @@ class Task:
*command,
exception=BuildFailed(self.package.base),
cwd=sources_dir,
logger=self.build_logger,
logger=self.logger,
user=self.uid)
# well it is not actually correct, but we can deal with it
packages = Task._check_output("makepkg", "--packagelist",
exception=BuildFailed(self.package.base),
cwd=sources_dir,
logger=self.build_logger).splitlines()
logger=self.logger).splitlines()
return [Path(package) for package in packages]
def init(self, sources_dir: Path, database: SQLite) -> None:

View File

@ -19,8 +19,6 @@
#
from __future__ import annotations
import logging
from importlib import import_module
from pathlib import Path
from pkgutil import iter_modules
@ -29,11 +27,12 @@ from typing import List, Type
from ahriman.core.configuration import Configuration
from ahriman.core.database.data import migrate_data
from ahriman.core.lazy_logging import LazyLogging
from ahriman.models.migration import Migration
from ahriman.models.migration_result import MigrationResult
class Migrations:
class Migrations(LazyLogging):
"""
simple migration wrapper for the sqlite
idea comes from https://www.ash.dev/blog/simple-migration-system-in-sqlite/
@ -41,7 +40,6 @@ class Migrations:
Attributes:
configuration(Configuration): configuration instance
connection(Connection): database connection
logger(logging.Logger): class logger
"""
def __init__(self, connection: Connection, configuration: Configuration) -> None:
@ -54,7 +52,6 @@ class Migrations:
"""
self.connection = connection
self.configuration = configuration
self.logger = logging.getLogger("database")
@classmethod
def migrate(cls: Type[Migrations], connection: Connection, configuration: Configuration) -> MigrationResult:
@ -86,7 +83,7 @@ class Migrations:
module = import_module(f"{__name__}.{module_name}")
steps: List[str] = getattr(module, "steps", [])
self.logger.debug("found migration %s at index %s with steps count %s", module_name, index, len(steps))
migrations.append(Migration(index, module_name, steps))
migrations.append(Migration(index=index, name=module_name, steps=steps))
return migrations
@ -100,7 +97,7 @@ class Migrations:
migrations = self.migrations()
current_version = self.user_version()
expected_version = len(migrations)
result = MigrationResult(current_version, expected_version)
result = MigrationResult(old_version=current_version, new_version=expected_version)
if not result.is_outdated:
self.logger.info("no migrations required")

View File

@ -58,7 +58,7 @@ class AuthOperations(Operations):
def run(connection: Connection) -> List[User]:
return [
User(cursor["username"], cursor["password"], UserAccess(cursor["access"]))
User(username=cursor["username"], password=cursor["password"], access=UserAccess(cursor["access"]))
for cursor in connection.execute(
"""
select * from users

View File

@ -17,23 +17,22 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import sqlite3
from pathlib import Path
from sqlite3 import Connection, Cursor
from typing import Any, Dict, Tuple, TypeVar, Callable
from ahriman.core.lazy_logging import LazyLogging
T = TypeVar("T")
class Operations:
class Operations(LazyLogging):
"""
base operation class
Attributes:
logger(logging.Logger): class logger
path(Path): path to the database file
"""
@ -45,7 +44,6 @@ class Operations:
path(Path): path to the database file
"""
self.path = path
self.logger = logging.getLogger("database")
@staticmethod
def factory(cursor: Cursor, row: Tuple[Any, ...]) -> Dict[str, Any]:

View File

@ -154,7 +154,11 @@ class PackageOperations(Operations):
Dict[str, Package]: map of the package base to its descriptor (without packages themselves)
"""
return {
row["package_base"]: Package(row["package_base"], row["version"], RemoteSource.from_json(row), {})
row["package_base"]: Package(
base=row["package_base"],
version=row["version"],
remote=RemoteSource.from_json(row),
packages={})
for row in connection.execute("""select * from package_bases""")
}

View File

@ -0,0 +1,64 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from typing import Any
class LazyLogging:
"""
wrapper for the logger library inspired by scala lazy logging module
Attributes:
logger(logging.Logger): class logger instance
"""
logger: logging.Logger
def __getattr__(self, item: str) -> Any:
"""
logger extractor
Args:
item(str) property name:
Returns:
Any: attribute by its name
Raises:
AttributeError: in case if no such attribute found
"""
if item == "logger":
logger = logging.getLogger(self.logger_name)
setattr(self, item, logger)
return logger
raise AttributeError(f"'{self.__class__.__qualname__}' object has no attribute '{item}'")
@property
def logger_name(self) -> str:
"""
extract logger name for the class
Returns:
str: logger name as combination of module name and class name
"""
clazz = self.__class__
prefix = "" if clazz.__module__ is None else f"{clazz.__module__}."
return f"{prefix}{self.__class__.__qualname__}"

View File

@ -18,11 +18,4 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.report.report import Report
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.console import Console
from ahriman.core.report.email import Email
from ahriman.core.report.html import HTML
from ahriman.core.report.telegram import Telegram
from ahriman.core.report.report_trigger import ReportTrigger

View File

@ -25,7 +25,8 @@ from email.mime.text import MIMEText
from typing import Dict, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.report import JinjaTemplate, Report
from ahriman.core.report import Report
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.util import pretty_datetime
from ahriman.models.package import Package
from ahriman.models.result import Result

View File

@ -20,7 +20,8 @@
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.report import JinjaTemplate, Report
from ahriman.core.report import Report
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.models.package import Package
from ahriman.models.result import Result

View File

@ -19,25 +19,23 @@
#
from __future__ import annotations
import logging
from typing import Iterable, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ReportFailed
from ahriman.core.lazy_logging import LazyLogging
from ahriman.models.package import Package
from ahriman.models.report_settings import ReportSettings
from ahriman.models.result import Result
class Report:
class Report(LazyLogging):
"""
base report generator
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
logger(logging.Logger): class logger
Examples:
``Report`` classes provide several method in order to operate with the report generation and additional class
@ -67,7 +65,6 @@ class Report:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration
@ -87,16 +84,16 @@ class Report:
section, provider_name = configuration.gettype(target, architecture)
provider = ReportSettings.from_option(provider_name)
if provider == ReportSettings.HTML:
from ahriman.core.report import HTML
from ahriman.core.report.html import HTML
return HTML(architecture, configuration, section)
if provider == ReportSettings.Email:
from ahriman.core.report import Email
from ahriman.core.report.email import Email
return Email(architecture, configuration, section)
if provider == ReportSettings.Console:
from ahriman.core.report import Console
from ahriman.core.report.console import Console
return Console(architecture, configuration, section)
if provider == ReportSettings.Telegram:
from ahriman.core.report import Telegram
from ahriman.core.report.telegram import Telegram
return Telegram(architecture, configuration, section)
return cls(architecture, configuration) # should never happen

View File

@ -23,7 +23,8 @@ import requests
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.report import JinjaTemplate, Report
from ahriman.core.report import Report
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.util import exception_response_text
from ahriman.models.package import Package
from ahriman.models.result import Result
@ -40,6 +41,7 @@ class Telegram(Report, JinjaTemplate):
chat_id(str): chat id to post message, either string with @ or integer
template_path(Path): path to template for built packages
template_type(str): template message type to be used in parse mode, one of MarkdownV2, HTML, Markdown
timeout(int): HTTP request timeout in seconds
"""
TELEGRAM_API_URL = "https://api.telegram.org"
@ -61,6 +63,7 @@ class Telegram(Report, JinjaTemplate):
self.chat_id = configuration.get(section, "chat_id")
self.template_path = configuration.getpath(section, "template_path")
self.template_type = configuration.get(section, "template_type", fallback="HTML")
self.timeout = configuration.getint(section, "timeout", fallback=30)
def _send(self, text: str) -> None:
"""
@ -72,7 +75,8 @@ class Telegram(Report, JinjaTemplate):
try:
response = requests.post(
f"{self.TELEGRAM_API_URL}/bot{self.api_key}/sendMessage",
data={"chat_id": self.chat_id, "text": text, "parse_mode": self.template_type})
data={"chat_id": self.chat_id, "text": text, "parse_mode": self.template_type},
timeout=self.timeout)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not perform request: %s", exception_response_text(e))
@ -94,8 +98,11 @@ class Telegram(Report, JinjaTemplate):
text = self.make_html(result, self.template_path)
# telegram content is limited by 4096 symbols, so we are going to split the message by new lines
# to fit into this restriction
if len(text) > self.TELEGRAM_MAX_CONTENT_LENGTH:
while len(text) > self.TELEGRAM_MAX_CONTENT_LENGTH:
position = text.rfind("\n", 0, self.TELEGRAM_MAX_CONTENT_LENGTH)
if position == -1:
# normally should not happen, but we allow templates editing
raise ValueError("substring not found")
portion, text = text[:position], text[position + 1:] # +1 to exclude newline we split
self._send(portion)
# send remaining (or full in case if size is less than max length) text

View File

@ -20,12 +20,14 @@
import shutil
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Iterable, List, Optional, Set
from ahriman.core.build_tools.task import Task
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.util import tmpdir
from ahriman.core.util import safe_filename
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
from ahriman.models.result import Result
@ -82,7 +84,8 @@ class Executor(Cleaner):
result = Result()
for single in updates:
with tmpdir() as build_dir:
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, \
(build_dir := Path(dir_name)): # pylint: disable=confusing-with-statement
try:
build_single(single, build_dir)
result.add_success(single)
@ -122,16 +125,16 @@ class Executor(Cleaner):
for local in self.packages():
if local.base in packages or all(package in requested for package in local.packages):
to_remove = {
package: Path(properties.filename)
package: properties.filepath
for package, properties in local.packages.items()
if properties.filename is not None
if properties.filepath is not None
}
remove_base(local.base)
elif requested.intersection(local.packages.keys()):
to_remove = {
package: Path(properties.filename)
package: properties.filepath
for package, properties in local.packages.items()
if package in requested and properties.filename is not None
if package in requested and properties.filepath is not None
}
else:
to_remove = {}
@ -160,17 +163,25 @@ class Executor(Cleaner):
Returns:
Result: path to repository database
"""
def rename(archive: PackageDescription, base: str) -> None:
if archive.filename is None:
self.logger.warning("received empty package name for base %s", base)
return # suppress type checking, it never can be none actually
if (safe := safe_filename(archive.filename)) != archive.filename:
shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe)
archive.filename = safe
def update_single(name: Optional[str], base: str) -> None:
if name is None:
self.logger.warning("received empty package name for base %s", base)
return # suppress type checking, it never can be none actually
# in theory it might be NOT packages directory, but we suppose it is
# in theory, it might be NOT packages directory, but we suppose it is
full_path = self.paths.packages / name
files = self.sign.process_sign_package(full_path, base)
for src in files:
dst = self.paths.repository / src.name
dst = self.paths.repository / safe_filename(src.name)
shutil.move(src, dst)
package_path = self.paths.repository / name
package_path = self.paths.repository / safe_filename(name)
self.repo.add(package_path)
current_packages = self.packages()
@ -181,6 +192,7 @@ class Executor(Cleaner):
for local in updates:
try:
for description in local.packages.values():
rename(description, local.base)
update_single(description.filename, local.base)
self.reporter.set_success(local)
result.add_success(local)

View File

@ -99,7 +99,7 @@ class Repository(Executor, UpdateHandler):
"""
return list(filter(package_like, self.paths.packages.iterdir()))
def packages_depends_on(self, depends_on: Optional[Iterable[str]]) -> List[Package]:
def packages_depend_on(self, depends_on: Optional[Iterable[str]]) -> List[Package]:
"""
extract list of packages which depends on specified package

View File

@ -17,20 +17,19 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.repo import Repo
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnsafeRun
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.client import Client
from ahriman.core.triggers import TriggerLoader
from ahriman.core.util import check_user
class RepositoryProperties:
class RepositoryProperties(LazyLogging):
"""
repository internal objects holder
@ -39,7 +38,6 @@ class RepositoryProperties:
configuration(Configuration): configuration instance
database(SQLite): database instance
ignore_list(List[str]): package bases which will be ignored during auto updates
logger(logging.Logger): class logger
name(str): repository name
pacman(Pacman): alpm wrapper instance
paths(RepositoryPaths): repository paths instance
@ -61,7 +59,6 @@ class RepositoryProperties:
no_report(bool): force disable reporting
unsafe(bool): if set no user check will be performed before path creation
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration
self.database = database

View File

@ -17,7 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import requests
from pathlib import Path
@ -25,23 +24,25 @@ from typing import List, Optional, Set, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import BuildFailed
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.util import check_output, exception_response_text
from ahriman.models.sign_settings import SignSettings
class GPG:
class GPG(LazyLogging):
"""
gnupg wrapper
Attributes:
DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds
architecture(str): repository architecture
configuration(Configuration): configuration instance
default_key(Optional[str]): default PGP key ID to use
logger(logging.Logger): class logger
targets(Set[SignSettings]): list of targets to sign (repository, package etc)
"""
_check_output = check_output
DEFAULT_TIMEOUT = 30
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
@ -51,7 +52,6 @@ class GPG:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("build_details")
self.architecture = architecture
self.configuration = configuration
self.targets, self.default_key = self.sign_options(configuration)
@ -122,7 +122,7 @@ class GPG:
"op": "get",
"options": "mr",
"search": key
})
}, timeout=self.DEFAULT_TIMEOUT)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception("could not download key %s from %s: %s", key, server, exception_response_text(e))

View File

@ -20,7 +20,6 @@
from __future__ import annotations
import argparse
import logging
import uuid
from multiprocessing import Process, Queue
@ -28,10 +27,11 @@ from threading import Lock, Thread
from typing import Callable, Dict, Iterable, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging
from ahriman.models.package_source import PackageSource
class Spawn(Thread):
class Spawn(Thread, LazyLogging):
"""
helper to spawn external ahriman process
MUST NOT be used directly, the only one usage allowed is to spawn process from web services
@ -40,7 +40,6 @@ class Spawn(Thread):
active(Dict[str, Process]): map of active child processes required to avoid zombies
architecture(str): repository architecture
configuration(Configuration): configuration instance
logger(logging.Logger): spawner logger
queue(Queue[Tuple[str, bool]]): multiprocessing queue to read updates from processes
"""
@ -57,7 +56,6 @@ class Spawn(Thread):
self.architecture = architecture
self.args_parser = args_parser
self.configuration = configuration
self.logger = logging.getLogger("http")
self.lock = Lock()
self.active: Dict[str, Process] = {}

View File

@ -80,7 +80,7 @@ class Client:
Returns:
InternalStatus: current internal (web) service status
"""
return InternalStatus(BuildStatus())
return InternalStatus(status=BuildStatus())
def remove(self, base: str) -> None:
"""

View File

@ -17,19 +17,18 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from typing import Dict, List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnknownPackage
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.package import Package
class Watcher:
class Watcher(LazyLogging):
"""
package status watcher
@ -38,7 +37,6 @@ class Watcher:
database(SQLite): database instance
known(Dict[str, Tuple[Package, BuildStatus]]): list of known packages. For the most cases ``packages`` should
be used instead
logger(logging.Logger): class logger
repository(Repository): repository object
status(BuildStatus): daemon status
"""
@ -52,8 +50,6 @@ class Watcher:
configuration(Configuration): configuration instance
database(SQLite): database instance
"""
self.logger = logging.getLogger("http")
self.architecture = architecture
self.database = database
self.repository = Repository(architecture, configuration, database, no_report=True, unsafe=False)

View File

@ -17,12 +17,12 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import requests
from typing import List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.util import exception_response_text
from ahriman.models.build_status import BuildStatusEnum, BuildStatus
@ -31,13 +31,12 @@ from ahriman.models.package import Package
from ahriman.models.user import User
class WebClient(Client):
class WebClient(Client, LazyLogging):
"""
build status reporter web client
Attributes:
address(str): address of the web service
logger(logging.Logger): class logger
user(Optional[User]): web service user descriptor
"""
@ -48,7 +47,6 @@ class WebClient(Client):
Args:
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("http")
self.address = self.parse_address(configuration)
self.user = User.from_option(
configuration.get("web", "username", fallback=None),
@ -191,7 +189,7 @@ class WebClient(Client):
self.logger.exception("could not get web service status: %s", exception_response_text(e))
except Exception:
self.logger.exception("could not get web service status")
return InternalStatus(BuildStatus())
return InternalStatus(status=BuildStatus())
def remove(self, base: str) -> None:
"""

View File

@ -19,11 +19,12 @@
#
from __future__ import annotations
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Iterable, List, Set, Type
from ahriman.core.build_tools.sources import Sources
from ahriman.core.database import SQLite
from ahriman.core.util import tmpdir
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
@ -71,7 +72,8 @@ class Leaf:
Returns:
Leaf: loaded class
"""
with tmpdir() as clone_dir:
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, \
(clone_dir := Path(dir_name)): # pylint: disable=confusing-with-statement
Sources.load(clone_dir, package, database.patches_get(package.base), paths)
dependencies = Package.dependencies(clone_dir)
return cls(package, dependencies)

View File

@ -17,23 +17,21 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging
from ahriman.models.package import Package
from ahriman.models.result import Result
class Trigger:
class Trigger(LazyLogging):
"""
trigger base class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
logger(logging.Logger): application logger
Examples:
This class must be used in order to create own extension. Basically idea is the following::
@ -61,7 +59,6 @@ class Trigger:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration

View File

@ -18,7 +18,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import importlib
import logging
import os
from pathlib import Path
@ -27,19 +26,19 @@ from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import InvalidExtension
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.triggers import Trigger
from ahriman.models.package import Package
from ahriman.models.result import Result
class TriggerLoader:
class TriggerLoader(LazyLogging):
"""
trigger loader class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
logger(logging.Logger): application logger
triggers(List[Trigger]): list of loaded triggers according to the configuration
Examples:
@ -66,7 +65,6 @@ class TriggerLoader:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration

View File

@ -18,10 +18,4 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.upload.upload import Upload
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.core.upload.github import Github
from ahriman.core.upload.rsync import Rsync
from ahriman.core.upload.s3 import S3
from ahriman.core.upload.upload_trigger import UploadTrigger

View File

@ -24,7 +24,7 @@ from pathlib import Path
from typing import Any, Dict, Iterable, Optional
from ahriman.core.configuration import Configuration
from ahriman.core.upload import HttpUpload
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.core.util import walk
from ahriman.models.package import Package

View File

@ -34,6 +34,7 @@ class HttpUpload(Upload):
Attributes:
auth(Tuple[str, str]): HTTP auth object
timeout(int): HTTP request timeout in seconds
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
@ -49,6 +50,7 @@ class HttpUpload(Upload):
password = configuration.get(section, "password")
username = configuration.get(section, "username")
self.auth = (password, username)
self.timeout = configuration.getint(section, "timeout", fallback=30)
@staticmethod
def calculate_hash(path: Path) -> str:
@ -108,7 +110,7 @@ class HttpUpload(Upload):
requests.Response: request response object
"""
try:
response = requests.request(method, url, auth=self.auth, **kwargs)
response = requests.request(method, url, auth=self.auth, timeout=self.timeout, **kwargs)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e))

View File

@ -19,25 +19,23 @@
#
from __future__ import annotations
import logging
from pathlib import Path
from typing import Iterable, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SyncFailed
from ahriman.core.lazy_logging import LazyLogging
from ahriman.models.package import Package
from ahriman.models.upload_settings import UploadSettings
class Upload:
class Upload(LazyLogging):
"""
base remote sync class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
logger(logging.Logger): application logger
Examples:
These classes provide the way to upload packages to remote sources as it is described in their implementations.
@ -66,7 +64,6 @@ class Upload:
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.logger = logging.getLogger("root")
self.architecture = architecture
self.configuration = configuration
@ -86,13 +83,13 @@ class Upload:
section, provider_name = configuration.gettype(target, architecture)
provider = UploadSettings.from_option(provider_name)
if provider == UploadSettings.Rsync:
from ahriman.core.upload import Rsync
from ahriman.core.upload.rsync import Rsync
return Rsync(architecture, configuration, section)
if provider == UploadSettings.S3:
from ahriman.core.upload import S3
from ahriman.core.upload.s3 import S3
return S3(architecture, configuration, section)
if provider == UploadSettings.Github:
from ahriman.core.upload import Github
from ahriman.core.upload.github import Github
return Github(architecture, configuration, section)
return cls(architecture, configuration) # should never happen

View File

@ -20,14 +20,11 @@
import datetime
import io
import os
from enum import Enum
import re
import requests
import shutil
import subprocess
import tempfile
from contextlib import contextmanager
from enum import Enum
from logging import Logger
from pathlib import Path
from typing import Any, Dict, Generator, IO, Iterable, List, Optional, Type, Union
@ -37,7 +34,7 @@ from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["check_output", "check_user", "exception_response_text", "filter_json", "full_version", "enum_values",
"package_like", "pretty_datetime", "pretty_size", "tmpdir", "walk"]
"package_like", "pretty_datetime", "pretty_size", "safe_filename", "walk"]
def check_output(*args: str, exception: Optional[Exception], cwd: Optional[Path] = None,
@ -273,26 +270,25 @@ def pretty_size(size: Optional[float], level: int = 0) -> str:
return pretty_size(size / 1024, level + 1)
@contextmanager
def tmpdir() -> Generator[Path, None, None]:
def safe_filename(source: str) -> str:
"""
wrapper for tempfile to remove directory after all
convert source string to its safe representation
Yields:
Path: path to the created directory
Args:
source(str): string to convert
Examples:
This function must be used only inside context manager as decorator states::
>>> with tmpdir() as path:
>>> do_something(path)
>>> raise Exception("Clear me after exception please")
Returns:
str: result string in which all unsafe characters are replaced by dash
"""
path = Path(tempfile.mkdtemp())
try:
yield path
finally:
shutil.rmtree(path, ignore_errors=True)
# RFC-3986 https://datatracker.ietf.org/doc/html/rfc3986 states that unreserved characters are
# https://datatracker.ietf.org/doc/html/rfc3986#section-2.3
# unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
# however we would like to allow some gen-delims characters in filename, because those characters are used
# as delimiter in other URI parts. The ones we allow are
# ":" - used as separator in schema and userinfo
# "[" and "]" - used for host part
# "@" - used as separator between host and userinfo
return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source)
def walk(directory_path: Path) -> Generator[Path, None, None]:

View File

@ -29,7 +29,7 @@ from typing import Any, Callable, Dict, List, Optional, Type
from ahriman.core.util import filter_json, full_version
@dataclass
@dataclass(frozen=True, kw_only=True)
class AURPackage:
"""
AUR package descriptor

View File

@ -47,7 +47,7 @@ class BuildStatusEnum(str, Enum):
Success = "success"
@dataclass
@dataclass(frozen=True)
class BuildStatus:
"""
build status holder
@ -64,7 +64,7 @@ class BuildStatus:
"""
convert status to enum type
"""
self.status = BuildStatusEnum(self.status)
object.__setattr__(self, "status", BuildStatusEnum(self.status))
@classmethod
def from_json(cls: Type[BuildStatus], dump: Dict[str, Any]) -> BuildStatus:

View File

@ -27,7 +27,7 @@ from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
@dataclass
@dataclass(frozen=True, kw_only=True)
class Counters:
"""
package counters

View File

@ -26,7 +26,7 @@ from ahriman.models.build_status import BuildStatus
from ahriman.models.counters import Counters
@dataclass
@dataclass(frozen=True, kw_only=True)
class InternalStatus:
"""
internal server status

View File

@ -21,7 +21,7 @@ from dataclasses import dataclass
from typing import List
@dataclass
@dataclass(frozen=True, kw_only=True)
class Migration:
"""
migration implementation

View File

@ -22,7 +22,7 @@ from dataclasses import dataclass
from ahriman.core.exceptions import MigrationError
@dataclass
@dataclass(frozen=True, kw_only=True)
class MigrationResult:
"""
migration result implementation model

View File

@ -20,7 +20,6 @@
from __future__ import annotations
import copy
import logging
from dataclasses import asdict, dataclass
from pathlib import Path
@ -31,6 +30,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Type
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.lazy_logging import LazyLogging
from ahriman.core.util import check_output, full_version
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
@ -38,8 +38,8 @@ from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
@dataclass
class Package:
@dataclass(kw_only=True)
class Package(LazyLogging):
"""
package properties representation
@ -147,7 +147,7 @@ class Package:
"""
package = pacman.handle.load_pkg(str(path))
description = PackageDescription.from_package(package, path)
return cls(package.base, package.version, remote, {package.name: description})
return cls(base=package.base, version=package.version, remote=remote, packages={package.name: description})
@classmethod
def from_aur(cls: Type[Package], name: str, pacman: Pacman) -> Package:
@ -163,7 +163,11 @@ class Package:
"""
package = AUR.info(name, pacman=pacman)
remote = RemoteSource.from_source(PackageSource.AUR, package.package_base, package.repository)
return cls(package.package_base, package.version, remote, {package.name: PackageDescription()})
return cls(
base=package.package_base,
version=package.version,
remote=remote,
packages={package.name: PackageDescription()})
@classmethod
def from_build(cls: Type[Package], path: Path) -> Package:
@ -186,7 +190,7 @@ class Package:
packages = {key: PackageDescription() for key in srcinfo["packages"]}
version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
return cls(srcinfo["pkgbase"], version, None, packages)
return cls(base=srcinfo["pkgbase"], version=version, remote=None, packages=packages)
@classmethod
def from_json(cls: Type[Package], dump: Dict[str, Any]) -> Package:
@ -204,11 +208,7 @@ class Package:
for key, value in dump.get("packages", {}).items()
}
remote = dump.get("remote", {})
return cls(
base=dump["base"],
version=dump["version"],
remote=RemoteSource.from_json(remote),
packages=packages)
return cls(base=dump["base"], version=dump["version"], remote=RemoteSource.from_json(remote), packages=packages)
@classmethod
def from_official(cls: Type[Package], name: str, pacman: Pacman, use_syncdb: bool = True) -> Package:
@ -225,7 +225,11 @@ class Package:
"""
package = OfficialSyncdb.info(name, pacman=pacman) if use_syncdb else Official.info(name, pacman=pacman)
remote = RemoteSource.from_source(PackageSource.Repository, package.package_base, package.repository)
return cls(package.package_base, package.version, remote, {package.name: PackageDescription()})
return cls(
base=package.package_base,
version=package.version,
remote=remote,
packages={package.name: PackageDescription()})
@staticmethod
def dependencies(path: Path) -> Set[str]:
@ -263,6 +267,26 @@ class Package:
packages = set(srcinfo["packages"].keys())
return (depends | makedepends) - packages
@staticmethod
def supported_architectures(path: Path) -> Set[str]:
"""
load supported architectures from package sources
Args:
path(Path): path to package sources directory
Returns:
Set[str]: list of package supported architectures
Raises:
InvalidPackageInfo: if there are parsing errors
"""
srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", exception=None, cwd=path)
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise InvalidPackageInfo(errors)
return set(srcinfo.get("arch", []))
def actual_version(self, paths: RepositoryPaths) -> str:
"""
additional method to handle VCS package versions
@ -281,23 +305,22 @@ class Package:
from ahriman.core.build_tools.sources import Sources
logger = logging.getLogger("build_details")
Sources.load(paths.cache_for(self.base), self, None, paths)
try:
# update pkgver first
Package._check_output("makepkg", "--nodeps", "--nobuild",
exception=None, cwd=paths.cache_for(self.base), logger=logger)
exception=None, cwd=paths.cache_for(self.base), logger=self.logger)
# generate new .SRCINFO and put it to parser
srcinfo_source = Package._check_output("makepkg", "--printsrcinfo",
exception=None, cwd=paths.cache_for(self.base), logger=logger)
exception=None, cwd=paths.cache_for(self.base), logger=self.logger)
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise InvalidPackageInfo(errors)
return full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
except Exception:
logger.exception("cannot determine version of VCS package, make sure that you have VCS tools installed")
self.logger.exception("cannot determine version of VCS package, make sure that VCS tools are installed")
return self.version

View File

@ -27,7 +27,7 @@ from typing import Any, Dict, List, Optional, Type
from ahriman.core.util import filter_json
@dataclass
@dataclass(kw_only=True)
class PackageDescription:
"""
package specific properties

View File

@ -0,0 +1,92 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import shlex
from dataclasses import dataclass, field
from pathlib import Path
from typing import List, Union
@dataclass(frozen=True)
class PkgbuildPatch:
"""
wrapper for patching PKBGUILDs
Attributes:
key(str): name of the property in PKGBUILD, e.g. version, url etc
value(Union[str, List[str]]): value of the stored PKGBUILD property. It must be either string or list of string
values
unsafe(bool): if set, value will be not quoted, might break PKGBUILD
"""
key: str
value: Union[str, List[str]]
unsafe: bool = field(default=False, kw_only=True)
@property
def is_function(self) -> bool:
"""
parse key and define whether it function or not
Returns:
bool: True in case if key ends with parentheses and False otherwise
"""
return self.key.endswith("()")
def quote(self, value: str) -> str:
"""
quote value according to the unsafe flag
Args:
value(str): value to be quoted
Returns:
str: quoted string in case if unsafe is False and as is otherwise
"""
return value if self.unsafe else shlex.quote(value)
def serialize(self) -> str:
"""
serialize key-value pair into PKBGBUILD string. List values will be put inside parentheses. All string
values (including the ones inside list values) will be put inside quotes, no shell variables expanding supported
at the moment
Returns:
str: serialized key-value pair, print-friendly
"""
if isinstance(self.value, list): # list like
value = " ".join(map(self.quote, self.value))
return f"""{self.key}=({value})"""
# we suppose that function values are only supported in string-like values
if self.is_function:
return f"{self.key} {self.value}" # no quoting enabled here
return f"""{self.key}={self.quote(self.value)}"""
def write(self, pkgbuild_path: Path) -> None:
"""
write serialized value into PKGBUILD by specified path
Args:
pkgbuild_path(Path): path to PKGBUILD file
"""
with pkgbuild_path.open("a") as pkgbuild:
pkgbuild.write("\n") # in case if file ends without new line we are appending it at the end
pkgbuild.write(self.serialize())
pkgbuild.write("\n") # append new line after the values

View File

@ -17,11 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Any
@dataclass
@dataclass(frozen=True)
class Property:
"""
holder of object properties descriptor
@ -34,4 +34,4 @@ class Property:
name: str
value: Any
is_required: bool = False
is_required: bool = field(default=False, kw_only=True)

View File

@ -27,7 +27,7 @@ from ahriman.core.util import filter_json
from ahriman.models.package_source import PackageSource
@dataclass
@dataclass(frozen=True, kw_only=True)
class RemoteSource:
"""
remote package source properties
@ -50,7 +50,7 @@ class RemoteSource:
"""
convert source to enum type
"""
self.source = PackageSource(self.source)
object.__setattr__(self, "source", PackageSource(self.source))
@property
def pkgbuild_dir(self) -> Path:

View File

@ -29,7 +29,7 @@ from typing import Set, Tuple, Type
from ahriman.core.exceptions import InvalidPath
@dataclass
@dataclass(frozen=True)
class RepositoryPaths:
"""
repository paths holder. For the most operations with paths you want to use this object

View File

@ -19,7 +19,7 @@
#
from __future__ import annotations
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import Optional, Type
from passlib.pwd import genword as generate_password # type: ignore
from passlib.handlers.sha2_crypt import sha512_crypt # type: ignore
@ -27,7 +27,7 @@ from passlib.handlers.sha2_crypt import sha512_crypt # type: ignore
from ahriman.models.user_access import UserAccess
@dataclass
@dataclass(frozen=True, kw_only=True)
class User:
"""
authorized web user model
@ -82,7 +82,7 @@ class User:
"""
if username is None or password is None:
return None
return cls(username, password, access)
return cls(username=username, password=password, access=access)
@staticmethod
def generate_password(length: int) -> str:
@ -130,7 +130,7 @@ class User:
# when we do not store any password here
return self
password_hash: str = self._HASHER.hash(self.password + salt)
return User(self.username, password_hash, self.access)
return replace(self, password=password_hash)
def verify_access(self, required: UserAccess) -> bool:
"""

View File

@ -25,7 +25,7 @@ from dataclasses import dataclass
from typing import Optional, Type
@dataclass
@dataclass(frozen=True)
class UserIdentity:
"""
user identity used inside web service

View File

@ -17,4 +17,4 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = "2.0.0"
__version__ = "2.2.1"

View File

@ -94,7 +94,7 @@ def setup_service(architecture: str, configuration: Configuration, spawner: Spaw
Returns:
web.Application: web application instance
"""
application = web.Application(logger=logging.getLogger("http"))
application = web.Application(logger=logging.getLogger(__name__))
application.on_shutdown.append(on_shutdown)
application.on_startup.append(on_startup)

View File

@ -111,7 +111,7 @@ def test_add_remote(application_packages: ApplicationPackages, package_descripti
application_packages._add_remote(url)
open_mock.assert_called_once_with("wb")
request_mock.assert_called_once_with(url, stream=True)
request_mock.assert_called_once_with(url, stream=True, timeout=None)
response_mock.raise_for_status.assert_called_once_with()

View File

@ -37,7 +37,7 @@ def test_run(args: argparse.Namespace, package_ahriman: Package,
result = Result()
result.add_success(package_ahriman)
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
application_packages_mock = mocker.patch("ahriman.core.repository.repository.Repository.packages_depends_on",
application_packages_mock = mocker.patch("ahriman.core.repository.repository.Repository.packages_depend_on",
return_value=[package_ahriman])
application_mock = mocker.patch("ahriman.application.application.Application.update", return_value=result)
check_mock = mocker.patch("ahriman.application.handlers.Handler.check_if_empty")
@ -71,7 +71,7 @@ def test_run_dry_run(args: argparse.Namespace, configuration: Configuration,
args = _default_args(args)
args.dry_run = True
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
mocker.patch("ahriman.core.repository.repository.Repository.packages_depends_on", return_value=[package_ahriman])
mocker.patch("ahriman.core.repository.repository.Repository.packages_depend_on", return_value=[package_ahriman])
application_mock = mocker.patch("ahriman.application.application.Application.update")
check_mock = mocker.patch("ahriman.application.handlers.Handler.check_if_empty")
@ -88,7 +88,7 @@ def test_run_filter(args: argparse.Namespace, configuration: Configuration, mock
args.depends_on = ["python-aur"]
mocker.patch("ahriman.application.application.Application.update")
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
application_packages_mock = mocker.patch("ahriman.core.repository.repository.Repository.packages_depends_on")
application_packages_mock = mocker.patch("ahriman.core.repository.repository.Repository.packages_depend_on")
Rebuild.run(args, "x86_64", configuration, True, False)
application_packages_mock.assert_called_once_with({"python-aur"})
@ -101,7 +101,7 @@ def test_run_without_filter(args: argparse.Namespace, configuration: Configurati
args = _default_args(args)
mocker.patch("ahriman.application.application.Application.update")
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
application_packages_mock = mocker.patch("ahriman.core.repository.repository.Repository.packages_depends_on")
application_packages_mock = mocker.patch("ahriman.core.repository.repository.Repository.packages_depend_on")
Rebuild.run(args, "x86_64", configuration, True, False)
application_packages_mock.assert_called_once_with(None)
@ -116,7 +116,7 @@ def test_run_update_empty_exception(args: argparse.Namespace, configuration: Con
args.exit_code = True
args.dry_run = True
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
mocker.patch("ahriman.core.repository.repository.Repository.packages_depends_on", return_value=[])
mocker.patch("ahriman.core.repository.repository.Repository.packages_depend_on", return_value=[])
check_mock = mocker.patch("ahriman.application.handlers.Handler.check_if_empty")
Rebuild.run(args, "x86_64", configuration, True, False)
@ -131,7 +131,7 @@ def test_run_build_empty_exception(args: argparse.Namespace, configuration: Conf
args = _default_args(args)
args.exit_code = True
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
mocker.patch("ahriman.core.repository.repository.Repository.packages_depends_on", return_value=[package_ahriman])
mocker.patch("ahriman.core.repository.repository.Repository.packages_depend_on", return_value=[package_ahriman])
mocker.patch("ahriman.application.application.Application.update", return_value=Result())
check_mock = mocker.patch("ahriman.application.handlers.Handler.check_if_empty")

View File

@ -38,7 +38,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, database: S
must run command
"""
args = _default_args(args)
user = User(args.username, args.password, args.role)
user = User(username=args.username, password=args.password, access=args.role)
mocker.patch("ahriman.core.database.SQLite.load", return_value=database)
mocker.patch("ahriman.models.user.User.hash_password", return_value=user)
get_auth_configuration_mock = mocker.patch("ahriman.application.handlers.Users.configuration_get")

View File

@ -426,7 +426,7 @@ def user() -> User:
Returns:
User: user descriptor instance
"""
return User("user", "pa55w0rd", UserAccess.Reporter)
return User(username="user", password="pa55w0rd", access=UserAccess.Reporter)
@pytest.fixture

View File

@ -78,7 +78,9 @@ def test_make_request(aur: AUR, aur_package_ahriman: AURPackage,
assert aur.make_request("info", "ahriman") == [aur_package_ahriman]
request_mock.assert_called_once_with(
"https://aur.archlinux.org/rpc", params={"v": "5", "type": "info", "arg": ["ahriman"]})
"https://aur.archlinux.org/rpc",
params={"v": "5", "type": "info", "arg": ["ahriman"]},
timeout=aur.DEFAULT_TIMEOUT)
def test_make_request_multi_arg(aur: AUR, aur_package_ahriman: AURPackage,
@ -92,7 +94,9 @@ def test_make_request_multi_arg(aur: AUR, aur_package_ahriman: AURPackage,
assert aur.make_request("search", "ahriman", "is", "cool") == [aur_package_ahriman]
request_mock.assert_called_once_with(
"https://aur.archlinux.org/rpc", params={"v": "5", "type": "search", "arg[]": ["ahriman", "is", "cool"]})
"https://aur.archlinux.org/rpc",
params={"v": "5", "type": "search", "arg[]": ["ahriman", "is", "cool"]},
timeout=aur.DEFAULT_TIMEOUT)
def test_make_request_with_kwargs(aur: AUR, aur_package_ahriman: AURPackage,
@ -106,7 +110,9 @@ def test_make_request_with_kwargs(aur: AUR, aur_package_ahriman: AURPackage,
assert aur.make_request("search", "ahriman", by="name") == [aur_package_ahriman]
request_mock.assert_called_once_with(
"https://aur.archlinux.org/rpc", params={"v": "5", "type": "search", "arg": ["ahriman"], "by": "name"})
"https://aur.archlinux.org/rpc",
params={"v": "5", "type": "search", "arg": ["ahriman"], "by": "name"},
timeout=aur.DEFAULT_TIMEOUT)
def test_make_request_failed(aur: AUR, mocker: MockerFixture) -> None:

View File

@ -84,8 +84,10 @@ def test_make_request(official: Official, aur_package_akonadi: AURPackage,
request_mock = mocker.patch("requests.get", return_value=response_mock)
assert official.make_request("akonadi", by="q") == [aur_package_akonadi]
request_mock.assert_called_once_with("https://archlinux.org/packages/search/json",
params={"q": ("akonadi",), "repo": Official.DEFAULT_SEARCH_REPOSITORIES})
request_mock.assert_called_once_with(
"https://archlinux.org/packages/search/json",
params={"q": ("akonadi",), "repo": Official.DEFAULT_SEARCH_REPOSITORIES},
timeout=official.DEFAULT_TIMEOUT)
def test_make_request_failed(official: Official, mocker: MockerFixture) -> None:

View File

@ -0,0 +1,14 @@
import pytest
from ahriman.core.build_tools.sources import Sources
@pytest.fixture
def sources() -> Sources:
"""
sources fixture
Returns:
Sources: sources instance
"""
return Sources()

View File

@ -10,62 +10,28 @@ from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
def test_add(mocker: MockerFixture) -> None:
def test_extend_architectures(mocker: MockerFixture) -> None:
"""
must add files to git
must update available architecture list
"""
glob_mock = mocker.patch("pathlib.Path.glob", return_value=[Path("local/1"), Path("local/2")])
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
mocker.patch("pathlib.Path.is_file", return_value=True)
archs_mock = mocker.patch("ahriman.models.package.Package.supported_architectures", return_value={"x86_64"})
write_mock = mocker.patch("ahriman.models.pkgbuild_patch.PkgbuildPatch.write")
local = Path("local")
Sources._add(local, "pattern1", "pattern2")
glob_mock.assert_has_calls([mock.call("pattern1"), mock.call("pattern2")])
check_output_mock.assert_called_once_with(
"git", "add", "--intent-to-add", "1", "2", "1", "2",
exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
Sources.extend_architectures(Path("local"), "i686")
archs_mock.assert_called_once_with(Path("local"))
write_mock.assert_called_once_with(Path("local") / "PKGBUILD")
def test_add_skip(mocker: MockerFixture) -> None:
def test_extend_architectures_skip(mocker: MockerFixture) -> None:
"""
must skip addition of files to index if no fiels found
must skip extending list of the architectures in case if no PKGBUILD file found
"""
mocker.patch("pathlib.Path.glob", return_value=[])
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
mocker.patch("pathlib.Path.is_file", return_value=False)
write_mock = mocker.patch("ahriman.models.pkgbuild_patch.PkgbuildPatch.write")
Sources._add(Path("local"), "pattern1")
check_output_mock.assert_not_called()
def test_diff(mocker: MockerFixture) -> None:
"""
must calculate diff
"""
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
local = Path("local")
assert Sources._diff(local)
check_output_mock.assert_called_once_with(
"git", "diff", exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
def test_move(mocker: MockerFixture) -> None:
"""
must move content between directories
"""
mocker.patch("ahriman.core.build_tools.sources.walk", return_value=[Path("/source/path")])
move_mock = mocker.patch("shutil.move")
Sources._move(Path("/source"), Path("/destination"))
move_mock.assert_called_once_with(Path("/source/path"), Path("/destination/path"))
def test_move_same(mocker: MockerFixture) -> None:
"""
must not do anything in case if directories are the same
"""
walk_mock = mocker.patch("ahriman.core.build_tools.sources.walk")
Sources._move(Path("/same"), Path("/same"))
walk_mock.assert_not_called()
Sources.extend_architectures(Path("local"), "i686")
write_mock.assert_not_called()
def test_fetch_empty(remote_source: RemoteSource, mocker: MockerFixture) -> None:
@ -87,7 +53,7 @@ def test_fetch_existing(remote_source: RemoteSource, mocker: MockerFixture) -> N
mocker.patch("pathlib.Path.is_dir", return_value=True)
mocker.patch("ahriman.core.build_tools.sources.Sources.has_remotes", return_value=True)
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
local = Path("local")
Sources.fetch(local, remote_source)
@ -108,7 +74,7 @@ def test_fetch_new(remote_source: RemoteSource, mocker: MockerFixture) -> None:
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
local = Path("local")
Sources.fetch(local, remote_source)
@ -129,7 +95,7 @@ def test_fetch_new_without_remote(mocker: MockerFixture) -> None:
"""
mocker.patch("pathlib.Path.is_dir", return_value=False)
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
local = Path("local")
Sources.fetch(local, None)
@ -147,7 +113,7 @@ def test_fetch_relative(remote_source: RemoteSource, mocker: MockerFixture) -> N
must process move correctly on relative directory
"""
mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._move")
move_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.move")
Sources.fetch(Path("path"), remote_source)
move_mock.assert_called_once_with(Path("path").resolve(), Path("path"))
@ -192,10 +158,12 @@ def test_load(package_ahriman: Package, repository_paths: RepositoryPaths, mocke
mocker.patch("pathlib.Path.is_dir", return_value=False)
fetch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.fetch")
patch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.patch_apply")
architectures_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.extend_architectures")
Sources.load(Path("local"), package_ahriman, "patch", repository_paths)
fetch_mock.assert_called_once_with(Path("local"), package_ahriman.remote)
patch_mock.assert_called_once_with(Path("local"), "patch")
architectures_mock.assert_called_once_with(Path("local"), repository_paths.architecture)
def test_load_no_patch(package_ahriman: Package, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
@ -222,26 +190,12 @@ def test_load_with_cache(package_ahriman: Package, repository_paths: RepositoryP
copytree_mock.assert_called_once() # we do not check full command here, sorry
def test_patch_apply(mocker: MockerFixture) -> None:
"""
must apply patches if any
"""
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
local = Path("local")
Sources.patch_apply(local, "patches")
check_output_mock.assert_called_once_with(
"git", "apply", "--ignore-space-change", "--ignore-whitespace",
exception=None, cwd=local, input_data="patches", logger=pytest.helpers.anyvar(int)
)
def test_patch_create(mocker: MockerFixture) -> None:
"""
must create patch set for the package
"""
add_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._add")
diff_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._diff")
add_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.add")
diff_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.diff")
Sources.patch_create(Path("local"), "glob")
add_mock.assert_called_once_with(Path("local"), "glob")
@ -252,6 +206,78 @@ def test_patch_create_with_newline(mocker: MockerFixture) -> None:
"""
created patch must have new line at the end
"""
mocker.patch("ahriman.core.build_tools.sources.Sources._add")
mocker.patch("ahriman.core.build_tools.sources.Sources._diff", return_value="diff")
mocker.patch("ahriman.core.build_tools.sources.Sources.add")
mocker.patch("ahriman.core.build_tools.sources.Sources.diff", return_value="diff")
assert Sources.patch_create(Path("local"), "glob").endswith("\n")
def test_add(sources: Sources, mocker: MockerFixture) -> None:
"""
must add files to git
"""
glob_mock = mocker.patch("pathlib.Path.glob", return_value=[Path("local/1"), Path("local/2")])
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
local = Path("local")
sources.add(local, "pattern1", "pattern2")
glob_mock.assert_has_calls([mock.call("pattern1"), mock.call("pattern2")])
check_output_mock.assert_called_once_with(
"git", "add", "--intent-to-add", "1", "2", "1", "2",
exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
def test_add_skip(sources: Sources, mocker: MockerFixture) -> None:
"""
must skip addition of files to index if no fiels found
"""
mocker.patch("pathlib.Path.glob", return_value=[])
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
sources.add(Path("local"), "pattern1")
check_output_mock.assert_not_called()
def test_diff(sources: Sources, mocker: MockerFixture) -> None:
"""
must calculate diff
"""
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
local = Path("local")
assert sources.diff(local)
check_output_mock.assert_called_once_with(
"git", "diff", exception=None, cwd=local, logger=pytest.helpers.anyvar(int))
def test_move(sources: Sources, mocker: MockerFixture) -> None:
"""
must move content between directories
"""
mocker.patch("ahriman.core.build_tools.sources.walk", return_value=[Path("/source/path")])
move_mock = mocker.patch("shutil.move")
sources.move(Path("/source"), Path("/destination"))
move_mock.assert_called_once_with(Path("/source/path"), Path("/destination/path"))
def test_move_same(sources: Sources, mocker: MockerFixture) -> None:
"""
must not do anything in case if directories are the same
"""
walk_mock = mocker.patch("ahriman.core.build_tools.sources.walk")
sources.move(Path("/same"), Path("/same"))
walk_mock.assert_not_called()
def test_patch_apply(sources: Sources, mocker: MockerFixture) -> None:
"""
must apply patches if any
"""
check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output")
local = Path("local")
sources.patch_apply(local, "patches")
check_output_mock.assert_called_once_with(
"git", "apply", "--ignore-space-change", "--ignore-whitespace",
exception=None, cwd=local, input_data="patches", logger=pytest.helpers.anyvar(int)
)

View File

@ -44,7 +44,7 @@ def test_run(migrations: Migrations, mocker: MockerFixture) -> None:
cursor = MagicMock()
mocker.patch("ahriman.core.database.migrations.Migrations.user_version", return_value=0)
mocker.patch("ahriman.core.database.migrations.Migrations.migrations",
return_value=[Migration(0, "test", ["select 1"])])
return_value=[Migration(index=0, name="test", steps=["select 1"])])
migrations.connection.cursor.return_value = cursor
validate_mock = mocker.patch("ahriman.models.migration_result.MigrationResult.validate")
migrate_data_mock = mocker.patch("ahriman.core.database.migrations.migrate_data")
@ -58,7 +58,8 @@ def test_run(migrations: Migrations, mocker: MockerFixture) -> None:
mock.call("commit"),
])
cursor.close.assert_called_once_with()
migrate_data_mock.assert_called_once_with(MigrationResult(0, 1), migrations.connection, migrations.configuration)
migrate_data_mock.assert_called_once_with(
MigrationResult(old_version=0, new_version=1), migrations.connection, migrations.configuration)
def test_run_migration_exception(migrations: Migrations, mocker: MockerFixture) -> None:
@ -69,7 +70,7 @@ def test_run_migration_exception(migrations: Migrations, mocker: MockerFixture)
mocker.patch("logging.Logger.info", side_effect=Exception())
mocker.patch("ahriman.core.database.migrations.Migrations.user_version", return_value=0)
mocker.patch("ahriman.core.database.migrations.Migrations.migrations",
return_value=[Migration(0, "test", ["select 1"])])
return_value=[Migration(index=0, name="test", steps=["select 1"])])
mocker.patch("ahriman.models.migration_result.MigrationResult.validate")
migrations.connection.cursor.return_value = cursor
@ -90,7 +91,7 @@ def test_run_sql_exception(migrations: Migrations, mocker: MockerFixture) -> Non
cursor.execute.side_effect = Exception()
mocker.patch("ahriman.core.database.migrations.Migrations.user_version", return_value=0)
mocker.patch("ahriman.core.database.migrations.Migrations.migrations",
return_value=[Migration(0, "test", ["select 1"])])
return_value=[Migration(index=0, name="test", steps=["select 1"])])
mocker.patch("ahriman.models.migration_result.MigrationResult.validate")
migrations.connection.cursor.return_value = cursor

View File

@ -16,21 +16,21 @@ def test_user_list(database: SQLite, user: User) -> None:
must return all users
"""
database.user_update(user)
database.user_update(User(user.password, user.username, user.access))
database.user_update(User(username=user.password, password=user.username, access=user.access))
users = database.user_list(None, None)
assert len(users) == 2
assert user in users
assert User(user.password, user.username, user.access) in users
assert User(username=user.password, password=user.username, access=user.access) in users
def test_user_list_filter_by_username(database: SQLite) -> None:
"""
must return users filtered by its id
"""
first = User("1", "", UserAccess.Read)
second = User("2", "", UserAccess.Full)
third = User("3", "", UserAccess.Read)
first = User(username="1", password="", access=UserAccess.Read)
second = User(username="2", password="", access=UserAccess.Full)
third = User(username="3", password="", access=UserAccess.Read)
database.user_update(first)
database.user_update(second)
@ -45,9 +45,9 @@ def test_user_list_filter_by_access(database: SQLite) -> None:
"""
must return users filtered by its access
"""
first = User("1", "", UserAccess.Read)
second = User("2", "", UserAccess.Full)
third = User("3", "", UserAccess.Read)
first = User(username="1", password="", access=UserAccess.Read)
second = User(username="2", password="", access=UserAccess.Full)
third = User(username="3", password="", access=UserAccess.Read)
database.user_update(first)
database.user_update(second)
@ -63,9 +63,9 @@ def test_user_list_filter_by_username_access(database: SQLite) -> None:
"""
must return users filtered by its access and username
"""
first = User("1", "", UserAccess.Read)
second = User("2", "", UserAccess.Full)
third = User("3", "", UserAccess.Read)
first = User(username="1", password="", access=UserAccess.Read)
second = User(username="2", password="", access=UserAccess.Full)
third = User(username="3", password="", access=UserAccess.Read)
database.user_update(first)
database.user_update(second)
@ -91,7 +91,6 @@ def test_user_update(database: SQLite, user: User) -> None:
database.user_update(user)
assert database.user_get(user.username) == user
new_user = user.hash_password("salt")
new_user.access = UserAccess.Full
new_user = User(username=user.username, password=user.hash_password("salt").password, access=UserAccess.Full)
database.user_update(new_user)
assert database.user_get(new_user.username) == new_user

View File

@ -2,7 +2,7 @@ from pytest_mock import MockerFixture
from unittest import mock
from ahriman.core.configuration import Configuration
from ahriman.core.report import Console
from ahriman.core.report.console import Console
from ahriman.models.package import Package
from ahriman.models.result import Result

View File

@ -3,7 +3,7 @@ import pytest
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.report import Email
from ahriman.core.report.email import Email
from ahriman.models.package import Package
from ahriman.models.result import Result
@ -90,7 +90,7 @@ def test_generate(configuration: Configuration, package_ahriman: Package, mocker
"""
must generate report
"""
send_mock = mocker.patch("ahriman.core.report.Email._send")
send_mock = mocker.patch("ahriman.core.report.email.Email._send")
report = Email("x86_64", configuration, "email")
report.generate([package_ahriman], Result())
@ -102,7 +102,7 @@ def test_generate_with_built(configuration: Configuration, package_ahriman: Pack
"""
must generate report with built packages
"""
send_mock = mocker.patch("ahriman.core.report.Email._send")
send_mock = mocker.patch("ahriman.core.report.email.Email._send")
report = Email("x86_64", configuration, "email")
report.generate([package_ahriman], result)
@ -117,7 +117,7 @@ def test_generate_with_built_and_full_path(
"""
must generate report with built packages and full packages lists
"""
send_mock = mocker.patch("ahriman.core.report.Email._send")
send_mock = mocker.patch("ahriman.core.report.email.Email._send")
report = Email("x86_64", configuration, "email")
report.full_template_path = report.template_path
@ -130,7 +130,7 @@ def test_generate_no_empty(configuration: Configuration, package_ahriman: Packag
must not generate report with built packages if no_empty_report is set
"""
configuration.set_option("email", "no_empty_report", "yes")
send_mock = mocker.patch("ahriman.core.report.Email._send")
send_mock = mocker.patch("ahriman.core.report.email.Email._send")
report = Email("x86_64", configuration, "email")
report.generate([package_ahriman], Result())
@ -143,7 +143,7 @@ def test_generate_no_empty_with_built(configuration: Configuration, package_ahri
must generate report with built packages if no_empty_report is set
"""
configuration.set_option("email", "no_empty_report", "yes")
send_mock = mocker.patch("ahriman.core.report.Email._send")
send_mock = mocker.patch("ahriman.core.report.email.Email._send")
report = Email("x86_64", configuration, "email")
report.generate([package_ahriman], result)

View File

@ -3,7 +3,7 @@ import pytest
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.report import HTML
from ahriman.core.report.html import HTML
from ahriman.models.package import Package

View File

@ -1,5 +1,5 @@
from ahriman.core.configuration import Configuration
from ahriman.core.report import JinjaTemplate
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.models.package import Package
from ahriman.models.result import Result

View File

@ -13,7 +13,7 @@ def test_report_failure(configuration: Configuration, mocker: MockerFixture) ->
"""
must raise ReportFailed on errors
"""
mocker.patch("ahriman.core.report.HTML.generate", side_effect=Exception())
mocker.patch("ahriman.core.report.html.HTML.generate", side_effect=Exception())
with pytest.raises(ReportFailed):
Report.load("x86_64", configuration, "html").run([], Result())
@ -32,7 +32,7 @@ def test_report_console(configuration: Configuration, result: Result, mocker: Mo
"""
must generate console report
"""
report_mock = mocker.patch("ahriman.core.report.Console.generate")
report_mock = mocker.patch("ahriman.core.report.console.Console.generate")
Report.load("x86_64", configuration, "console").run(result, [])
report_mock.assert_called_once_with([], result)
@ -41,7 +41,7 @@ def test_report_email(configuration: Configuration, result: Result, mocker: Mock
"""
must generate email report
"""
report_mock = mocker.patch("ahriman.core.report.Email.generate")
report_mock = mocker.patch("ahriman.core.report.email.Email.generate")
Report.load("x86_64", configuration, "email").run(result, [])
report_mock.assert_called_once_with([], result)
@ -50,7 +50,7 @@ def test_report_html(configuration: Configuration, result: Result, mocker: Mocke
"""
must generate html report
"""
report_mock = mocker.patch("ahriman.core.report.HTML.generate")
report_mock = mocker.patch("ahriman.core.report.html.HTML.generate")
Report.load("x86_64", configuration, "html").run(result, [])
report_mock.assert_called_once_with([], result)
@ -59,6 +59,6 @@ def test_report_telegram(configuration: Configuration, result: Result, mocker: M
"""
must generate telegram report
"""
report_mock = mocker.patch("ahriman.core.report.Telegram.generate")
report_mock = mocker.patch("ahriman.core.report.telegram.Telegram.generate")
Report.load("x86_64", configuration, "telegram").run(result, [])
report_mock.assert_called_once_with([], result)

View File

@ -5,7 +5,7 @@ from pytest_mock import MockerFixture
from unittest import mock
from ahriman.core.configuration import Configuration
from ahriman.core.report import Telegram
from ahriman.core.report.telegram import Telegram
from ahriman.models.package import Package
from ahriman.models.result import Result
@ -20,7 +20,8 @@ def test_send(configuration: Configuration, mocker: MockerFixture) -> None:
report._send("a text")
request_mock.assert_called_once_with(
pytest.helpers.anyvar(str, strict=True),
data={"chat_id": pytest.helpers.anyvar(str, strict=True), "text": "a text", "parse_mode": "HTML"})
data={"chat_id": pytest.helpers.anyvar(str, strict=True), "text": "a text", "parse_mode": "HTML"},
timeout=report.timeout)
def test_send_failed(configuration: Configuration, mocker: MockerFixture) -> None:
@ -50,20 +51,32 @@ def test_generate(configuration: Configuration, package_ahriman: Package, result
"""
must generate report
"""
send_mock = mocker.patch("ahriman.core.report.Telegram._send")
send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send")
report = Telegram("x86_64", configuration, "telegram")
report.generate([package_ahriman], result)
send_mock.assert_called_once_with(pytest.helpers.anyvar(int))
def test_generate_big_text_without_spaces(configuration: Configuration, package_ahriman: Package, result: Result,
mocker: MockerFixture) -> None:
"""
must raise ValueError in case if there are no new lines in text
"""
mocker.patch("ahriman.core.report.jinja_template.JinjaTemplate.make_html", return_value="ab" * 4096)
report = Telegram("x86_64", configuration, "telegram")
with pytest.raises(ValueError):
report.generate([package_ahriman], result)
def test_generate_big_text(configuration: Configuration, package_ahriman: Package, result: Result,
mocker: MockerFixture) -> None:
"""
must generate report with big text
"""
mocker.patch("ahriman.core.report.JinjaTemplate.make_html", return_value="a\n" * 4096)
send_mock = mocker.patch("ahriman.core.report.Telegram._send")
mocker.patch("ahriman.core.report.jinja_template.JinjaTemplate.make_html", return_value="a\n" * 4096)
send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send")
report = Telegram("x86_64", configuration, "telegram")
report.generate([package_ahriman], result)
@ -72,11 +85,28 @@ def test_generate_big_text(configuration: Configuration, package_ahriman: Packag
])
def test_generate_very_big_text(configuration: Configuration, package_ahriman: Package, result: Result,
mocker: MockerFixture) -> None:
"""
must generate report with very big text
"""
mocker.patch("ahriman.core.report.jinja_template.JinjaTemplate.make_html", return_value="ab\n" * 4096)
send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send")
report = Telegram("x86_64", configuration, "telegram")
report.generate([package_ahriman], result)
send_mock.assert_has_calls([
mock.call(pytest.helpers.anyvar(str, strict=True)),
mock.call(pytest.helpers.anyvar(str, strict=True)),
mock.call(pytest.helpers.anyvar(str, strict=True)),
])
def test_generate_no_empty(configuration: Configuration, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must generate report
"""
send_mock = mocker.patch("ahriman.core.report.Telegram._send")
send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send")
report = Telegram("x86_64", configuration, "telegram")
report.generate([package_ahriman], Result())

View File

@ -206,6 +206,27 @@ def test_process_update_group(executor: Executor, package_python_schedule: Packa
remove_mock.assert_called_once_with([])
def test_process_update_unsafe(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must encode file name
"""
path = "gconf-3.2.6+11+g07808097-10-x86_64.pkg.tar.zst"
safe_path = "gconf-3.2.6-11-g07808097-10-x86_64.pkg.tar.zst"
package_ahriman.packages[package_ahriman.base].filename = path
mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman])
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
move_mock = mocker.patch("shutil.move")
repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add")
executor.process_update([Path(path)])
move_mock.assert_has_calls([
mock.call(executor.paths.packages / path, executor.paths.packages / safe_path),
mock.call(executor.paths.packages / safe_path, executor.paths.repository / safe_path)
])
repo_add_mock.assert_called_once_with(executor.paths.repository / safe_path)
def test_process_empty_filename(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must skip update for package which does not have path

View File

@ -85,21 +85,21 @@ def test_packages_built(repository: Repository, mocker: MockerFixture) -> None:
assert repository.packages_built() == [Path("b.pkg.tar.xz")]
def test_packages_depends_on(repository: Repository, package_ahriman: Package, package_python_schedule: Package,
mocker: MockerFixture) -> None:
def test_packages_depend_on(repository: Repository, package_ahriman: Package, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must filter packages by depends list
"""
mocker.patch("ahriman.core.repository.repository.Repository.packages",
return_value=[package_ahriman, package_python_schedule])
assert repository.packages_depends_on(["python-aur"]) == [package_ahriman]
assert repository.packages_depend_on(["python-aur"]) == [package_ahriman]
def test_packages_depends_on_empty(repository: Repository, package_ahriman: Package, package_python_schedule: Package,
mocker: MockerFixture) -> None:
def test_packages_depend_on_empty(repository: Repository, package_ahriman: Package, package_python_schedule: Package,
mocker: MockerFixture) -> None:
"""
must return all packages in case if no filter is provided
"""
mocker.patch("ahriman.core.repository.repository.Repository.packages",
return_value=[package_ahriman, package_python_schedule])
assert repository.packages_depends_on(None) == [package_ahriman, package_python_schedule]
assert repository.packages_depend_on(None) == [package_ahriman, package_python_schedule]

View File

@ -83,7 +83,9 @@ def test_key_download(gpg: GPG, mocker: MockerFixture) -> None:
requests_mock = mocker.patch("requests.get")
gpg.key_download("pgp.mit.edu", "0xE989490C")
requests_mock.assert_called_once_with(
"http://pgp.mit.edu/pks/lookup", params={"op": "get", "options": "mr", "search": "0xE989490C"})
"http://pgp.mit.edu/pks/lookup",
params={"op": "get", "options": "mr", "search": "0xE989490C"},
timeout=gpg.DEFAULT_TIMEOUT)
def test_key_download_failure(gpg: GPG, mocker: MockerFixture) -> None:

View File

@ -51,9 +51,8 @@ def test_get_internal(client: Client) -> None:
"""
must return dummy status for web service
"""
expected = InternalStatus(BuildStatus())
actual = client.get_internal()
actual.status.timestamp = expected.status.timestamp
expected = InternalStatus(status=BuildStatus(timestamp=actual.status.timestamp))
assert actual == expected

View File

@ -164,8 +164,9 @@ def test_get_internal(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must return web service status
"""
status = InternalStatus(status=BuildStatus(), architecture="x86_64")
response_obj = Response()
response_obj._content = json.dumps(InternalStatus(BuildStatus(), architecture="x86_64").view()).encode("utf8")
response_obj._content = json.dumps(status.view()).encode("utf8")
response_obj.status_code = 200
requests_mock = mocker.patch("requests.Session.get", return_value=response_obj)

View File

@ -267,7 +267,7 @@ def test_load_logging_fallback(configuration: Configuration, mocker: MockerFixtu
"""
must fallback to stderr without errors
"""
mocker.patch("logging.config.fileConfig", side_effect=PermissionError())
mocker.patch("ahriman.core.configuration.fileConfig", side_effect=PermissionError())
configuration.load_logging(quiet=False)

View File

@ -0,0 +1,28 @@
import pytest
from ahriman.core.alpm.repo import Repo
from ahriman.core.database import SQLite
def test_logger(database: SQLite) -> None:
"""
must set logger attribute
"""
assert database.logger
assert database.logger.name == "ahriman.core.database.sqlite.SQLite"
def test_logger_attribute_error(database: SQLite) -> None:
"""
must raise AttributeError in case if no attribute found
"""
with pytest.raises(AttributeError):
database.loggerrrr
def test_logger_name(database: SQLite, repo: Repo) -> None:
"""
must correctly generate logger name
"""
assert database.logger_name == "ahriman.core.database.sqlite.SQLite"
assert repo.logger_name == "ahriman.core.alpm.repo.Repo"

View File

@ -43,19 +43,15 @@ def test_leaf_load(package_ahriman: Package, repository_paths: RepositoryPaths,
"""
must load with dependencies
"""
tempdir_mock = mocker.patch("tempfile.mkdtemp")
load_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.load")
dependencies_mock = mocker.patch("ahriman.models.package.Package.dependencies", return_value={"ahriman-dependency"})
rmtree_mock = mocker.patch("shutil.rmtree")
leaf = Leaf.load(package_ahriman, repository_paths, database)
assert leaf.package == package_ahriman
assert leaf.dependencies == {"ahriman-dependency"}
tempdir_mock.assert_called_once_with()
load_mock.assert_called_once_with(
pytest.helpers.anyvar(int), package_ahriman, None, repository_paths)
dependencies_mock.assert_called_once_with(pytest.helpers.anyvar(int))
rmtree_mock.assert_called_once_with(pytest.helpers.anyvar(int), ignore_errors=True)
def test_tree_levels(leaf_ahriman: Leaf, leaf_python_schedule: Leaf) -> None:

Some files were not shown because too many files have changed in this diff Show More