arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-22 02:09:56 +00:00
Code Issues Projects Releases Wiki Activity

feat: improve template processing (#112)

Browse Source 
* Improve template processing

* docs update, config validation rules update
This commit is contained in:
Evgenii Alekseev
2023-09-08 23:38:07 +03:00
committed by GitHub
parent a56fe28003
commit 018d9589e1
28 changed files with 413 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/ahriman/core/configuration/configuration.py
View File

@ -82,6 +82,7 @@ class Configuration(configparser.RawConfigParser):
converters={
"list": shlex.split,
"path": self._convert_path,
"pathlist": lambda value: [self._convert_path(element) for element in shlex.split(value)],
}
)
@ -241,6 +242,8 @@ class Configuration(configparser.RawConfigParser):
def getpath(self, *args: Any, **kwargs: Any) -> Path: ... # type: ignore[empty-body]
def getpathlist(self, *args: Any, **kwargs: Any) -> list[Path]: ... # type: ignore[empty-body]
def gettype(self, section: str, repository_id: RepositoryId, *, fallback: str | None = None) -> tuple[str, str]:
"""
get type variable with fallback to old logic. Despite the fact that it has same semantics as other get* methods,

75
src/ahriman/core/configuration/schema.py
View File

@ -44,12 +44,14 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"coerce": "absolute_path",
"required": True,
"path_exists": True,
"path_type": "dir",
},
"logging": {
"type": "path",
"coerce": "absolute_path",
"required": True,
"path_exists": True,
"path_type": "file",
},
"suppress_http_log_errors": {
"type": "boolean",
@ -68,12 +70,16 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"mirror": {
"type": "string",
"required": True,
"empty": False,
"is_url": [],
},
"repositories": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
"required": True,
"empty": False,
},
@ -82,6 +88,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"coerce": "absolute_path",
"required": True,
"path_exists": True,
"path_type": "dir",
},
"use_ahriman_cache": {
"type": "boolean",
@ -113,9 +120,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
},
"client_id": {
"type": "string",
"empty": False,
},
"client_secret": {
"type": "string",
"empty": False,
},
"cookie_secret_key": {
"type": "string",
@ -129,9 +138,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
},
"oauth_provider": {
"type": "string",
"empty": False,
},
"oauth_scopes": {
"type": "string",
"empty": False,
},
"salt": {
"type": "string",
@ -144,36 +155,55 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"archbuild_flags": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
"build_command": {
"type": "string",
"required": True,
"empty": False,
},
"ignore_packages": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
"makepkg_flags": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
"makechrootpkg_flags": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
"triggers": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
"triggers_known": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
"vcs_allowed_age": {
"type": "integer",
@ -187,10 +217,14 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"schema": {
"name": {
"type": "string",
"empty": False,
},
"root": {
"type": "string",
"type": "path",
"coerce": "absolute_path",
"required": True,
"path_exists": True,
"path_type": "dir",
},
},
},
@ -208,6 +242,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
},
"key": {
"type": "string",
"empty": False,
},
},
},
@ -216,6 +251,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"schema": {
"address": {
"type": "string",
"empty": False,
"is_url": ["http", "https"],
},
"debug": {
@ -229,7 +265,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"debug_allowed_hosts": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
"enable_archive_upload": {
"type": "boolean",
@ -237,10 +276,12 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
},
"host": {
"type": "string",
"empty": False,
"is_ip_address": ["localhost"],
},
"index_url": {
"type": "string",
"empty": False,
"is_url": ["http", "https"],
},
"max_body_size": {
@ -250,6 +291,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
},
"password": {
"type": "string",
"empty": False,
},
"port": {
"type": "integer",
@ -262,12 +304,18 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"coerce": "absolute_path",
"required": True,
"path_exists": True,
"path_type": "dir",
},
"templates": {
"type": "path",
"coerce": "absolute_path",
"required": True,
"path_exists": True,
"type": "list",
"coerce": "list",
"schema": {
"type": "path",
"coerce": "absolute_path",
"path_exists": True,
"path_type": "dir",
},
"empty": False,
},
"timeout": {
"type": "integer",
@ -284,6 +332,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
},
"username": {
"type": "string",
"empty": False,
},
"wait_timeout": {
"type": "integer",

18
src/ahriman/core/configuration/validator.py
View File

@ -162,3 +162,21 @@ class Validator(RootValidator):
self._error(field, f"Path {value} must not exist")
case False if constraint:
self._error(field, f"Path {value} must exist")
def _validate_path_type(self, constraint: str, field: str, value: Path) -> None:
"""
check if paths is file, directory or whatever. The match will be performed as call of ``is_{constraint}``
method of the path object
Args:
constraint(str): path type to be matched
field(str): field name to be checked
value(Path): value to be checked
Examples:
The rule's arguments are validated against this schema:
{"type": "string"}
"""
fn = getattr(value, f"is_{constraint}")
if not fn():
self._error(field, f"Path {value} must be type of {constraint}")

7
src/ahriman/core/gitremote/remote_pull_trigger.py
View File

@ -38,7 +38,10 @@ class RemotePullTrigger(Trigger):
"target": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
},
},
@ -48,9 +51,11 @@ class RemotePullTrigger(Trigger):
"pull_url": {
"type": "string",
"required": True,
"empty": False,
},
"pull_branch": {
"type": "string",
"empty": False,
},
},
},

9
src/ahriman/core/gitremote/remote_push_trigger.py
View File

@ -43,7 +43,10 @@ class RemotePushTrigger(Trigger):
"target": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
},
},
@ -52,16 +55,20 @@ class RemotePushTrigger(Trigger):
"schema": {
"commit_email": {
"type": "string",
"empty": False,
},
"commit_user": {
"type": "string",
"empty": False,
},
"push_url": {
"type": "string",
"required": True,
"empty": False,
},
"push_branch": {
"type": "string",
"empty": False,
},
},
},

21
src/ahriman/core/report/email.py
View File

@ -37,7 +37,6 @@ class Email(Report, JinjaTemplate):
email report generator
Attributes:
full_template_path(Path): path to template for full package list
host(str): SMTP host to connect
no_empty_report(bool): skip empty report generation
password(str | None): password to authenticate via SMTP
@ -45,7 +44,8 @@ class Email(Report, JinjaTemplate):
receivers(list[str]): list of receivers emails
sender(str): sender email address
ssl(SmtpSSLSettings): SSL mode for SMTP connection
template_path(Path): path to template for built packages
template(Path | str): path or name to template for built packages
template_full(Path | str | None): path or name to template for full package list
user(str | None): username to authenticate via SMTP
"""
@ -61,8 +61,10 @@ class Email(Report, JinjaTemplate):
Report.__init__(self, repository_id, configuration)
JinjaTemplate.__init__(self, repository_id, configuration, section)
self.full_template_path = configuration.getpath(section, "full_template_path", fallback=None)
self.template_path = configuration.getpath(section, "template_path")
self.template = configuration.get(section, "template", fallback=None) or \
configuration.getpath(section, "template_path")
self.template_full = configuration.get(section, "template_full", fallback=None) or \
configuration.getpath(section, "full_template_path", fallback=None)
# base smtp settings
self.host = configuration.get(section, "host")
@ -114,9 +116,10 @@ class Email(Report, JinjaTemplate):
"""
if self.no_empty_report and not result.success:
return
text = self.make_html(result, self.template_path)
if self.full_template_path is not None:
attachments = {"index.html": self.make_html(Result(success=packages), self.full_template_path)}
else:
attachments = {}
text = self.make_html(result, self.template)
attachments = {}
if self.template_full is not None:
attachments["index.html"] = self.make_html(Result(success=packages), self.template_full)
self._send(text, attachments)

7
src/ahriman/core/report/html.py
View File

@ -31,7 +31,7 @@ class HTML(Report, JinjaTemplate):
Attributes:
report_path(Path): output path to html report
template_path(Path): path to template for full package list
template(Path | str): name or path to template for full package list
"""
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
@ -47,7 +47,8 @@ class HTML(Report, JinjaTemplate):
JinjaTemplate.__init__(self, repository_id, configuration, section)
self.report_path = configuration.getpath(section, "path")
self.template_path = configuration.getpath(section, "template_path")
self.template = configuration.get(section, "template", fallback=None) or \
configuration.getpath(section, "template_path")
def generate(self, packages: list[Package], result: Result) -> None:
"""
@ -57,5 +58,5 @@ class HTML(Report, JinjaTemplate):
packages(list[Package]): list of packages to generate report
result(Result): build result
"""
html = self.make_html(Result(success=packages), self.template_path)
html = self.make_html(Result(success=packages), self.template)
self.report_path.write_text(html, encoding="utf8")

22
src/ahriman/core/report/jinja_template.py
View File

@ -57,11 +57,12 @@ class JinjaTemplate:
* repository - repository name, string, required
Attributes:
default_pgp_key(str | None): default PGP key
homepage(str | None): homepage link if any (for footer)
link_path(str): prefix fo packages to download
name(str): repository name
default_pgp_key(str | None): default PGP key
sign_targets(set[SignSettings]): targets to sign enabled in configuration
templates(list[Path]): list of directories with templates
"""
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
@ -73,26 +74,31 @@ class JinjaTemplate:
configuration(Configuration): configuration instance
section(str): settings section name
"""
self.link_path = configuration.get(section, "link_path")
self.templates = configuration.getpathlist(section, "templates", fallback=[])
# base template vars
self.homepage = configuration.get(section, "homepage", fallback=None)
self.link_path = configuration.get(section, "link_path")
self.name = repository_id.name
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
def make_html(self, result: Result, template_path: Path) -> str:
def make_html(self, result: Result, template_name: Path | str) -> str:
"""
generate report for the specified packages
Args:
result(Result): build result
template_path(Path): path to jinja template
template_name(Path | str): name of the template or path to it (legacy configuration)
"""
templates = self.templates[:]
if isinstance(template_name, Path):
templates.append(template_name.parent)
template_name = template_name.name
# idea comes from https://stackoverflow.com/a/38642558
loader = jinja2.FileSystemLoader(searchpath=template_path.parent)
environment = jinja2.Environment(loader=loader, autoescape=True)
template = environment.get_template(template_path.name)
loader = jinja2.FileSystemLoader(searchpath=templates)
environment = jinja2.Environment(trim_blocks=True, lstrip_blocks=True, autoescape=True, loader=loader)
template = environment.get_template(template_name)
content = [
{

92
src/ahriman/core/report/report_trigger.py
View File

@ -40,7 +40,10 @@ class ReportTrigger(Trigger):
"target": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
},
},
@ -67,19 +70,25 @@ class ReportTrigger(Trigger):
"full_template_path": {
"type": "path",
"coerce": "absolute_path",
"excludes": ["template_full"],
"required": True,
"path_exists": True,
"path_type": "file",
},
"homepage": {
"type": "string",
"empty": False,
"is_url": ["http", "https"],
},
"host": {
"type": "string",
"required": True,
"empty": False,
},
"link_path": {
"type": "string",
"required": True,
"empty": False,
"is_url": [],
},
"no_empty_report": {
@ -88,6 +97,7 @@ class ReportTrigger(Trigger):
},
"password": {
"type": "string",
"empty": False,
},
"port": {
"type": "integer",
@ -99,26 +109,58 @@ class ReportTrigger(Trigger):
"receivers": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
"required": True,
"empty": False,
},
"sender": {
"type": "string",
"required": True,
"empty": False,
},
"ssl": {
"type": "string",
"allowed": ["ssl", "starttls", "disabled"],
},
"template": {
"type": "string",
"excludes": ["template_path"],
"dependencies": ["templates"],
"required": True,
"empty": False,
},
"template_full": {
"type": "string",
"excludes": ["template_path"],
"dependencies": ["templates"],
"required": True,
"empty": False,
},
"template_path": {
"type": "path",
"coerce": "absolute_path",
"excludes": ["template"],
"required": True,
"path_exists": True,
"path_type": "file",
},
"templates": {
"type": "list",
"coerce": "list",
"schema": {
"type": "path",
"coerce": "absolute_path",
"path_exists": True,
"path_type": "dir",
},
"empty": False,
},
"user": {
"type": "string",
"empty": False,
},
},
},
@ -131,11 +173,13 @@ class ReportTrigger(Trigger):
},
"homepage": {
"type": "string",
"empty": False,
"is_url": ["http", "https"],
},
"link_path": {
"type": "string",
"required": True,
"empty": False,
"is_url": [],
},
"path": {
@ -143,11 +187,31 @@ class ReportTrigger(Trigger):
"coerce": "absolute_path",
"required": True,
},
"template": {
"type": "string",
"excludes": ["template_path"],
"dependencies": ["templates"],
"required": True,
"empty": False,
},
"template_path": {
"type": "path",
"coerce": "absolute_path",
"excludes": ["template"],
"required": True,
"path_exists": True,
"path_type": "file",
},
"templates": {
"type": "list",
"coerce": "list",
"schema": {
"type": "path",
"coerce": "absolute_path",
"path_exists": True,
"path_type": "dir",
},
"empty": False,
},
},
},
@ -161,30 +225,54 @@ class ReportTrigger(Trigger):
"api_key": {
"type": "string",
"required": True,
"empty": False,
},
"chat_id": {
"type": "string",
"required": True,
"empty": False,
},
"homepage": {
"type": "string",
"empty": False,
"is_url": ["http", "https"],
},
"link_path": {
"type": "string",
"required": True,
"empty": False,
"is_url": [],
},
"template": {
"type": "string",
"excludes": ["template_path"],
"dependencies": ["templates"],
"required": True,
"empty": False,
},
"template_path": {
"type": "path",
"coerce": "absolute_path",
"excludes": ["template"],
"required": True,
"path_exists": True,
"path_type": "file",
},
"template_type": {
"type": "string",
"allowed": ["MarkdownV2", "HTML", "Markdown"],
},
"templates": {
"type": "list",
"coerce": "list",
"schema": {
"type": "path",
"coerce": "absolute_path",
"path_exists": True,
"path_type": "dir",
},
"empty": False,
},
"timeout": {
"type": "integer",
"coerce": "integer",

7
src/ahriman/core/report/telegram.py
View File

@ -35,7 +35,7 @@ class Telegram(Report, JinjaTemplate, SyncHttpClient):
TELEGRAM_MAX_CONTENT_LENGTH(int): (class attribute) max content length of the message
api_key(str): bot api key
chat_id(str): chat id to post message, either string with @ or integer
template_path(Path): path to template for built packages
template(Path | str): name or path to template for built packages
template_type(str): template message type to be used in parse mode, one of MarkdownV2, HTML, Markdown
"""
@ -57,7 +57,8 @@ class Telegram(Report, JinjaTemplate, SyncHttpClient):
self.api_key = configuration.get(section, "api_key")
self.chat_id = configuration.get(section, "chat_id")
self.template_path = configuration.getpath(section, "template_path")
self.template = configuration.get(section, "template", fallback=None) or \
configuration.getpath(section, "template_path")
self.template_type = configuration.get(section, "template_type", fallback="HTML")
def _send(self, text: str) -> None:
@ -83,7 +84,7 @@ class Telegram(Report, JinjaTemplate, SyncHttpClient):
"""
if not result.success:
return
text = self.make_html(result, self.template_path)
text = self.make_html(result, self.template)
# telegram content is limited by 4096 symbols, so we are going to split the message by new lines
# to fit into this restriction
while len(text) > self.TELEGRAM_MAX_CONTENT_LENGTH:

24
src/ahriman/core/support/keyring_trigger.py
View File

@ -43,7 +43,10 @@ class KeyringTrigger(Trigger):
"target": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
},
},
@ -56,28 +59,47 @@ class KeyringTrigger(Trigger):
},
"description": {
"type": "string",
"empty": False,
},
"homepage": {
"type": "string",
"empty": False,
},
"license": {
"type": "list",
"coerce": "list",
"schema": {
"type": "string",
"empty": False,
},
},
"package": {
"type": "string",
"empty": False,
},
"packagers": {
"type": "list",
"coerce": "list",
"schema": {
"type": "string",
"empty": False,
},
},
"revoked": {
"type": "list",
"coerce": "list",
"schema": {
"type": "string",
"empty": False,
},
},
"trusted": {
"type": "list",
"coerce": "list",
"schema": {
"type": "string",
"empty": False,
},
},
},
},

17
src/ahriman/core/support/mirrorlist_trigger.py
View File

@ -39,7 +39,10 @@ class MirrorlistTrigger(Trigger):
"target": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
},
},
@ -52,16 +55,23 @@ class MirrorlistTrigger(Trigger):
},
"description": {
"type": "string",
"empty": False,
},
"homepage": {
"type": "string",
"empty": False,
},
"license": {
"type": "list",
"coerce": "list",
"schema": {
"type": "string",
"empty": False,
},
},
"package": {
"type": "string",
"empty": False,
},
"path": {
"type": "path",
@ -70,7 +80,12 @@ class MirrorlistTrigger(Trigger):
"servers": {
"type": "list",
"coerce": "list",
"schema": {
"type": "string",
"empty": False,
},
"required": True,
"empty": False,
},
},
},

20
src/ahriman/core/upload/upload_trigger.py
View File

@ -40,7 +40,10 @@ class UploadTrigger(Trigger):
"target": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
},
},
},
@ -54,13 +57,16 @@ class UploadTrigger(Trigger):
"owner": {
"type": "string",
"required": True,
"empty": False,
},
"password": {
"type": "string",
"empty": False,
},
"repository": {
"type": "string",
"required": True,
"empty": False,
},
"timeout": {
"type": "integer",
@ -73,6 +79,7 @@ class UploadTrigger(Trigger):
},
"username": {
"type": "string",
"empty": False,
},
},
},
@ -86,13 +93,17 @@ class UploadTrigger(Trigger):
"command": {
"type": "list",
"coerce": "list",
"schema": {"type": "string"},
"schema": {
"type": "string",
"empty": False,
},
"required": True,
"empty": False,
},
"remote": {
"type": "string",
"required": True,
"empty": False,
},
},
},
@ -120,10 +131,12 @@ class UploadTrigger(Trigger):
"access_key": {
"type": "string",
"required": True,
"empty": False,
},
"bucket": {
"type": "string",
"required": True,
"empty": False,
},
"chunk_size": {
"type": "integer",
@ -132,14 +145,17 @@ class UploadTrigger(Trigger):
},
"object_path": {
"type": "string",
"empty": False,
},
"region": {
"type": "string",
"required": True,
"empty": False,
},
"secret_key": {
"type": "string",
"required": True,
"empty": False,
},
},
},

3
src/ahriman/web/web.py
View File

@ -147,7 +147,8 @@ def setup_service(repository_id: RepositoryId, configuration: Configuration, spa
setup_cors(application)
application.logger.info("setup templates")
aiohttp_jinja2.setup(application, loader=jinja2.FileSystemLoader(configuration.getpath("web", "templates")))
loader = jinja2.FileSystemLoader(searchpath=configuration.getpathlist("web", "templates"))
aiohttp_jinja2.setup(application, trim_blocks=True, lstrip_blocks=True, autoescape=True, loader=loader)
application.logger.info("setup configuration")
application["configuration"] = configuration