From c6ccf5376880ca080d940ada1242c7d8b6a81c06 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Tue, 6 Apr 2021 05:45:17 +0300 Subject: [PATCH] Email report (#11) * Demo email report implementation * improved ssl mode * correct default option spelling and more fields to be hidden for not extended reports --- CONFIGURING.md | 17 ++- Makefile | 2 +- package/etc/ahriman.ini | 7 +- package/share/ahriman/repo-index.jinja2 | 48 +++---- src/ahriman/application/application.py | 23 ++-- src/ahriman/application/handlers/report.py | 2 +- src/ahriman/application/handlers/sync.py | 2 +- src/ahriman/core/report/email.py | 101 +++++++++++++++ src/ahriman/core/report/html.py | 85 ++----------- src/ahriman/core/report/jinja_template.py | 117 ++++++++++++++++++ src/ahriman/core/report/report.py | 11 +- src/ahriman/core/repository/executor.py | 10 +- src/ahriman/core/repository/repository.py | 4 +- src/ahriman/core/upload/rsync.py | 5 +- src/ahriman/core/upload/s3.py | 5 +- src/ahriman/core/upload/upload.py | 11 +- src/ahriman/core/util.py | 4 +- src/ahriman/models/report_settings.py | 5 + src/ahriman/models/smtp_ssl_settings.py | 49 ++++++++ src/ahriman/models/upload_settings.py | 1 + tests/ahriman/application/test_application.py | 9 +- tests/ahriman/core/report/test_email.py | 105 ++++++++++++++++ tests/ahriman/core/report/test_html.py | 2 +- .../core/report/test_jinja_tempalte.py | 19 +++ tests/ahriman/core/report/test_report.py | 15 ++- .../ahriman/core/repository/test_executor.py | 8 +- tests/ahriman/core/upload/test_rsync.py | 2 +- tests/ahriman/core/upload/test_s3.py | 2 +- tests/ahriman/core/upload/test_upload.py | 8 +- tests/ahriman/models/test_report_settings.py | 3 + tests/ahriman/models/test_smtp_settings.py | 21 ++++ tests/testresources/core/ahriman.ini | 8 ++ 32 files changed, 560 insertions(+), 151 deletions(-) create mode 100644 src/ahriman/core/report/email.py create mode 100644 src/ahriman/core/report/jinja_template.py create mode 100644 src/ahriman/models/smtp_ssl_settings.py create mode 100644 tests/ahriman/core/report/test_email.py create mode 100644 tests/ahriman/core/report/test_jinja_tempalte.py create mode 100644 tests/ahriman/models/test_smtp_settings.py diff --git a/CONFIGURING.md b/CONFIGURING.md index 3c140905..e0421ccd 100644 --- a/CONFIGURING.md +++ b/CONFIGURING.md @@ -47,7 +47,22 @@ Settings for signing packages or repository. Group name must refer to architectu Report generation settings. -* `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`. +* `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`, `email`. + +### `email:*` groups + +Group name must refer to architecture, e.g. it should be `email:x86_64` for x86_64 architecture. + +* `homepage` - link to homepage, string, optional. +* `host` - SMTP host for sending emails, string, required. +* `link_path` - prefix for HTML links, string, required. +* `password` - SMTP password to authenticate, string, optional. +* `port` - SMTP port for sending emails, int, required. +* `receivers` - SMTP receiver addresses, space separated list of strings, required. +* `sender` - SMTP sender address, string, required. +* `ssl` - SSL mode for SMTP connection, one of `ssl`, `starttls`, `disabled`, optional, default `disabled`. +* `template_path` - path to Jinja2 template, string, required. +* `user` - SMTP user to authenticate, string, optional. ### `html:*` groups diff --git a/Makefile b/Makefile index d9a6e713..75883ec2 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ archlinux: archive check: clean cd src && mypy --implicit-reexport --strict -p "$(PROJECT)" - find "src/$(PROJECT)" tests -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} + + find "src/$(PROJECT)" "tests/$(PROJECT)" -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} + cd src && pylint --rcfile=../.pylintrc "$(PROJECT)" clean: diff --git a/package/etc/ahriman.ini b/package/etc/ahriman.ini index 7d1b3582..1d60e650 100644 --- a/package/etc/ahriman.ini +++ b/package/etc/ahriman.ini @@ -25,10 +25,11 @@ target = [report] target = +[email] +template_path = /usr/share/ahriman/repo-index.jinja2 +ssl = disabled + [html] -path = -homepage = -link_path = template_path = /usr/share/ahriman/repo-index.jinja2 [upload] diff --git a/package/share/ahriman/repo-index.jinja2 b/package/share/ahriman/repo-index.jinja2 index 3edbb86f..71caf7ef 100644 --- a/package/share/ahriman/repo-index.jinja2 +++ b/package/share/ahriman/repo-index.jinja2 @@ -5,26 +5,30 @@ {% include "style.jinja2" %} - {% include "sorttable.jinja2" %} - {% include "search.jinja2" %} + {% if extended_report %} + {% include "sorttable.jinja2" %} + {% include "search.jinja2" %} + {% endif %}
-

Archlinux user repository

+ {% if extended_report %} +

Archlinux user repository

-
- {% if pgp_key is not none %} -

This repository is signed with {{ pgp_key|e }} by default.

- {% endif %} +
+ {% if pgp_key is not none %} +

This repository is signed with {{ pgp_key|e }} by default.

+ {% endif %} - - $ cat /etc/pacman.conf
- [{{ repository|e }}]
- Server = {{ link_path|e }}
- SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly - -
+ + $ cat /etc/pacman.conf
+ [{{ repository|e }}]
+ Server = {{ link_path|e }}
+ SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly + +
+ {% endif %} {% include "search-line.jinja2" %} @@ -50,13 +54,15 @@ -
-
    - {% if homepage is not none %} -
  • Homepage
    • - {% endif %} -
-
+ {% if extended_report %} +
+
    + {% if homepage is not none %} +
  • Homepage
    • + {% endif %} +
+
+ {% endif %}
diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index 31293c72..8272143e 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -51,12 +51,12 @@ class Application: self.architecture = architecture self.repository = Repository(architecture, configuration) - def _finalize(self) -> None: + def _finalize(self, built_packages: Iterable[Package]) -> None: """ generate report and sync to remote server """ - self.report([]) - self.sync([]) + self.report([], built_packages) + self.sync([], built_packages) def _known_packages(self) -> Set[str]: """ @@ -160,15 +160,16 @@ class Application: :param names: list of packages (either base or name) to remove """ self.repository.process_remove(names) - self._finalize() + self._finalize([]) - def report(self, target: Iterable[str]) -> None: + def report(self, target: Iterable[str], built_packages: Iterable[Package]) -> None: """ generate report :param target: list of targets to run (e.g. html) + :param built_packages: list of packages which has just been built """ targets = target or None - self.repository.process_report(targets) + self.repository.process_report(targets, built_packages) def sign(self, packages: Iterable[str]) -> None: """ @@ -191,15 +192,16 @@ class Application: self.update([]) # sign repository database if set self.repository.sign.sign_repository(self.repository.repo.repo_path) - self._finalize() + self._finalize([]) - def sync(self, target: Iterable[str]) -> None: + def sync(self, target: Iterable[str], built_packages: Iterable[Package]) -> None: """ sync to remote server :param target: list of targets to run (e.g. s3) + :param built_packages: list of packages which has just been built """ targets = target or None - self.repository.process_sync(targets) + self.repository.process_sync(targets, built_packages) def update(self, updates: Iterable[Package]) -> None: """ @@ -207,8 +209,9 @@ class Application: :param updates: list of packages to update """ def process_update(paths: Iterable[Path]) -> None: + updated = [Package.load(path, self.repository.pacman, self.repository.aur_url) for path in paths] self.repository.process_update(paths) - self._finalize() + self._finalize(updated) # process built packages packages = self.repository.packages_built() diff --git a/src/ahriman/application/handlers/report.py b/src/ahriman/application/handlers/report.py index 23a40276..79ce3c07 100644 --- a/src/ahriman/application/handlers/report.py +++ b/src/ahriman/application/handlers/report.py @@ -39,4 +39,4 @@ class Report(Handler): :param architecture: repository architecture :param configuration: configuration instance """ - Application(architecture, configuration).report(args.target) + Application(architecture, configuration).report(args.target, []) diff --git a/src/ahriman/application/handlers/sync.py b/src/ahriman/application/handlers/sync.py index 46fff7a6..bedd0416 100644 --- a/src/ahriman/application/handlers/sync.py +++ b/src/ahriman/application/handlers/sync.py @@ -39,4 +39,4 @@ class Sync(Handler): :param architecture: repository architecture :param configuration: configuration instance """ - Application(architecture, configuration).sync(args.target) + Application(architecture, configuration).sync(args.target, []) diff --git a/src/ahriman/core/report/email.py b/src/ahriman/core/report/email.py new file mode 100644 index 00000000..af42e0ad --- /dev/null +++ b/src/ahriman/core/report/email.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import datetime +import smtplib + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from typing import Dict, Iterable + +from ahriman.core.configuration import Configuration +from ahriman.core.report.jinja_template import JinjaTemplate +from ahriman.core.report.report import Report +from ahriman.core.util import pretty_datetime +from ahriman.models.package import Package +from ahriman.models.smtp_ssl_settings import SmtpSSLSettings + + +class Email(Report, JinjaTemplate): + """ + email report generator + :ivar host: SMTP host to connect + :ivar password: password to authenticate via SMTP + :ivar port: SMTP port to connect + :ivar receivers: list of receivers emails + :ivar sender: sender email address + :ivar ssl: SSL mode for SMTP connection + :ivar user: username to authenticate via SMTP + """ + + def __init__(self, architecture: str, configuration: Configuration) -> None: + """ + default constructor + :param architecture: repository architecture + :param configuration: configuration instance + """ + Report.__init__(self, architecture, configuration) + JinjaTemplate.__init__(self, "email", configuration) + + # base smtp settings + self.host = configuration.get("email", "host") + self.password = configuration.get("email", "password", fallback=None) + self.port = configuration.getint("email", "port") + self.receivers = configuration.getlist("email", "receivers") + self.sender = configuration.get("email", "sender") + self.ssl = SmtpSSLSettings.from_option(configuration.get("email", "ssl", fallback="disabled")) + self.user = configuration.get("email", "user", fallback=None) + + def _send(self, text: str, attachment: Dict[str, str]) -> None: + """ + send email callback + :param text: email body text + :param attachment: map of attachment filename to attachment content + """ + message = MIMEMultipart() + message["From"] = self.sender + message["To"] = ", ".join(self.receivers) + message["Subject"] = f"{self.name} build report at {pretty_datetime(datetime.datetime.utcnow().timestamp())}" + + message.attach(MIMEText(text, "html")) + for filename, content in attachment.items(): + attach = MIMEText(content, "html") + attach.add_header("Content-Disposition", "attachment", filename=filename) + message.attach(attach) + + if self.ssl != SmtpSSLSettings.SSL: + session = smtplib.SMTP(self.host, self.port) + if self.ssl == SmtpSSLSettings.STARTTLS: + session.starttls() + else: + session = smtplib.SMTP_SSL(self.host, self.port) + if self.user is not None and self.password is not None: + session.login(self.user, self.password) + session.sendmail(self.sender, self.receivers, message.as_string()) + session.quit() + + def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None: + """ + generate report for the specified packages + :param packages: list of packages to generate report + :param built_packages: list of packages which has just been built + """ + text = self.make_html(built_packages, False) + attachments = {"index.html": self.make_html(packages, True)} + self._send(text, attachments) diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index 4d9da26b..e547baf6 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -17,51 +17,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import jinja2 - -from typing import Callable, Dict, Iterable +from typing import Iterable from ahriman.core.configuration import Configuration +from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.report.report import Report -from ahriman.core.sign.gpg import GPG -from ahriman.core.util import pretty_datetime, pretty_size from ahriman.models.package import Package -from ahriman.models.sign_settings import SignSettings -class HTML(Report): +class HTML(Report, JinjaTemplate): """ html report generator - - It uses jinja2 templates for report generation, the following variables are allowed: - - homepage - link to homepage, string, optional - link_path - prefix fo packages to download, string, required - has_package_signed - True in case if package sign enabled, False otherwise, required - has_repo_signed - True in case if repository database sign enabled, False otherwise, required - packages - sorted list of packages properties, required - * architecture, string - * archive_size, pretty printed size, string - * build_date, pretty printed datetime, string - * depends, sorted list of strings - * description, string - * filename, string, - * groups, sorted list of strings - * installed_size, pretty printed datetime, string - * licenses, sorted list of strings - * name, string - * url, string - * version, string - pgp_key - default PGP key ID, string, optional - repository - repository name, string, required - - :ivar homepage: homepage link if any (for footer) - :ivar link_path: prefix fo packages to download - :ivar name: repository name - :ivar default_pgp_key: default PGP key :ivar report_path: output path to html report - :ivar sign_targets: targets to sign enabled in configuration - :ivar template_path: path to directory with jinja templates """ def __init__(self, architecture: str, configuration: Configuration) -> None: @@ -71,51 +38,15 @@ class HTML(Report): :param configuration: configuration instance """ Report.__init__(self, architecture, configuration) + JinjaTemplate.__init__(self, "html", configuration) + self.report_path = configuration.getpath("html", "path") - self.link_path = configuration.get("html", "link_path") - self.template_path = configuration.getpath("html", "template_path") - # base template vars - self.homepage = configuration.get("html", "homepage", fallback=None) - self.name = configuration.get("repository", "name") - - self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration) - - def generate(self, packages: Iterable[Package]) -> None: + def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None: """ generate report for the specified packages :param packages: list of packages to generate report + :param built_packages: list of packages which has just been built """ - # idea comes from https://stackoverflow.com/a/38642558 - loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent) - environment = jinja2.Environment(loader=loader) - template = environment.get_template(self.template_path.name) - - content = [ - { - "architecture": properties.architecture or "", - "archive_size": pretty_size(properties.archive_size), - "build_date": pretty_datetime(properties.build_date), - "depends": properties.depends, - "description": properties.description or "", - "filename": properties.filename, - "groups": properties.groups, - "installed_size": pretty_size(properties.installed_size), - "licenses": properties.licenses, - "name": package, - "url": properties.url or "", - "version": base.version - } for base in packages for package, properties in base.packages.items() - ] - comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"] - - html = template.render( - homepage=self.homepage, - link_path=self.link_path, - has_package_signed=SignSettings.Packages in self.sign_targets, - has_repo_signed=SignSettings.Repository in self.sign_targets, - packages=sorted(content, key=comparator), - pgp_key=self.default_pgp_key, - repository=self.name) - + html = self.make_html(packages, True) self.report_path.write_text(html) diff --git a/src/ahriman/core/report/jinja_template.py b/src/ahriman/core/report/jinja_template.py new file mode 100644 index 00000000..5e50ee2a --- /dev/null +++ b/src/ahriman/core/report/jinja_template.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import jinja2 + +from typing import Callable, Dict, Iterable + +from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG +from ahriman.core.util import pretty_datetime, pretty_size +from ahriman.models.package import Package +from ahriman.models.sign_settings import SignSettings + + +class JinjaTemplate: + """ + jinja based report generator + + It uses jinja2 templates for report generation, the following variables are allowed: + + homepage - link to homepage, string, optional + link_path - prefix fo packages to download, string, required + has_package_signed - True in case if package sign enabled, False otherwise, required + has_repo_signed - True in case if repository database sign enabled, False otherwise, required + packages - sorted list of packages properties, required + * architecture, string + * archive_size, pretty printed size, string + * build_date, pretty printed datetime, string + * depends, sorted list of strings + * description, string + * filename, string, + * groups, sorted list of strings + * installed_size, pretty printed datetime, string + * licenses, sorted list of strings + * name, string + * url, string + * version, string + pgp_key - default PGP key ID, string, optional + repository - repository name, string, required + + :ivar homepage: homepage link if any (for footer) + :ivar link_path: prefix fo packages to download + :ivar name: repository name + :ivar default_pgp_key: default PGP key + :ivar sign_targets: targets to sign enabled in configuration + :ivar template_path: path to directory with jinja templates + """ + + def __init__(self, section: str, configuration: Configuration) -> None: + """ + default constructor + :param section: settings section name + :param configuration: configuration instance + """ + self.link_path = configuration.get(section, "link_path") + self.template_path = configuration.getpath(section, "template_path") + + # base template vars + self.homepage = configuration.get(section, "homepage", fallback=None) + self.name = configuration.get("repository", "name") + + self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration) + + def make_html(self, packages: Iterable[Package], extended_report: bool) -> str: + """ + generate report for the specified packages + :param packages: list of packages to generate report + :param extended_report: include additional blocks to the report + """ + # idea comes from https://stackoverflow.com/a/38642558 + loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent) + environment = jinja2.Environment(loader=loader) + template = environment.get_template(self.template_path.name) + + content = [ + { + "architecture": properties.architecture or "", + "archive_size": pretty_size(properties.archive_size), + "build_date": pretty_datetime(properties.build_date), + "depends": properties.depends, + "description": properties.description or "", + "filename": properties.filename, + "groups": properties.groups, + "installed_size": pretty_size(properties.installed_size), + "licenses": properties.licenses, + "name": package, + "url": properties.url or "", + "version": base.version + } for base in packages for package, properties in base.packages.items() + ] + comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"] + + return template.render( + extended_report=extended_report, + homepage=self.homepage, + link_path=self.link_path, + has_package_signed=SignSettings.Packages in self.sign_targets, + has_repo_signed=SignSettings.Repository in self.sign_targets, + packages=sorted(content, key=comparator), + pgp_key=self.default_pgp_key, + repository=self.name) diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index 20406ae9..86e79c51 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -60,21 +60,26 @@ class Report: if provider == ReportSettings.HTML: from ahriman.core.report.html import HTML return HTML(architecture, configuration) + if provider == ReportSettings.Email: + from ahriman.core.report.email import Email + return Email(architecture, configuration) return cls(architecture, configuration) # should never happen - def generate(self, packages: Iterable[Package]) -> None: + def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None: """ generate report for the specified packages :param packages: list of packages to generate report + :param built_packages: list of packages which has just been built """ - def run(self, packages: Iterable[Package]) -> None: + def run(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None: """ run report generation :param packages: list of packages to generate report + :param built_packages: list of packages which has just been built """ try: - self.generate(packages) + self.generate(packages, built_packages) except Exception: self.logger.exception("report generation failed") raise ReportFailed() diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 2a94eb0f..9d691260 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -100,27 +100,29 @@ class Executor(Cleaner): return self.repo.repo_path - def process_report(self, targets: Optional[Iterable[str]]) -> None: + def process_report(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None: """ generate reports :param targets: list of targets to generate reports. Configuration option will be used if it is not set + :param built_packages: list of packages which has just been built """ if targets is None: targets = self.configuration.getlist("report", "target") for target in targets: runner = Report.load(self.architecture, self.configuration, target) - runner.run(self.packages()) + runner.run(self.packages(), built_packages) - def process_sync(self, targets: Optional[Iterable[str]]) -> None: + def process_sync(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None: """ process synchronization to remote servers :param targets: list of targets to sync. Configuration option will be used if it is not set + :param built_packages: list of packages which has just been built """ if targets is None: targets = self.configuration.getlist("upload", "target") for target in targets: runner = Upload.load(self.architecture, self.configuration, target) - runner.run(self.paths.repository) + runner.run(self.paths.repository, built_packages) def process_update(self, packages: Iterable[Path]) -> Path: """ diff --git a/src/ahriman/core/repository/repository.py b/src/ahriman/core/repository/repository.py index f849a4f6..53328869 100644 --- a/src/ahriman/core/repository/repository.py +++ b/src/ahriman/core/repository/repository.py @@ -37,9 +37,7 @@ class Repository(Executor, UpdateHandler): :return: list of packages properties """ result: Dict[str, Package] = {} - for full_path in self.paths.repository.iterdir(): - if not package_like(full_path): - continue + for full_path in filter(package_like, self.paths.repository.iterdir()): try: local = Package.load(full_path, self.pacman, self.aur_url) result.setdefault(local.base, local).packages.update(local.packages) diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index a154320d..a978ae0b 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -18,10 +18,12 @@ # along with this program. If not, see . # from pathlib import Path +from typing import Iterable from ahriman.core.configuration import Configuration from ahriman.core.upload.upload import Upload from ahriman.core.util import check_output +from ahriman.models.package import Package class Rsync(Upload): @@ -43,9 +45,10 @@ class Rsync(Upload): self.command = configuration.getlist("rsync", "command") self.remote = configuration.get("rsync", "remote") - def sync(self, path: Path) -> None: + def sync(self, path: Path, built_packages: Iterable[Package]) -> None: """ sync data to remote server :param path: local path to sync + :param built_packages: list of packages which has just been built """ Rsync._check_output(*self.command, str(path), self.remote, exception=None, logger=self.logger) diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index a48da67f..30e22d24 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -18,10 +18,12 @@ # along with this program. If not, see . # from pathlib import Path +from typing import Iterable from ahriman.core.configuration import Configuration from ahriman.core.upload.upload import Upload from ahriman.core.util import check_output +from ahriman.models.package import Package class S3(Upload): @@ -43,10 +45,11 @@ class S3(Upload): self.bucket = configuration.get("s3", "bucket") self.command = configuration.getlist("s3", "command") - def sync(self, path: Path) -> None: + def sync(self, path: Path, built_packages: Iterable[Package]) -> None: """ sync data to remote server :param path: local path to sync + :param built_packages: list of packages which has just been built """ # TODO rewrite to boto, but it is bullshit S3._check_output(*self.command, str(path), self.bucket, exception=None, logger=self.logger) diff --git a/src/ahriman/core/upload/upload.py b/src/ahriman/core/upload/upload.py index c5016404..52eb8907 100644 --- a/src/ahriman/core/upload/upload.py +++ b/src/ahriman/core/upload/upload.py @@ -22,10 +22,11 @@ from __future__ import annotations import logging from pathlib import Path -from typing import Type +from typing import Iterable, Type from ahriman.core.configuration import Configuration from ahriman.core.exceptions import SyncFailed +from ahriman.models.package import Package from ahriman.models.upload_settings import UploadSettings @@ -65,19 +66,21 @@ class Upload: return S3(architecture, configuration) return cls(architecture, configuration) # should never happen - def run(self, path: Path) -> None: + def run(self, path: Path, built_packages: Iterable[Package]) -> None: """ run remote sync :param path: local path to sync + :param built_packages: list of packages which has just been built """ try: - self.sync(path) + self.sync(path, built_packages) except Exception: self.logger.exception("remote sync failed") raise SyncFailed() - def sync(self, path: Path) -> None: + def sync(self, path: Path, built_packages: Iterable[Package]) -> None: """ sync data to remote server :param path: local path to sync + :param built_packages: list of packages which has just been built """ diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 93b19ed2..8abd5456 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -22,7 +22,7 @@ import subprocess from logging import Logger from pathlib import Path -from typing import Optional +from typing import Optional, Union from ahriman.core.exceptions import InvalidOption @@ -60,7 +60,7 @@ def package_like(filename: Path) -> bool: return ".pkg." in name and not name.endswith(".sig") -def pretty_datetime(timestamp: Optional[int]) -> str: +def pretty_datetime(timestamp: Optional[Union[float, int]]) -> str: """ convert datetime object to string :param timestamp: datetime to convert diff --git a/src/ahriman/models/report_settings.py b/src/ahriman/models/report_settings.py index cfa3e739..4c0cfa8e 100644 --- a/src/ahriman/models/report_settings.py +++ b/src/ahriman/models/report_settings.py @@ -28,11 +28,14 @@ from ahriman.core.exceptions import InvalidOption class ReportSettings(Enum): """ report targets enumeration + :cvar Disabled: option which generates no report for testing purpose :cvar HTML: html report generation + :cvar Email: email report generation """ Disabled = auto() # for testing purpose HTML = auto() + Email = auto() @classmethod def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings: @@ -43,4 +46,6 @@ class ReportSettings(Enum): """ if value.lower() in ("html",): return cls.HTML + if value.lower() in ("email",): + return cls.Email raise InvalidOption(value) diff --git a/src/ahriman/models/smtp_ssl_settings.py b/src/ahriman/models/smtp_ssl_settings.py new file mode 100644 index 00000000..473a9297 --- /dev/null +++ b/src/ahriman/models/smtp_ssl_settings.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +from enum import Enum, auto +from typing import Type + + +class SmtpSSLSettings(Enum): + """ + SMTP SSL mode enumeration + :cvar Disabled: no SSL enabled + :cvar SSL: use SMTP_SSL instead of normal SMTP client + :cvar STARTTLS: use STARTTLS in normal SMTP client + """ + + Disabled = auto() + SSL = auto() + STARTTLS = auto() + + @classmethod + def from_option(cls: Type[SmtpSSLSettings], value: str) -> SmtpSSLSettings: + """ + construct value from configuration + :param value: configuration value + :return: parsed value + """ + if value.lower() in ("ssl", "ssl/tls"): + return cls.SSL + if value.lower() in ("starttls",): + return cls.STARTTLS + return cls.Disabled diff --git a/src/ahriman/models/upload_settings.py b/src/ahriman/models/upload_settings.py index c1aeaf6f..dc1c6c1c 100644 --- a/src/ahriman/models/upload_settings.py +++ b/src/ahriman/models/upload_settings.py @@ -28,6 +28,7 @@ from ahriman.core.exceptions import InvalidOption class UploadSettings(Enum): """ remote synchronization targets enumeration + :cvar Disabled: no sync will be performed, required for testing purpose :cvar Rsync: sync via rsync :cvar S3: sync to Amazon S3 """ diff --git a/tests/ahriman/application/test_application.py b/tests/ahriman/application/test_application.py index ecd5728d..3150ba17 100644 --- a/tests/ahriman/application/test_application.py +++ b/tests/ahriman/application/test_application.py @@ -15,7 +15,7 @@ def test_finalize(application: Application, mocker: MockerFixture) -> None: report_mock = mocker.patch("ahriman.application.application.Application.report") sync_mock = mocker.patch("ahriman.application.application.Application.sync") - application._finalize() + application._finalize([]) report_mock.assert_called_once() sync_mock.assert_called_once() @@ -218,7 +218,7 @@ def test_report(application: Application, mocker: MockerFixture) -> None: must generate report """ executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_report") - application.report([]) + application.report([], []) executor_mock.assert_called_once() @@ -279,7 +279,7 @@ def test_sync(application: Application, mocker: MockerFixture) -> None: must sync to remote """ executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_sync") - application.sync([]) + application.sync([], []) executor_mock.assert_called_once() @@ -292,6 +292,7 @@ def test_update(application: Application, package_ahriman: Package, mocker: Mock mocker.patch("ahriman.core.tree.Tree.load", return_value=tree) mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[]) + mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman) build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=paths) update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update") finalize_mock = mocker.patch("ahriman.application.application.Application._finalize") @@ -299,4 +300,4 @@ def test_update(application: Application, package_ahriman: Package, mocker: Mock application.update([package_ahriman]) build_mock.assert_called_once() update_mock.assert_has_calls([mock.call([]), mock.call(paths)]) - finalize_mock.assert_has_calls([mock.call(), mock.call()]) + finalize_mock.assert_has_calls([mock.call([]), mock.call([package_ahriman])]) diff --git a/tests/ahriman/core/report/test_email.py b/tests/ahriman/core/report/test_email.py new file mode 100644 index 00000000..26d883c7 --- /dev/null +++ b/tests/ahriman/core/report/test_email.py @@ -0,0 +1,105 @@ +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.report.email import Email +from ahriman.models.package import Package + + +def test_send(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must send an email with attachment + """ + smtp_mock = mocker.patch("smtplib.SMTP") + + report = Email("x86_64", configuration) + report._send("a text", {"attachment.html": "an attachment"}) + smtp_mock.return_value.starttls.assert_not_called() + smtp_mock.return_value.login.assert_not_called() + smtp_mock.return_value.sendmail.assert_called_once() + smtp_mock.return_value.quit.assert_called_once() + + +def test_send_auth(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must send an email with attachment with auth + """ + configuration.set("email", "user", "username") + configuration.set("email", "password", "password") + smtp_mock = mocker.patch("smtplib.SMTP") + + report = Email("x86_64", configuration) + report._send("a text", {"attachment.html": "an attachment"}) + smtp_mock.return_value.login.assert_called_once() + + +def test_send_auth_no_password(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must send an email with attachment without auth if no password supplied + """ + configuration.set("email", "user", "username") + smtp_mock = mocker.patch("smtplib.SMTP") + + report = Email("x86_64", configuration) + report._send("a text", {"attachment.html": "an attachment"}) + smtp_mock.return_value.login.assert_not_called() + + +def test_send_auth_no_user(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must send an email with attachment without auth if no user supplied + """ + configuration.set("email", "password", "password") + smtp_mock = mocker.patch("smtplib.SMTP") + + report = Email("x86_64", configuration) + report._send("a text", {"attachment.html": "an attachment"}) + smtp_mock.return_value.login.assert_not_called() + + +def test_send_ssl_tls(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must send an email with attachment with ssl/tls + """ + configuration.set("email", "ssl", "ssl") + smtp_mock = mocker.patch("smtplib.SMTP_SSL") + + report = Email("x86_64", configuration) + report._send("a text", {"attachment.html": "an attachment"}) + smtp_mock.return_value.starttls.assert_not_called() + smtp_mock.return_value.login.assert_not_called() + smtp_mock.return_value.sendmail.assert_called_once() + smtp_mock.return_value.quit.assert_called_once() + + +def test_send_starttls(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must send an email with attachment with starttls + """ + configuration.set("email", "ssl", "starttls") + smtp_mock = mocker.patch("smtplib.SMTP") + + report = Email("x86_64", configuration) + report._send("a text", {"attachment.html": "an attachment"}) + smtp_mock.return_value.starttls.assert_called_once() + + +def test_generate(configuration: Configuration, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must generate report + """ + send_mock = mocker.patch("ahriman.core.report.email.Email._send") + + report = Email("x86_64", configuration) + report.generate([package_ahriman], []) + send_mock.assert_called_once() + + +def test_generate_with_built(configuration: Configuration, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must generate report with built packages + """ + send_mock = mocker.patch("ahriman.core.report.email.Email._send") + + report = Email("x86_64", configuration) + report.generate([package_ahriman], [package_ahriman]) + send_mock.assert_called_once() diff --git a/tests/ahriman/core/report/test_html.py b/tests/ahriman/core/report/test_html.py index 075ea1ca..4b611324 100644 --- a/tests/ahriman/core/report/test_html.py +++ b/tests/ahriman/core/report/test_html.py @@ -12,5 +12,5 @@ def test_generate(configuration: Configuration, package_ahriman: Package, mocker write_mock = mocker.patch("pathlib.Path.write_text") report = HTML("x86_64", configuration) - report.generate([package_ahriman]) + report.generate([package_ahriman], []) write_mock.assert_called_once() diff --git a/tests/ahriman/core/report/test_jinja_tempalte.py b/tests/ahriman/core/report/test_jinja_tempalte.py new file mode 100644 index 00000000..a23d6e54 --- /dev/null +++ b/tests/ahriman/core/report/test_jinja_tempalte.py @@ -0,0 +1,19 @@ +from ahriman.core.configuration import Configuration +from ahriman.core.report.jinja_template import JinjaTemplate +from ahriman.models.package import Package + + +def test_generate(configuration: Configuration, package_ahriman: Package) -> None: + """ + must generate html report + """ + report = JinjaTemplate("html", configuration) + assert report.make_html([package_ahriman], extended_report=False) + + +def test_generate_extended(configuration: Configuration, package_ahriman: Package) -> None: + """ + must generate extended html report + """ + report = JinjaTemplate("html", configuration) + assert report.make_html([package_ahriman], extended_report=True) diff --git a/tests/ahriman/core/report/test_report.py b/tests/ahriman/core/report/test_report.py index 8971d504..4300bde4 100644 --- a/tests/ahriman/core/report/test_report.py +++ b/tests/ahriman/core/report/test_report.py @@ -15,7 +15,7 @@ def test_report_failure(configuration: Configuration, mocker: MockerFixture) -> """ mocker.patch("ahriman.core.report.html.HTML.generate", side_effect=Exception()) with pytest.raises(ReportFailed): - Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path")) + Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path"), []) def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> None: @@ -24,7 +24,16 @@ def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> No """ mocker.patch("ahriman.models.report_settings.ReportSettings.from_option", return_value=ReportSettings.Disabled) report_mock = mocker.patch("ahriman.core.report.report.Report.generate") - Report.load("x86_64", configuration, ReportSettings.Disabled.name).run(Path("path")) + Report.load("x86_64", configuration, ReportSettings.Disabled.name).run(Path("path"), []) + report_mock.assert_called_once() + + +def test_report_email(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must generate email report + """ + report_mock = mocker.patch("ahriman.core.report.email.Email.generate") + Report.load("x86_64", configuration, ReportSettings.Email.name).run(Path("path"), []) report_mock.assert_called_once() @@ -33,5 +42,5 @@ def test_report_html(configuration: Configuration, mocker: MockerFixture) -> Non must generate html report """ report_mock = mocker.patch("ahriman.core.report.html.HTML.generate") - Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path")) + Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path"), []) report_mock.assert_called_once() diff --git a/tests/ahriman/core/repository/test_executor.py b/tests/ahriman/core/repository/test_executor.py index 7a7ddef8..8bbd6ba4 100644 --- a/tests/ahriman/core/repository/test_executor.py +++ b/tests/ahriman/core/repository/test_executor.py @@ -133,7 +133,7 @@ def test_process_report(executor: Executor, package_ahriman: Package, mocker: Mo mocker.patch("ahriman.core.report.report.Report.load", return_value=Report("x86_64", executor.configuration)) report_mock = mocker.patch("ahriman.core.report.report.Report.run") - executor.process_report(["dummy"]) + executor.process_report(["dummy"], []) report_mock.assert_called_once() @@ -143,7 +143,7 @@ def test_process_report_auto(executor: Executor, mocker: MockerFixture) -> None: """ configuration_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist") - executor.process_report(None) + executor.process_report(None, []) configuration_getlist_mock.assert_called_once() @@ -154,7 +154,7 @@ def test_process_upload(executor: Executor, mocker: MockerFixture) -> None: mocker.patch("ahriman.core.upload.upload.Upload.load", return_value=Upload("x86_64", executor.configuration)) upload_mock = mocker.patch("ahriman.core.upload.upload.Upload.run") - executor.process_sync(["dummy"]) + executor.process_sync(["dummy"], []) upload_mock.assert_called_once() @@ -164,7 +164,7 @@ def test_process_upload_auto(executor: Executor, mocker: MockerFixture) -> None: """ configuration_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist") - executor.process_sync(None) + executor.process_sync(None, []) configuration_getlist_mock.assert_called_once() diff --git a/tests/ahriman/core/upload/test_rsync.py b/tests/ahriman/core/upload/test_rsync.py index 71c6977d..69a4bc7d 100644 --- a/tests/ahriman/core/upload/test_rsync.py +++ b/tests/ahriman/core/upload/test_rsync.py @@ -12,5 +12,5 @@ def test_sync(configuration: Configuration, mocker: MockerFixture) -> None: check_output_mock = mocker.patch("ahriman.core.upload.rsync.Rsync._check_output") upload = Rsync("x86_64", configuration) - upload.sync(Path("path")) + upload.sync(Path("path"), []) check_output_mock.assert_called_once() diff --git a/tests/ahriman/core/upload/test_s3.py b/tests/ahriman/core/upload/test_s3.py index c5b35821..e8bc4a72 100644 --- a/tests/ahriman/core/upload/test_s3.py +++ b/tests/ahriman/core/upload/test_s3.py @@ -12,5 +12,5 @@ def test_sync(configuration: Configuration, mocker: MockerFixture) -> None: check_output_mock = mocker.patch("ahriman.core.upload.s3.S3._check_output") upload = S3("x86_64", configuration) - upload.sync(Path("path")) + upload.sync(Path("path"), []) check_output_mock.assert_called_once() diff --git a/tests/ahriman/core/upload/test_upload.py b/tests/ahriman/core/upload/test_upload.py index 37664fc1..5a4278bb 100644 --- a/tests/ahriman/core/upload/test_upload.py +++ b/tests/ahriman/core/upload/test_upload.py @@ -15,7 +15,7 @@ def test_upload_failure(configuration: Configuration, mocker: MockerFixture) -> """ mocker.patch("ahriman.core.upload.rsync.Rsync.sync", side_effect=Exception()) with pytest.raises(SyncFailed): - Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path")) + Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path"), []) def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> None: @@ -24,7 +24,7 @@ def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> No """ mocker.patch("ahriman.models.upload_settings.UploadSettings.from_option", return_value=UploadSettings.Disabled) upload_mock = mocker.patch("ahriman.core.upload.upload.Upload.sync") - Upload.load("x86_64", configuration, UploadSettings.Disabled.name).run(Path("path")) + Upload.load("x86_64", configuration, UploadSettings.Disabled.name).run(Path("path"), []) upload_mock.assert_called_once() @@ -33,7 +33,7 @@ def test_upload_rsync(configuration: Configuration, mocker: MockerFixture) -> No must upload via rsync """ upload_mock = mocker.patch("ahriman.core.upload.rsync.Rsync.sync") - Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path")) + Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path"), []) upload_mock.assert_called_once() @@ -42,5 +42,5 @@ def test_upload_s3(configuration: Configuration, mocker: MockerFixture) -> None: must upload via s3 """ upload_mock = mocker.patch("ahriman.core.upload.s3.S3.sync") - Upload.load("x86_64", configuration, UploadSettings.S3.name).run(Path("path")) + Upload.load("x86_64", configuration, UploadSettings.S3.name).run(Path("path"), []) upload_mock.assert_called_once() diff --git a/tests/ahriman/models/test_report_settings.py b/tests/ahriman/models/test_report_settings.py index 8fa53f5d..effb67c9 100644 --- a/tests/ahriman/models/test_report_settings.py +++ b/tests/ahriman/models/test_report_settings.py @@ -18,3 +18,6 @@ def test_from_option_valid() -> None: """ assert ReportSettings.from_option("html") == ReportSettings.HTML assert ReportSettings.from_option("HTML") == ReportSettings.HTML + + assert ReportSettings.from_option("email") == ReportSettings.Email + assert ReportSettings.from_option("EmAil") == ReportSettings.Email diff --git a/tests/ahriman/models/test_smtp_settings.py b/tests/ahriman/models/test_smtp_settings.py new file mode 100644 index 00000000..06bd0c7d --- /dev/null +++ b/tests/ahriman/models/test_smtp_settings.py @@ -0,0 +1,21 @@ +from ahriman.models.smtp_ssl_settings import SmtpSSLSettings + + +def test_from_option_invalid() -> None: + """ + must return disabled value on invalid option + """ + assert SmtpSSLSettings.from_option("invalid") == SmtpSSLSettings.Disabled + + +def test_from_option_valid() -> None: + """ + must return value from valid options + """ + assert SmtpSSLSettings.from_option("ssl") == SmtpSSLSettings.SSL + assert SmtpSSLSettings.from_option("SSL") == SmtpSSLSettings.SSL + assert SmtpSSLSettings.from_option("ssl/tls") == SmtpSSLSettings.SSL + assert SmtpSSLSettings.from_option("SSL/TLS") == SmtpSSLSettings.SSL + + assert SmtpSSLSettings.from_option("starttls") == SmtpSSLSettings.STARTTLS + assert SmtpSSLSettings.from_option("STARTTLS") == SmtpSSLSettings.STARTTLS diff --git a/tests/testresources/core/ahriman.ini b/tests/testresources/core/ahriman.ini index b5432f05..e80977a6 100644 --- a/tests/testresources/core/ahriman.ini +++ b/tests/testresources/core/ahriman.ini @@ -25,6 +25,14 @@ target = [report] target = +[email] +host = 0.0.0.0 +link_path = +port = 587 +receivers = mail@example.com +sender = mail@example.com +template_path = ../web/templates/repo-index.jinja2 + [html] path = homepage =