arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 15:27:17 +00:00
Code Issues Projects Releases Wiki Activity

Email report (#11)

Browse Source 
* Demo email report implementation

* improved ssl mode

* correct default option spelling and more fields to be hidden for not
extended reports
This commit is contained in:
Evgenii Alekseev 2021-04-06 05:45:17 +03:00 committed by GitHub
parent ce0c07cbd9
commit c6ccf53768
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 560 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
CONFIGURING.md
View File

@ -47,7 +47,22 @@ Settings for signing packages or repository. Group name must refer to architectu
Report generation settings. Report generation settings.
* `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`. * `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`, `email`.
### `email:*` groups
Group name must refer to architecture, e.g. it should be `email:x86_64` for x86_64 architecture.
* `homepage` - link to homepage, string, optional.
* `host` - SMTP host for sending emails, string, required.
* `link_path` - prefix for HTML links, string, required.
* `password` - SMTP password to authenticate, string, optional.
* `port` - SMTP port for sending emails, int, required.
* `receivers` - SMTP receiver addresses, space separated list of strings, required.
* `sender` - SMTP sender address, string, required.
* `ssl` - SSL mode for SMTP connection, one of `ssl`, `starttls`, `disabled`, optional, default `disabled`.
* `template_path` - path to Jinja2 template, string, required.
* `user` - SMTP user to authenticate, string, optional.
### `html:*` groups ### `html:*` groups

2
Makefile
View File

@ -25,7 +25,7 @@ archlinux: archive
check: clean check: clean
cd src && mypy --implicit-reexport --strict -p "$(PROJECT)" cd src && mypy --implicit-reexport --strict -p "$(PROJECT)"
find "src/$(PROJECT)" tests -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} + find "src/$(PROJECT)" "tests/$(PROJECT)" -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} +
cd src && pylint --rcfile=../.pylintrc "$(PROJECT)" cd src && pylint --rcfile=../.pylintrc "$(PROJECT)"
clean: clean:

7
package/etc/ahriman.ini
View File

@ -25,10 +25,11 @@ target =
[report] [report]
target = target =
[email]
template_path = /usr/share/ahriman/repo-index.jinja2
ssl = disabled
[html] [html]
path =
homepage =
link_path =
template_path = /usr/share/ahriman/repo-index.jinja2 template_path = /usr/share/ahriman/repo-index.jinja2
[upload] [upload]

48
package/share/ahriman/repo-index.jinja2
View File

@ -5,26 +5,30 @@
{% include "style.jinja2" %} {% include "style.jinja2" %}
{% include "sorttable.jinja2" %} {% if extended_report %}
{% include "search.jinja2" %} {% include "sorttable.jinja2" %}
{% include "search.jinja2" %}
{% endif %}
</head> </head>
<body> <body>
<div class="root"> <div class="root">
<h1>Archlinux user repository</h1> {% if extended_report %}
<h1>Archlinux user repository</h1>
<section class="element"> <section class="element">
{% if pgp_key is not none %} {% if pgp_key is not none %}
<p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}&fingerprint=on&op=index" title="key search">{{ pgp_key|e }}</a> by default.</p> <p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}&fingerprint=on&op=index" title="key search">{{ pgp_key|e }}</a> by default.</p>
{% endif %} {% endif %}
<code> <code>
$ cat /etc/pacman.conf<br> $ cat /etc/pacman.conf<br>
[{{ repository|e }}]<br> [{{ repository|e }}]<br>
Server = {{ link_path|e }}<br> Server = {{ link_path|e }}<br>
SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly
</code> </code>
</section> </section>
{% endif %}
{% include "search-line.jinja2" %} {% include "search-line.jinja2" %}
@ -50,13 +54,15 @@
</table> </table>
</section> </section>
<footer> {% if extended_report %}
<ul class="navigation"> <footer>
{% if homepage is not none %} <ul class="navigation">
<li><a href="{{ homepage|e }}" title="homepage">Homepage</a></li> {% if homepage is not none %}
{% endif %} <li><a href="{{ homepage|e }}" title="homepage">Homepage</a></li>
</ul> {% endif %}
</footer> </ul>
</footer>
{% endif %}
</div> </div>
</body> </body>
</html> </html>

23
src/ahriman/application/application.py
View File

@ -51,12 +51,12 @@ class Application:
self.architecture = architecture self.architecture = architecture
self.repository = Repository(architecture, configuration) self.repository = Repository(architecture, configuration)
def _finalize(self) -> None: def _finalize(self, built_packages: Iterable[Package]) -> None:
""" """
generate report and sync to remote server generate report and sync to remote server
""" """
self.report([]) self.report([], built_packages)
self.sync([]) self.sync([], built_packages)
def _known_packages(self) -> Set[str]: def _known_packages(self) -> Set[str]:
""" """
@ -160,15 +160,16 @@ class Application:
:param names: list of packages (either base or name) to remove :param names: list of packages (either base or name) to remove
""" """
self.repository.process_remove(names) self.repository.process_remove(names)
self._finalize() self._finalize([])
def report(self, target: Iterable[str]) -> None: def report(self, target: Iterable[str], built_packages: Iterable[Package]) -> None:
""" """
generate report generate report
:param target: list of targets to run (e.g. html) :param target: list of targets to run (e.g. html)
:param built_packages: list of packages which has just been built
""" """
targets = target or None targets = target or None
self.repository.process_report(targets) self.repository.process_report(targets, built_packages)
def sign(self, packages: Iterable[str]) -> None: def sign(self, packages: Iterable[str]) -> None:
""" """
@ -191,15 +192,16 @@ class Application:
self.update([]) self.update([])
# sign repository database if set # sign repository database if set
self.repository.sign.sign_repository(self.repository.repo.repo_path) self.repository.sign.sign_repository(self.repository.repo.repo_path)
self._finalize() self._finalize([])
def sync(self, target: Iterable[str]) -> None: def sync(self, target: Iterable[str], built_packages: Iterable[Package]) -> None:
""" """
sync to remote server sync to remote server
:param target: list of targets to run (e.g. s3) :param target: list of targets to run (e.g. s3)
:param built_packages: list of packages which has just been built
""" """
targets = target or None targets = target or None
self.repository.process_sync(targets) self.repository.process_sync(targets, built_packages)
def update(self, updates: Iterable[Package]) -> None: def update(self, updates: Iterable[Package]) -> None:
""" """
@ -207,8 +209,9 @@ class Application:
:param updates: list of packages to update :param updates: list of packages to update
""" """
def process_update(paths: Iterable[Path]) -> None: def process_update(paths: Iterable[Path]) -> None:
updated = [Package.load(path, self.repository.pacman, self.repository.aur_url) for path in paths]
self.repository.process_update(paths) self.repository.process_update(paths)
self._finalize() self._finalize(updated)
# process built packages # process built packages
packages = self.repository.packages_built() packages = self.repository.packages_built()

2
src/ahriman/application/handlers/report.py
View File

@ -39,4 +39,4 @@ class Report(Handler):
:param architecture: repository architecture :param architecture: repository architecture
:param configuration: configuration instance :param configuration: configuration instance
""" """
Application(architecture, configuration).report(args.target) Application(architecture, configuration).report(args.target, [])

2
src/ahriman/application/handlers/sync.py
View File

@ -39,4 +39,4 @@ class Sync(Handler):
:param architecture: repository architecture :param architecture: repository architecture
:param configuration: configuration instance :param configuration: configuration instance
""" """
Application(architecture, configuration).sync(args.target) Application(architecture, configuration).sync(args.target, [])

101
src/ahriman/core/report/email.py Normal file
View File

@ -0,0 +1,101 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import Dict, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.core.util import pretty_datetime
from ahriman.models.package import Package
from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
class Email(Report, JinjaTemplate):
"""
email report generator
:ivar host: SMTP host to connect
:ivar password: password to authenticate via SMTP
:ivar port: SMTP port to connect
:ivar receivers: list of receivers emails
:ivar sender: sender email address
:ivar ssl: SSL mode for SMTP connection
:ivar user: username to authenticate via SMTP
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
default constructor
:param architecture: repository architecture
:param configuration: configuration instance
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, "email", configuration)
# base smtp settings
self.host = configuration.get("email", "host")
self.password = configuration.get("email", "password", fallback=None)
self.port = configuration.getint("email", "port")
self.receivers = configuration.getlist("email", "receivers")
self.sender = configuration.get("email", "sender")
self.ssl = SmtpSSLSettings.from_option(configuration.get("email", "ssl", fallback="disabled"))
self.user = configuration.get("email", "user", fallback=None)
def _send(self, text: str, attachment: Dict[str, str]) -> None:
"""
send email callback
:param text: email body text
:param attachment: map of attachment filename to attachment content
"""
message = MIMEMultipart()
message["From"] = self.sender
message["To"] = ", ".join(self.receivers)
message["Subject"] = f"{self.name} build report at {pretty_datetime(datetime.datetime.utcnow().timestamp())}"
message.attach(MIMEText(text, "html"))
for filename, content in attachment.items():
attach = MIMEText(content, "html")
attach.add_header("Content-Disposition", "attachment", filename=filename)
message.attach(attach)
if self.ssl != SmtpSSLSettings.SSL:
session = smtplib.SMTP(self.host, self.port)
if self.ssl == SmtpSSLSettings.STARTTLS:
session.starttls()
else:
session = smtplib.SMTP_SSL(self.host, self.port)
if self.user is not None and self.password is not None:
session.login(self.user, self.password)
session.sendmail(self.sender, self.receivers, message.as_string())
session.quit()
def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
"""
generate report for the specified packages
:param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
"""
text = self.make_html(built_packages, False)
attachments = {"index.html": self.make_html(packages, True)}
self._send(text, attachments)

85
src/ahriman/core/report/html.py
View File

@ -17,51 +17,18 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
import jinja2 from typing import Iterable
from typing import Callable, Dict, Iterable
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report from ahriman.core.report.report import Report
from ahriman.core.sign.gpg import GPG
from ahriman.core.util import pretty_datetime, pretty_size
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.sign_settings import SignSettings
class HTML(Report): class HTML(Report, JinjaTemplate):
""" """
html report generator html report generator
It uses jinja2 templates for report generation, the following variables are allowed:
homepage - link to homepage, string, optional
link_path - prefix fo packages to download, string, required
has_package_signed - True in case if package sign enabled, False otherwise, required
has_repo_signed - True in case if repository database sign enabled, False otherwise, required
packages - sorted list of packages properties, required
* architecture, string
* archive_size, pretty printed size, string
* build_date, pretty printed datetime, string
* depends, sorted list of strings
* description, string
* filename, string,
* groups, sorted list of strings
* installed_size, pretty printed datetime, string
* licenses, sorted list of strings
* name, string
* url, string
* version, string
pgp_key - default PGP key ID, string, optional
repository - repository name, string, required
:ivar homepage: homepage link if any (for footer)
:ivar link_path: prefix fo packages to download
:ivar name: repository name
:ivar default_pgp_key: default PGP key
:ivar report_path: output path to html report :ivar report_path: output path to html report
:ivar sign_targets: targets to sign enabled in configuration
:ivar template_path: path to directory with jinja templates
""" """
def __init__(self, architecture: str, configuration: Configuration) -> None: def __init__(self, architecture: str, configuration: Configuration) -> None:
@ -71,51 +38,15 @@ class HTML(Report):
:param configuration: configuration instance :param configuration: configuration instance
""" """
Report.__init__(self, architecture, configuration) Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, "html", configuration)
self.report_path = configuration.getpath("html", "path") self.report_path = configuration.getpath("html", "path")
self.link_path = configuration.get("html", "link_path")
self.template_path = configuration.getpath("html", "template_path")
# base template vars def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
self.homepage = configuration.get("html", "homepage", fallback=None)
self.name = configuration.get("repository", "name")
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
def generate(self, packages: Iterable[Package]) -> None:
""" """
generate report for the specified packages generate report for the specified packages
:param packages: list of packages to generate report :param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
""" """
# idea comes from https://stackoverflow.com/a/38642558 html = self.make_html(packages, True)
loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent)
environment = jinja2.Environment(loader=loader)
template = environment.get_template(self.template_path.name)
content = [
{
"architecture": properties.architecture or "",
"archive_size": pretty_size(properties.archive_size),
"build_date": pretty_datetime(properties.build_date),
"depends": properties.depends,
"description": properties.description or "",
"filename": properties.filename,
"groups": properties.groups,
"installed_size": pretty_size(properties.installed_size),
"licenses": properties.licenses,
"name": package,
"url": properties.url or "",
"version": base.version
} for base in packages for package, properties in base.packages.items()
]
comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
html = template.render(
homepage=self.homepage,
link_path=self.link_path,
has_package_signed=SignSettings.Packages in self.sign_targets,
has_repo_signed=SignSettings.Repository in self.sign_targets,
packages=sorted(content, key=comparator),
pgp_key=self.default_pgp_key,
repository=self.name)
self.report_path.write_text(html) self.report_path.write_text(html)

117
src/ahriman/core/report/jinja_template.py Normal file
View File

@ -0,0 +1,117 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import jinja2
from typing import Callable, Dict, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.util import pretty_datetime, pretty_size
from ahriman.models.package import Package
from ahriman.models.sign_settings import SignSettings
class JinjaTemplate:
"""
jinja based report generator
It uses jinja2 templates for report generation, the following variables are allowed:
homepage - link to homepage, string, optional
link_path - prefix fo packages to download, string, required
has_package_signed - True in case if package sign enabled, False otherwise, required
has_repo_signed - True in case if repository database sign enabled, False otherwise, required
packages - sorted list of packages properties, required
* architecture, string
* archive_size, pretty printed size, string
* build_date, pretty printed datetime, string
* depends, sorted list of strings
* description, string
* filename, string,
* groups, sorted list of strings
* installed_size, pretty printed datetime, string
* licenses, sorted list of strings
* name, string
* url, string
* version, string
pgp_key - default PGP key ID, string, optional
repository - repository name, string, required
:ivar homepage: homepage link if any (for footer)
:ivar link_path: prefix fo packages to download
:ivar name: repository name
:ivar default_pgp_key: default PGP key
:ivar sign_targets: targets to sign enabled in configuration
:ivar template_path: path to directory with jinja templates
"""
def __init__(self, section: str, configuration: Configuration) -> None:
"""
default constructor
:param section: settings section name
:param configuration: configuration instance
"""
self.link_path = configuration.get(section, "link_path")
self.template_path = configuration.getpath(section, "template_path")
# base template vars
self.homepage = configuration.get(section, "homepage", fallback=None)
self.name = configuration.get("repository", "name")
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
def make_html(self, packages: Iterable[Package], extended_report: bool) -> str:
"""
generate report for the specified packages
:param packages: list of packages to generate report
:param extended_report: include additional blocks to the report
"""
# idea comes from https://stackoverflow.com/a/38642558
loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent)
environment = jinja2.Environment(loader=loader)
template = environment.get_template(self.template_path.name)
content = [
{
"architecture": properties.architecture or "",
"archive_size": pretty_size(properties.archive_size),
"build_date": pretty_datetime(properties.build_date),
"depends": properties.depends,
"description": properties.description or "",
"filename": properties.filename,
"groups": properties.groups,
"installed_size": pretty_size(properties.installed_size),
"licenses": properties.licenses,
"name": package,
"url": properties.url or "",
"version": base.version
} for base in packages for package, properties in base.packages.items()
]
comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
return template.render(
extended_report=extended_report,
homepage=self.homepage,
link_path=self.link_path,
has_package_signed=SignSettings.Packages in self.sign_targets,
has_repo_signed=SignSettings.Repository in self.sign_targets,
packages=sorted(content, key=comparator),
pgp_key=self.default_pgp_key,
repository=self.name)

11
src/ahriman/core/report/report.py
View File

@ -60,21 +60,26 @@ class Report:
if provider == ReportSettings.HTML: if provider == ReportSettings.HTML:
from ahriman.core.report.html import HTML from ahriman.core.report.html import HTML
return HTML(architecture, configuration) return HTML(architecture, configuration)
if provider == ReportSettings.Email:
from ahriman.core.report.email import Email
return Email(architecture, configuration)
return cls(architecture, configuration) # should never happen return cls(architecture, configuration) # should never happen
def generate(self, packages: Iterable[Package]) -> None: def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
""" """
generate report for the specified packages generate report for the specified packages
:param packages: list of packages to generate report :param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
""" """
def run(self, packages: Iterable[Package]) -> None: def run(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
""" """
run report generation run report generation
:param packages: list of packages to generate report :param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
""" """
try: try:
self.generate(packages) self.generate(packages, built_packages)
except Exception: except Exception:
self.logger.exception("report generation failed") self.logger.exception("report generation failed")
raise ReportFailed() raise ReportFailed()

10
src/ahriman/core/repository/executor.py
View File

@ -100,27 +100,29 @@ class Executor(Cleaner):
return self.repo.repo_path return self.repo.repo_path
def process_report(self, targets: Optional[Iterable[str]]) -> None: def process_report(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None:
""" """
generate reports generate reports
:param targets: list of targets to generate reports. Configuration option will be used if it is not set :param targets: list of targets to generate reports. Configuration option will be used if it is not set
:param built_packages: list of packages which has just been built
""" """
if targets is None: if targets is None:
targets = self.configuration.getlist("report", "target") targets = self.configuration.getlist("report", "target")
for target in targets: for target in targets:
runner = Report.load(self.architecture, self.configuration, target) runner = Report.load(self.architecture, self.configuration, target)
runner.run(self.packages()) runner.run(self.packages(), built_packages)
def process_sync(self, targets: Optional[Iterable[str]]) -> None: def process_sync(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None:
""" """
process synchronization to remote servers process synchronization to remote servers
:param targets: list of targets to sync. Configuration option will be used if it is not set :param targets: list of targets to sync. Configuration option will be used if it is not set
:param built_packages: list of packages which has just been built
""" """
if targets is None: if targets is None:
targets = self.configuration.getlist("upload", "target") targets = self.configuration.getlist("upload", "target")
for target in targets: for target in targets:
runner = Upload.load(self.architecture, self.configuration, target) runner = Upload.load(self.architecture, self.configuration, target)
runner.run(self.paths.repository) runner.run(self.paths.repository, built_packages)
def process_update(self, packages: Iterable[Path]) -> Path: def process_update(self, packages: Iterable[Path]) -> Path:
""" """

4
src/ahriman/core/repository/repository.py
View File

@ -37,9 +37,7 @@ class Repository(Executor, UpdateHandler):
:return: list of packages properties :return: list of packages properties
""" """
result: Dict[str, Package] = {} result: Dict[str, Package] = {}
for full_path in self.paths.repository.iterdir(): for full_path in filter(package_like, self.paths.repository.iterdir()):
if not package_like(full_path):
continue
try: try:
local = Package.load(full_path, self.pacman, self.aur_url) local = Package.load(full_path, self.pacman, self.aur_url)
result.setdefault(local.base, local).packages.update(local.packages) result.setdefault(local.base, local).packages.update(local.packages)

5
src/ahriman/core/upload/rsync.py
View File

@ -18,10 +18,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from pathlib import Path from pathlib import Path
from typing import Iterable
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload from ahriman.core.upload.upload import Upload
from ahriman.core.util import check_output from ahriman.core.util import check_output
from ahriman.models.package import Package
class Rsync(Upload): class Rsync(Upload):
@ -43,9 +45,10 @@ class Rsync(Upload):
self.command = configuration.getlist("rsync", "command") self.command = configuration.getlist("rsync", "command")
self.remote = configuration.get("rsync", "remote") self.remote = configuration.get("rsync", "remote")
def sync(self, path: Path) -> None: def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
""" """
sync data to remote server sync data to remote server
:param path: local path to sync :param path: local path to sync
:param built_packages: list of packages which has just been built
""" """
Rsync._check_output(*self.command, str(path), self.remote, exception=None, logger=self.logger) Rsync._check_output(*self.command, str(path), self.remote, exception=None, logger=self.logger)

5
src/ahriman/core/upload/s3.py
View File

@ -18,10 +18,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from pathlib import Path from pathlib import Path
from typing import Iterable
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload from ahriman.core.upload.upload import Upload
from ahriman.core.util import check_output from ahriman.core.util import check_output
from ahriman.models.package import Package
class S3(Upload): class S3(Upload):
@ -43,10 +45,11 @@ class S3(Upload):
self.bucket = configuration.get("s3", "bucket") self.bucket = configuration.get("s3", "bucket")
self.command = configuration.getlist("s3", "command") self.command = configuration.getlist("s3", "command")
def sync(self, path: Path) -> None: def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
""" """
sync data to remote server sync data to remote server
:param path: local path to sync :param path: local path to sync
:param built_packages: list of packages which has just been built
""" """
# TODO rewrite to boto, but it is bullshit # TODO rewrite to boto, but it is bullshit
S3._check_output(*self.command, str(path), self.bucket, exception=None, logger=self.logger) S3._check_output(*self.command, str(path), self.bucket, exception=None, logger=self.logger)

11
src/ahriman/core/upload/upload.py
View File

@ -22,10 +22,11 @@ from __future__ import annotations
import logging import logging
from pathlib import Path from pathlib import Path
from typing import Type from typing import Iterable, Type
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SyncFailed from ahriman.core.exceptions import SyncFailed
from ahriman.models.package import Package
from ahriman.models.upload_settings import UploadSettings from ahriman.models.upload_settings import UploadSettings
@ -65,19 +66,21 @@ class Upload:
return S3(architecture, configuration) return S3(architecture, configuration)
return cls(architecture, configuration) # should never happen return cls(architecture, configuration) # should never happen
def run(self, path: Path) -> None: def run(self, path: Path, built_packages: Iterable[Package]) -> None:
""" """
run remote sync run remote sync
:param path: local path to sync :param path: local path to sync
:param built_packages: list of packages which has just been built
""" """
try: try:
self.sync(path) self.sync(path, built_packages)
except Exception: except Exception:
self.logger.exception("remote sync failed") self.logger.exception("remote sync failed")
raise SyncFailed() raise SyncFailed()
def sync(self, path: Path) -> None: def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
""" """
sync data to remote server sync data to remote server
:param path: local path to sync :param path: local path to sync
:param built_packages: list of packages which has just been built
""" """

4
src/ahriman/core/util.py
View File

@ -22,7 +22,7 @@ import subprocess
from logging import Logger from logging import Logger
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional, Union
from ahriman.core.exceptions import InvalidOption from ahriman.core.exceptions import InvalidOption
@ -60,7 +60,7 @@ def package_like(filename: Path) -> bool:
return ".pkg." in name and not name.endswith(".sig") return ".pkg." in name and not name.endswith(".sig")
def pretty_datetime(timestamp: Optional[int]) -> str: def pretty_datetime(timestamp: Optional[Union[float, int]]) -> str:
""" """
convert datetime object to string convert datetime object to string
:param timestamp: datetime to convert :param timestamp: datetime to convert

5
src/ahriman/models/report_settings.py
View File

@ -28,11 +28,14 @@ from ahriman.core.exceptions import InvalidOption
class ReportSettings(Enum): class ReportSettings(Enum):
""" """
report targets enumeration report targets enumeration
:cvar Disabled: option which generates no report for testing purpose
:cvar HTML: html report generation :cvar HTML: html report generation
:cvar Email: email report generation
""" """
Disabled = auto() # for testing purpose Disabled = auto() # for testing purpose
HTML = auto() HTML = auto()
Email = auto()
@classmethod @classmethod
def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings: def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings:
@ -43,4 +46,6 @@ class ReportSettings(Enum):
""" """
if value.lower() in ("html",): if value.lower() in ("html",):
return cls.HTML return cls.HTML
if value.lower() in ("email",):
return cls.Email
raise InvalidOption(value) raise InvalidOption(value)

49
src/ahriman/models/smtp_ssl_settings.py Normal file
View File

@ -0,0 +1,49 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
from enum import Enum, auto
from typing import Type
class SmtpSSLSettings(Enum):
"""
SMTP SSL mode enumeration
:cvar Disabled: no SSL enabled
:cvar SSL: use SMTP_SSL instead of normal SMTP client
:cvar STARTTLS: use STARTTLS in normal SMTP client
"""
Disabled = auto()
SSL = auto()
STARTTLS = auto()
@classmethod
def from_option(cls: Type[SmtpSSLSettings], value: str) -> SmtpSSLSettings:
"""
construct value from configuration
:param value: configuration value
:return: parsed value
"""
if value.lower() in ("ssl", "ssl/tls"):
return cls.SSL
if value.lower() in ("starttls",):
return cls.STARTTLS
return cls.Disabled

1
src/ahriman/models/upload_settings.py
View File

@ -28,6 +28,7 @@ from ahriman.core.exceptions import InvalidOption
class UploadSettings(Enum): class UploadSettings(Enum):
""" """
remote synchronization targets enumeration remote synchronization targets enumeration
:cvar Disabled: no sync will be performed, required for testing purpose
:cvar Rsync: sync via rsync :cvar Rsync: sync via rsync
:cvar S3: sync to Amazon S3 :cvar S3: sync to Amazon S3
""" """

9
tests/ahriman/application/test_application.py
View File

@ -15,7 +15,7 @@ def test_finalize(application: Application, mocker: MockerFixture) -> None:
report_mock = mocker.patch("ahriman.application.application.Application.report") report_mock = mocker.patch("ahriman.application.application.Application.report")
sync_mock = mocker.patch("ahriman.application.application.Application.sync") sync_mock = mocker.patch("ahriman.application.application.Application.sync")
application._finalize() application._finalize([])
report_mock.assert_called_once() report_mock.assert_called_once()
sync_mock.assert_called_once() sync_mock.assert_called_once()
@ -218,7 +218,7 @@ def test_report(application: Application, mocker: MockerFixture) -> None:
must generate report must generate report
""" """
executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_report") executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_report")
application.report([]) application.report([], [])
executor_mock.assert_called_once() executor_mock.assert_called_once()
@ -279,7 +279,7 @@ def test_sync(application: Application, mocker: MockerFixture) -> None:
must sync to remote must sync to remote
""" """
executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_sync") executor_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_sync")
application.sync([]) application.sync([], [])
executor_mock.assert_called_once() executor_mock.assert_called_once()
@ -292,6 +292,7 @@ def test_update(application: Application, package_ahriman: Package, mocker: Mock
mocker.patch("ahriman.core.tree.Tree.load", return_value=tree) mocker.patch("ahriman.core.tree.Tree.load", return_value=tree)
mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[]) mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[])
mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman)
build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=paths) build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=paths)
update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update") update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update")
finalize_mock = mocker.patch("ahriman.application.application.Application._finalize") finalize_mock = mocker.patch("ahriman.application.application.Application._finalize")
@ -299,4 +300,4 @@ def test_update(application: Application, package_ahriman: Package, mocker: Mock
application.update([package_ahriman]) application.update([package_ahriman])
build_mock.assert_called_once() build_mock.assert_called_once()
update_mock.assert_has_calls([mock.call([]), mock.call(paths)]) update_mock.assert_has_calls([mock.call([]), mock.call(paths)])
finalize_mock.assert_has_calls([mock.call(), mock.call()]) finalize_mock.assert_has_calls([mock.call([]), mock.call([package_ahriman])])

105
tests/ahriman/core/report/test_email.py Normal file
View File

@ -0,0 +1,105 @@
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.report.email import Email
from ahriman.models.package import Package
def test_send(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must send an email with attachment
"""
smtp_mock = mocker.patch("smtplib.SMTP")
report = Email("x86_64", configuration)
report._send("a text", {"attachment.html": "an attachment"})
smtp_mock.return_value.starttls.assert_not_called()
smtp_mock.return_value.login.assert_not_called()
smtp_mock.return_value.sendmail.assert_called_once()
smtp_mock.return_value.quit.assert_called_once()
def test_send_auth(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must send an email with attachment with auth
"""
configuration.set("email", "user", "username")
configuration.set("email", "password", "password")
smtp_mock = mocker.patch("smtplib.SMTP")
report = Email("x86_64", configuration)
report._send("a text", {"attachment.html": "an attachment"})
smtp_mock.return_value.login.assert_called_once()
def test_send_auth_no_password(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must send an email with attachment without auth if no password supplied
"""
configuration.set("email", "user", "username")
smtp_mock = mocker.patch("smtplib.SMTP")
report = Email("x86_64", configuration)
report._send("a text", {"attachment.html": "an attachment"})
smtp_mock.return_value.login.assert_not_called()
def test_send_auth_no_user(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must send an email with attachment without auth if no user supplied
"""
configuration.set("email", "password", "password")
smtp_mock = mocker.patch("smtplib.SMTP")
report = Email("x86_64", configuration)
report._send("a text", {"attachment.html": "an attachment"})
smtp_mock.return_value.login.assert_not_called()
def test_send_ssl_tls(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must send an email with attachment with ssl/tls
"""
configuration.set("email", "ssl", "ssl")
smtp_mock = mocker.patch("smtplib.SMTP_SSL")
report = Email("x86_64", configuration)
report._send("a text", {"attachment.html": "an attachment"})
smtp_mock.return_value.starttls.assert_not_called()
smtp_mock.return_value.login.assert_not_called()
smtp_mock.return_value.sendmail.assert_called_once()
smtp_mock.return_value.quit.assert_called_once()
def test_send_starttls(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must send an email with attachment with starttls
"""
configuration.set("email", "ssl", "starttls")
smtp_mock = mocker.patch("smtplib.SMTP")
report = Email("x86_64", configuration)
report._send("a text", {"attachment.html": "an attachment"})
smtp_mock.return_value.starttls.assert_called_once()
def test_generate(configuration: Configuration, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must generate report
"""
send_mock = mocker.patch("ahriman.core.report.email.Email._send")
report = Email("x86_64", configuration)
report.generate([package_ahriman], [])
send_mock.assert_called_once()
def test_generate_with_built(configuration: Configuration, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must generate report with built packages
"""
send_mock = mocker.patch("ahriman.core.report.email.Email._send")
report = Email("x86_64", configuration)
report.generate([package_ahriman], [package_ahriman])
send_mock.assert_called_once()

2
tests/ahriman/core/report/test_html.py
View File

@ -12,5 +12,5 @@ def test_generate(configuration: Configuration, package_ahriman: Package, mocker
write_mock = mocker.patch("pathlib.Path.write_text") write_mock = mocker.patch("pathlib.Path.write_text")
report = HTML("x86_64", configuration) report = HTML("x86_64", configuration)
report.generate([package_ahriman]) report.generate([package_ahriman], [])
write_mock.assert_called_once() write_mock.assert_called_once()

19
tests/ahriman/core/report/test_jinja_tempalte.py Normal file
View File

@ -0,0 +1,19 @@
from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.models.package import Package
def test_generate(configuration: Configuration, package_ahriman: Package) -> None:
"""
must generate html report
"""
report = JinjaTemplate("html", configuration)
assert report.make_html([package_ahriman], extended_report=False)
def test_generate_extended(configuration: Configuration, package_ahriman: Package) -> None:
"""
must generate extended html report
"""
report = JinjaTemplate("html", configuration)
assert report.make_html([package_ahriman], extended_report=True)

15
tests/ahriman/core/report/test_report.py
View File

@ -15,7 +15,7 @@ def test_report_failure(configuration: Configuration, mocker: MockerFixture) ->
""" """
mocker.patch("ahriman.core.report.html.HTML.generate", side_effect=Exception()) mocker.patch("ahriman.core.report.html.HTML.generate", side_effect=Exception())
with pytest.raises(ReportFailed): with pytest.raises(ReportFailed):
Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path")) Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path"), [])
def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> None: def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> None:
@ -24,7 +24,16 @@ def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> No
""" """
mocker.patch("ahriman.models.report_settings.ReportSettings.from_option", return_value=ReportSettings.Disabled) mocker.patch("ahriman.models.report_settings.ReportSettings.from_option", return_value=ReportSettings.Disabled)
report_mock = mocker.patch("ahriman.core.report.report.Report.generate") report_mock = mocker.patch("ahriman.core.report.report.Report.generate")
Report.load("x86_64", configuration, ReportSettings.Disabled.name).run(Path("path")) Report.load("x86_64", configuration, ReportSettings.Disabled.name).run(Path("path"), [])
report_mock.assert_called_once()
def test_report_email(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must generate email report
"""
report_mock = mocker.patch("ahriman.core.report.email.Email.generate")
Report.load("x86_64", configuration, ReportSettings.Email.name).run(Path("path"), [])
report_mock.assert_called_once() report_mock.assert_called_once()
@ -33,5 +42,5 @@ def test_report_html(configuration: Configuration, mocker: MockerFixture) -> Non
must generate html report must generate html report
""" """
report_mock = mocker.patch("ahriman.core.report.html.HTML.generate") report_mock = mocker.patch("ahriman.core.report.html.HTML.generate")
Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path")) Report.load("x86_64", configuration, ReportSettings.HTML.name).run(Path("path"), [])
report_mock.assert_called_once() report_mock.assert_called_once()

8
tests/ahriman/core/repository/test_executor.py
View File

@ -133,7 +133,7 @@ def test_process_report(executor: Executor, package_ahriman: Package, mocker: Mo
mocker.patch("ahriman.core.report.report.Report.load", return_value=Report("x86_64", executor.configuration)) mocker.patch("ahriman.core.report.report.Report.load", return_value=Report("x86_64", executor.configuration))
report_mock = mocker.patch("ahriman.core.report.report.Report.run") report_mock = mocker.patch("ahriman.core.report.report.Report.run")
executor.process_report(["dummy"]) executor.process_report(["dummy"], [])
report_mock.assert_called_once() report_mock.assert_called_once()
@ -143,7 +143,7 @@ def test_process_report_auto(executor: Executor, mocker: MockerFixture) -> None:
""" """
configuration_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist") configuration_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist")
executor.process_report(None) executor.process_report(None, [])
configuration_getlist_mock.assert_called_once() configuration_getlist_mock.assert_called_once()
@ -154,7 +154,7 @@ def test_process_upload(executor: Executor, mocker: MockerFixture) -> None:
mocker.patch("ahriman.core.upload.upload.Upload.load", return_value=Upload("x86_64", executor.configuration)) mocker.patch("ahriman.core.upload.upload.Upload.load", return_value=Upload("x86_64", executor.configuration))
upload_mock = mocker.patch("ahriman.core.upload.upload.Upload.run") upload_mock = mocker.patch("ahriman.core.upload.upload.Upload.run")
executor.process_sync(["dummy"]) executor.process_sync(["dummy"], [])
upload_mock.assert_called_once() upload_mock.assert_called_once()
@ -164,7 +164,7 @@ def test_process_upload_auto(executor: Executor, mocker: MockerFixture) -> None:
""" """
configuration_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist") configuration_getlist_mock = mocker.patch("ahriman.core.configuration.Configuration.getlist")
executor.process_sync(None) executor.process_sync(None, [])
configuration_getlist_mock.assert_called_once() configuration_getlist_mock.assert_called_once()

2
tests/ahriman/core/upload/test_rsync.py
View File

@ -12,5 +12,5 @@ def test_sync(configuration: Configuration, mocker: MockerFixture) -> None:
check_output_mock = mocker.patch("ahriman.core.upload.rsync.Rsync._check_output") check_output_mock = mocker.patch("ahriman.core.upload.rsync.Rsync._check_output")
upload = Rsync("x86_64", configuration) upload = Rsync("x86_64", configuration)
upload.sync(Path("path")) upload.sync(Path("path"), [])
check_output_mock.assert_called_once() check_output_mock.assert_called_once()

2
tests/ahriman/core/upload/test_s3.py
View File

@ -12,5 +12,5 @@ def test_sync(configuration: Configuration, mocker: MockerFixture) -> None:
check_output_mock = mocker.patch("ahriman.core.upload.s3.S3._check_output") check_output_mock = mocker.patch("ahriman.core.upload.s3.S3._check_output")
upload = S3("x86_64", configuration) upload = S3("x86_64", configuration)
upload.sync(Path("path")) upload.sync(Path("path"), [])
check_output_mock.assert_called_once() check_output_mock.assert_called_once()

8
tests/ahriman/core/upload/test_upload.py
View File

@ -15,7 +15,7 @@ def test_upload_failure(configuration: Configuration, mocker: MockerFixture) ->
""" """
mocker.patch("ahriman.core.upload.rsync.Rsync.sync", side_effect=Exception()) mocker.patch("ahriman.core.upload.rsync.Rsync.sync", side_effect=Exception())
with pytest.raises(SyncFailed): with pytest.raises(SyncFailed):
Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path")) Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path"), [])
def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> None: def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> None:
@ -24,7 +24,7 @@ def test_report_dummy(configuration: Configuration, mocker: MockerFixture) -> No
""" """
mocker.patch("ahriman.models.upload_settings.UploadSettings.from_option", return_value=UploadSettings.Disabled) mocker.patch("ahriman.models.upload_settings.UploadSettings.from_option", return_value=UploadSettings.Disabled)
upload_mock = mocker.patch("ahriman.core.upload.upload.Upload.sync") upload_mock = mocker.patch("ahriman.core.upload.upload.Upload.sync")
Upload.load("x86_64", configuration, UploadSettings.Disabled.name).run(Path("path")) Upload.load("x86_64", configuration, UploadSettings.Disabled.name).run(Path("path"), [])
upload_mock.assert_called_once() upload_mock.assert_called_once()
@ -33,7 +33,7 @@ def test_upload_rsync(configuration: Configuration, mocker: MockerFixture) -> No
must upload via rsync must upload via rsync
""" """
upload_mock = mocker.patch("ahriman.core.upload.rsync.Rsync.sync") upload_mock = mocker.patch("ahriman.core.upload.rsync.Rsync.sync")
Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path")) Upload.load("x86_64", configuration, UploadSettings.Rsync.name).run(Path("path"), [])
upload_mock.assert_called_once() upload_mock.assert_called_once()
@ -42,5 +42,5 @@ def test_upload_s3(configuration: Configuration, mocker: MockerFixture) -> None:
must upload via s3 must upload via s3
""" """
upload_mock = mocker.patch("ahriman.core.upload.s3.S3.sync") upload_mock = mocker.patch("ahriman.core.upload.s3.S3.sync")
Upload.load("x86_64", configuration, UploadSettings.S3.name).run(Path("path")) Upload.load("x86_64", configuration, UploadSettings.S3.name).run(Path("path"), [])
upload_mock.assert_called_once() upload_mock.assert_called_once()

3
tests/ahriman/models/test_report_settings.py
View File

@ -18,3 +18,6 @@ def test_from_option_valid() -> None:
""" """
assert ReportSettings.from_option("html") == ReportSettings.HTML assert ReportSettings.from_option("html") == ReportSettings.HTML
assert ReportSettings.from_option("HTML") == ReportSettings.HTML assert ReportSettings.from_option("HTML") == ReportSettings.HTML
assert ReportSettings.from_option("email") == ReportSettings.Email
assert ReportSettings.from_option("EmAil") == ReportSettings.Email

21
tests/ahriman/models/test_smtp_settings.py Normal file
View File

@ -0,0 +1,21 @@
from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
def test_from_option_invalid() -> None:
"""
must return disabled value on invalid option
"""
assert SmtpSSLSettings.from_option("invalid") == SmtpSSLSettings.Disabled
def test_from_option_valid() -> None:
"""
must return value from valid options
"""
assert SmtpSSLSettings.from_option("ssl") == SmtpSSLSettings.SSL
assert SmtpSSLSettings.from_option("SSL") == SmtpSSLSettings.SSL
assert SmtpSSLSettings.from_option("ssl/tls") == SmtpSSLSettings.SSL
assert SmtpSSLSettings.from_option("SSL/TLS") == SmtpSSLSettings.SSL
assert SmtpSSLSettings.from_option("starttls") == SmtpSSLSettings.STARTTLS
assert SmtpSSLSettings.from_option("STARTTLS") == SmtpSSLSettings.STARTTLS

8
tests/testresources/core/ahriman.ini
View File

@ -25,6 +25,14 @@ target =
[report] [report]
target = target =
[email]
host = 0.0.0.0
link_path =
port = 587
receivers = mail@example.com
sender = mail@example.com
template_path = ../web/templates/repo-index.jinja2
[html] [html]
path = path =
homepage = homepage =