arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-15 06:55:48 +00:00
Code Issues Projects Releases Wiki Activity

Email report (#11)

Browse Source 
* Demo email report implementation

* improved ssl mode

* correct default option spelling and more fields to be hidden for not
extended reports
This commit is contained in:
Evgenii Alekseev
2021-04-06 05:45:17 +03:00
committed by GitHub
parent ce0c07cbd9
commit c6ccf53768
32 changed files with 560 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/ahriman/application/application.py
View File

@ -51,12 +51,12 @@ class Application:
self.architecture = architecture
self.repository = Repository(architecture, configuration)
def _finalize(self) -> None:
def _finalize(self, built_packages: Iterable[Package]) -> None:
"""
generate report and sync to remote server
"""
self.report([])
self.sync([])
self.report([], built_packages)
self.sync([], built_packages)
def _known_packages(self) -> Set[str]:
"""
@ -160,15 +160,16 @@ class Application:
:param names: list of packages (either base or name) to remove
"""
self.repository.process_remove(names)
self._finalize()
self._finalize([])
def report(self, target: Iterable[str]) -> None:
def report(self, target: Iterable[str], built_packages: Iterable[Package]) -> None:
"""
generate report
:param target: list of targets to run (e.g. html)
:param built_packages: list of packages which has just been built
"""
targets = target or None
self.repository.process_report(targets)
self.repository.process_report(targets, built_packages)
def sign(self, packages: Iterable[str]) -> None:
"""
@ -191,15 +192,16 @@ class Application:
self.update([])
# sign repository database if set
self.repository.sign.sign_repository(self.repository.repo.repo_path)
self._finalize()
self._finalize([])
def sync(self, target: Iterable[str]) -> None:
def sync(self, target: Iterable[str], built_packages: Iterable[Package]) -> None:
"""
sync to remote server
:param target: list of targets to run (e.g. s3)
:param built_packages: list of packages which has just been built
"""
targets = target or None
self.repository.process_sync(targets)
self.repository.process_sync(targets, built_packages)
def update(self, updates: Iterable[Package]) -> None:
"""
@ -207,8 +209,9 @@ class Application:
:param updates: list of packages to update
"""
def process_update(paths: Iterable[Path]) -> None:
updated = [Package.load(path, self.repository.pacman, self.repository.aur_url) for path in paths]
self.repository.process_update(paths)
self._finalize()
self._finalize(updated)
# process built packages
packages = self.repository.packages_built()

2
src/ahriman/application/handlers/report.py
View File

@ -39,4 +39,4 @@ class Report(Handler):
:param architecture: repository architecture
:param configuration: configuration instance
"""
Application(architecture, configuration).report(args.target)
Application(architecture, configuration).report(args.target, [])

2
src/ahriman/application/handlers/sync.py
View File

@ -39,4 +39,4 @@ class Sync(Handler):
:param architecture: repository architecture
:param configuration: configuration instance
"""
Application(architecture, configuration).sync(args.target)
Application(architecture, configuration).sync(args.target, [])

101
src/ahriman/core/report/email.py Normal file
View File

@ -0,0 +1,101 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import Dict, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.core.util import pretty_datetime
from ahriman.models.package import Package
from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
class Email(Report, JinjaTemplate):
"""
email report generator
:ivar host: SMTP host to connect
:ivar password: password to authenticate via SMTP
:ivar port: SMTP port to connect
:ivar receivers: list of receivers emails
:ivar sender: sender email address
:ivar ssl: SSL mode for SMTP connection
:ivar user: username to authenticate via SMTP
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
default constructor
:param architecture: repository architecture
:param configuration: configuration instance
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, "email", configuration)
# base smtp settings
self.host = configuration.get("email", "host")
self.password = configuration.get("email", "password", fallback=None)
self.port = configuration.getint("email", "port")
self.receivers = configuration.getlist("email", "receivers")
self.sender = configuration.get("email", "sender")
self.ssl = SmtpSSLSettings.from_option(configuration.get("email", "ssl", fallback="disabled"))
self.user = configuration.get("email", "user", fallback=None)
def _send(self, text: str, attachment: Dict[str, str]) -> None:
"""
send email callback
:param text: email body text
:param attachment: map of attachment filename to attachment content
"""
message = MIMEMultipart()
message["From"] = self.sender
message["To"] = ", ".join(self.receivers)
message["Subject"] = f"{self.name} build report at {pretty_datetime(datetime.datetime.utcnow().timestamp())}"
message.attach(MIMEText(text, "html"))
for filename, content in attachment.items():
attach = MIMEText(content, "html")
attach.add_header("Content-Disposition", "attachment", filename=filename)
message.attach(attach)
if self.ssl != SmtpSSLSettings.SSL:
session = smtplib.SMTP(self.host, self.port)
if self.ssl == SmtpSSLSettings.STARTTLS:
session.starttls()
else:
session = smtplib.SMTP_SSL(self.host, self.port)
if self.user is not None and self.password is not None:
session.login(self.user, self.password)
session.sendmail(self.sender, self.receivers, message.as_string())
session.quit()
def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
"""
generate report for the specified packages
:param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
"""
text = self.make_html(built_packages, False)
attachments = {"index.html": self.make_html(packages, True)}
self._send(text, attachments)

85
src/ahriman/core/report/html.py
View File

@ -17,51 +17,18 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import jinja2
from typing import Callable, Dict, Iterable
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.core.sign.gpg import GPG
from ahriman.core.util import pretty_datetime, pretty_size
from ahriman.models.package import Package
from ahriman.models.sign_settings import SignSettings
class HTML(Report):
class HTML(Report, JinjaTemplate):
"""
html report generator
It uses jinja2 templates for report generation, the following variables are allowed:
homepage - link to homepage, string, optional
link_path - prefix fo packages to download, string, required
has_package_signed - True in case if package sign enabled, False otherwise, required
has_repo_signed - True in case if repository database sign enabled, False otherwise, required
packages - sorted list of packages properties, required
* architecture, string
* archive_size, pretty printed size, string
* build_date, pretty printed datetime, string
* depends, sorted list of strings
* description, string
* filename, string,
* groups, sorted list of strings
* installed_size, pretty printed datetime, string
* licenses, sorted list of strings
* name, string
* url, string
* version, string
pgp_key - default PGP key ID, string, optional
repository - repository name, string, required
:ivar homepage: homepage link if any (for footer)
:ivar link_path: prefix fo packages to download
:ivar name: repository name
:ivar default_pgp_key: default PGP key
:ivar report_path: output path to html report
:ivar sign_targets: targets to sign enabled in configuration
:ivar template_path: path to directory with jinja templates
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
@ -71,51 +38,15 @@ class HTML(Report):
:param configuration: configuration instance
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, "html", configuration)
self.report_path = configuration.getpath("html", "path")
self.link_path = configuration.get("html", "link_path")
self.template_path = configuration.getpath("html", "template_path")
# base template vars
self.homepage = configuration.get("html", "homepage", fallback=None)
self.name = configuration.get("repository", "name")
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
def generate(self, packages: Iterable[Package]) -> None:
def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
"""
generate report for the specified packages
:param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
"""
# idea comes from https://stackoverflow.com/a/38642558
loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent)
environment = jinja2.Environment(loader=loader)
template = environment.get_template(self.template_path.name)
content = [
{
"architecture": properties.architecture or "",
"archive_size": pretty_size(properties.archive_size),
"build_date": pretty_datetime(properties.build_date),
"depends": properties.depends,
"description": properties.description or "",
"filename": properties.filename,
"groups": properties.groups,
"installed_size": pretty_size(properties.installed_size),
"licenses": properties.licenses,
"name": package,
"url": properties.url or "",
"version": base.version
} for base in packages for package, properties in base.packages.items()
]
comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
html = template.render(
homepage=self.homepage,
link_path=self.link_path,
has_package_signed=SignSettings.Packages in self.sign_targets,
has_repo_signed=SignSettings.Repository in self.sign_targets,
packages=sorted(content, key=comparator),
pgp_key=self.default_pgp_key,
repository=self.name)
html = self.make_html(packages, True)
self.report_path.write_text(html)

117
src/ahriman/core/report/jinja_template.py Normal file
View File

@ -0,0 +1,117 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import jinja2
from typing import Callable, Dict, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.util import pretty_datetime, pretty_size
from ahriman.models.package import Package
from ahriman.models.sign_settings import SignSettings
class JinjaTemplate:
"""
jinja based report generator
It uses jinja2 templates for report generation, the following variables are allowed:
homepage - link to homepage, string, optional
link_path - prefix fo packages to download, string, required
has_package_signed - True in case if package sign enabled, False otherwise, required
has_repo_signed - True in case if repository database sign enabled, False otherwise, required
packages - sorted list of packages properties, required
* architecture, string
* archive_size, pretty printed size, string
* build_date, pretty printed datetime, string
* depends, sorted list of strings
* description, string
* filename, string,
* groups, sorted list of strings
* installed_size, pretty printed datetime, string
* licenses, sorted list of strings
* name, string
* url, string
* version, string
pgp_key - default PGP key ID, string, optional
repository - repository name, string, required
:ivar homepage: homepage link if any (for footer)
:ivar link_path: prefix fo packages to download
:ivar name: repository name
:ivar default_pgp_key: default PGP key
:ivar sign_targets: targets to sign enabled in configuration
:ivar template_path: path to directory with jinja templates
"""
def __init__(self, section: str, configuration: Configuration) -> None:
"""
default constructor
:param section: settings section name
:param configuration: configuration instance
"""
self.link_path = configuration.get(section, "link_path")
self.template_path = configuration.getpath(section, "template_path")
# base template vars
self.homepage = configuration.get(section, "homepage", fallback=None)
self.name = configuration.get("repository", "name")
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
def make_html(self, packages: Iterable[Package], extended_report: bool) -> str:
"""
generate report for the specified packages
:param packages: list of packages to generate report
:param extended_report: include additional blocks to the report
"""
# idea comes from https://stackoverflow.com/a/38642558
loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent)
environment = jinja2.Environment(loader=loader)
template = environment.get_template(self.template_path.name)
content = [
{
"architecture": properties.architecture or "",
"archive_size": pretty_size(properties.archive_size),
"build_date": pretty_datetime(properties.build_date),
"depends": properties.depends,
"description": properties.description or "",
"filename": properties.filename,
"groups": properties.groups,
"installed_size": pretty_size(properties.installed_size),
"licenses": properties.licenses,
"name": package,
"url": properties.url or "",
"version": base.version
} for base in packages for package, properties in base.packages.items()
]
comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
return template.render(
extended_report=extended_report,
homepage=self.homepage,
link_path=self.link_path,
has_package_signed=SignSettings.Packages in self.sign_targets,
has_repo_signed=SignSettings.Repository in self.sign_targets,
packages=sorted(content, key=comparator),
pgp_key=self.default_pgp_key,
repository=self.name)

11
src/ahriman/core/report/report.py
View File

@ -60,21 +60,26 @@ class Report:
if provider == ReportSettings.HTML:
from ahriman.core.report.html import HTML
return HTML(architecture, configuration)
if provider == ReportSettings.Email:
from ahriman.core.report.email import Email
return Email(architecture, configuration)
return cls(architecture, configuration) # should never happen
def generate(self, packages: Iterable[Package]) -> None:
def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
"""
generate report for the specified packages
:param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
"""
def run(self, packages: Iterable[Package]) -> None:
def run(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
"""
run report generation
:param packages: list of packages to generate report
:param built_packages: list of packages which has just been built
"""
try:
self.generate(packages)
self.generate(packages, built_packages)
except Exception:
self.logger.exception("report generation failed")
raise ReportFailed()

10
src/ahriman/core/repository/executor.py
View File

@ -100,27 +100,29 @@ class Executor(Cleaner):
return self.repo.repo_path
def process_report(self, targets: Optional[Iterable[str]]) -> None:
def process_report(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None:
"""
generate reports
:param targets: list of targets to generate reports. Configuration option will be used if it is not set
:param built_packages: list of packages which has just been built
"""
if targets is None:
targets = self.configuration.getlist("report", "target")
for target in targets:
runner = Report.load(self.architecture, self.configuration, target)
runner.run(self.packages())
runner.run(self.packages(), built_packages)
def process_sync(self, targets: Optional[Iterable[str]]) -> None:
def process_sync(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None:
"""
process synchronization to remote servers
:param targets: list of targets to sync. Configuration option will be used if it is not set
:param built_packages: list of packages which has just been built
"""
if targets is None:
targets = self.configuration.getlist("upload", "target")
for target in targets:
runner = Upload.load(self.architecture, self.configuration, target)
runner.run(self.paths.repository)
runner.run(self.paths.repository, built_packages)
def process_update(self, packages: Iterable[Path]) -> Path:
"""

4
src/ahriman/core/repository/repository.py
View File

@ -37,9 +37,7 @@ class Repository(Executor, UpdateHandler):
:return: list of packages properties
"""
result: Dict[str, Package] = {}
for full_path in self.paths.repository.iterdir():
if not package_like(full_path):
continue
for full_path in filter(package_like, self.paths.repository.iterdir()):
try:
local = Package.load(full_path, self.pacman, self.aur_url)
result.setdefault(local.base, local).packages.update(local.packages)

5
src/ahriman/core/upload/rsync.py
View File

@ -18,10 +18,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pathlib import Path
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import check_output
from ahriman.models.package import Package
class Rsync(Upload):
@ -43,9 +45,10 @@ class Rsync(Upload):
self.command = configuration.getlist("rsync", "command")
self.remote = configuration.get("rsync", "remote")
def sync(self, path: Path) -> None:
def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
"""
sync data to remote server
:param path: local path to sync
:param built_packages: list of packages which has just been built
"""
Rsync._check_output(*self.command, str(path), self.remote, exception=None, logger=self.logger)

5
src/ahriman/core/upload/s3.py
View File

@ -18,10 +18,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pathlib import Path
from typing import Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import check_output
from ahriman.models.package import Package
class S3(Upload):
@ -43,10 +45,11 @@ class S3(Upload):
self.bucket = configuration.get("s3", "bucket")
self.command = configuration.getlist("s3", "command")
def sync(self, path: Path) -> None:
def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
"""
sync data to remote server
:param path: local path to sync
:param built_packages: list of packages which has just been built
"""
# TODO rewrite to boto, but it is bullshit
S3._check_output(*self.command, str(path), self.bucket, exception=None, logger=self.logger)

11
src/ahriman/core/upload/upload.py
View File

@ -22,10 +22,11 @@ from __future__ import annotations
import logging
from pathlib import Path
from typing import Type
from typing import Iterable, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SyncFailed
from ahriman.models.package import Package
from ahriman.models.upload_settings import UploadSettings
@ -65,19 +66,21 @@ class Upload:
return S3(architecture, configuration)
return cls(architecture, configuration) # should never happen
def run(self, path: Path) -> None:
def run(self, path: Path, built_packages: Iterable[Package]) -> None:
"""
run remote sync
:param path: local path to sync
:param built_packages: list of packages which has just been built
"""
try:
self.sync(path)
self.sync(path, built_packages)
except Exception:
self.logger.exception("remote sync failed")
raise SyncFailed()
def sync(self, path: Path) -> None:
def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
"""
sync data to remote server
:param path: local path to sync
:param built_packages: list of packages which has just been built
"""

4
src/ahriman/core/util.py
View File

@ -22,7 +22,7 @@ import subprocess
from logging import Logger
from pathlib import Path
from typing import Optional
from typing import Optional, Union
from ahriman.core.exceptions import InvalidOption
@ -60,7 +60,7 @@ def package_like(filename: Path) -> bool:
return ".pkg." in name and not name.endswith(".sig")
def pretty_datetime(timestamp: Optional[int]) -> str:
def pretty_datetime(timestamp: Optional[Union[float, int]]) -> str:
"""
convert datetime object to string
:param timestamp: datetime to convert

5
src/ahriman/models/report_settings.py
View File

@ -28,11 +28,14 @@ from ahriman.core.exceptions import InvalidOption
class ReportSettings(Enum):
"""
report targets enumeration
:cvar Disabled: option which generates no report for testing purpose
:cvar HTML: html report generation
:cvar Email: email report generation
"""
Disabled = auto() # for testing purpose
HTML = auto()
Email = auto()
@classmethod
def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings:
@ -43,4 +46,6 @@ class ReportSettings(Enum):
"""
if value.lower() in ("html",):
return cls.HTML
if value.lower() in ("email",):
return cls.Email
raise InvalidOption(value)

49
src/ahriman/models/smtp_ssl_settings.py Normal file
View File

@ -0,0 +1,49 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
from enum import Enum, auto
from typing import Type
class SmtpSSLSettings(Enum):
"""
SMTP SSL mode enumeration
:cvar Disabled: no SSL enabled
:cvar SSL: use SMTP_SSL instead of normal SMTP client
:cvar STARTTLS: use STARTTLS in normal SMTP client
"""
Disabled = auto()
SSL = auto()
STARTTLS = auto()
@classmethod
def from_option(cls: Type[SmtpSSLSettings], value: str) -> SmtpSSLSettings:
"""
construct value from configuration
:param value: configuration value
:return: parsed value
"""
if value.lower() in ("ssl", "ssl/tls"):
return cls.SSL
if value.lower() in ("starttls",):
return cls.STARTTLS
return cls.Disabled

1
src/ahriman/models/upload_settings.py
View File

@ -28,6 +28,7 @@ from ahriman.core.exceptions import InvalidOption
class UploadSettings(Enum):
"""
remote synchronization targets enumeration
:cvar Disabled: no sync will be performed, required for testing purpose
:cvar Rsync: sync via rsync
:cvar S3: sync to Amazon S3
"""