Extended package status page (#76)

* implement log storage at backend
* handle process id during removal. During one process we can write logs from different packages in different times (e.g. check and update later) and we would like to store all logs belong to the same process
* set package context in main functions
* implement logs support in interface
* filter out logs posting http logs
* add timestamp to log records
* hide getting logs under reporter permission

List of breaking changes:

* `ahriman.core.lazy_logging.LazyLogging` has been renamed to `ahriman.core.log.LazyLogging`
* `ahriman.core.configuration.Configuration.from_path` does not have `quiet` attribute now
* `ahriman.core.configuration.Configuration` class does not have `load_logging` method now
* `ahriman.core.status.client.Client.load` requires `report` argument now
This commit is contained in:
Evgenii Alekseev 2022-11-22 02:58:22 +03:00 committed by GitHub
parent 8a6854c867
commit 137d62e2f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
90 changed files with 1650 additions and 360 deletions

View File

@ -61,7 +61,7 @@ Again, the most checks can be performed by `make check` command, though some add
* The file size mentioned above must be applicable in general. In case of big classes consider splitting them into traits. Note, however, that `pylint` includes comments and docstrings into counter, thus you need to check file size by other tools. * The file size mentioned above must be applicable in general. In case of big classes consider splitting them into traits. Note, however, that `pylint` includes comments and docstrings into counter, thus you need to check file size by other tools.
* No global variable is allowed outside of `ahriman.version` module. * No global variable is allowed outside of `ahriman.version` module.
* Single quotes are not allowed. The reason behind this restriction is the fact that docstrings must be written by using double quotes only, and we would like to make style consistent. * Single quotes are not allowed. The reason behind this restriction is the fact that docstrings must be written by using double quotes only, and we would like to make style consistent.
* If your class writes anything to log, the `ahriman.core.lazy_logging.LazyLogging` trait must be used. * If your class writes anything to log, the `ahriman.core.log.LazyLogging` trait must be used.
### Other checks ### Other checks

View File

@ -36,6 +36,14 @@ ahriman.core.database.migrations.m003\_patch\_variables module
:no-undoc-members: :no-undoc-members:
:show-inheritance: :show-inheritance:
ahriman.core.database.migrations.m004\_logs module
--------------------------------------------------
.. automodule:: ahriman.core.database.migrations.m004_logs
:members:
:no-undoc-members:
:show-inheritance:
Module contents Module contents
--------------- ---------------

View File

@ -20,6 +20,14 @@ ahriman.core.database.operations.build\_operations module
:no-undoc-members: :no-undoc-members:
:show-inheritance: :show-inheritance:
ahriman.core.database.operations.logs\_operations module
--------------------------------------------------------
.. automodule:: ahriman.core.database.operations.logs_operations
:members:
:no-undoc-members:
:show-inheritance:
ahriman.core.database.operations.operations module ahriman.core.database.operations.operations module
-------------------------------------------------- --------------------------------------------------

37
docs/ahriman.core.log.rst Normal file
View File

@ -0,0 +1,37 @@
ahriman.core.log package
========================
Submodules
----------
ahriman.core.log.http\_log\_handler module
------------------------------------------
.. automodule:: ahriman.core.log.http_log_handler
:members:
:no-undoc-members:
:show-inheritance:
ahriman.core.log.lazy\_logging module
-------------------------------------
.. automodule:: ahriman.core.log.lazy_logging
:members:
:no-undoc-members:
:show-inheritance:
ahriman.core.log.log module
---------------------------
.. automodule:: ahriman.core.log.log
:members:
:no-undoc-members:
:show-inheritance:
Module contents
---------------
.. automodule:: ahriman.core.log
:members:
:no-undoc-members:
:show-inheritance:

View File

@ -13,6 +13,7 @@ Subpackages
ahriman.core.database ahriman.core.database
ahriman.core.formatters ahriman.core.formatters
ahriman.core.gitremote ahriman.core.gitremote
ahriman.core.log
ahriman.core.report ahriman.core.report
ahriman.core.repository ahriman.core.repository
ahriman.core.sign ahriman.core.sign
@ -39,14 +40,6 @@ ahriman.core.exceptions module
:no-undoc-members: :no-undoc-members:
:show-inheritance: :show-inheritance:
ahriman.core.lazy\_logging module
---------------------------------
.. automodule:: ahriman.core.lazy_logging
:members:
:no-undoc-members:
:show-inheritance:
ahriman.core.spawn module ahriman.core.spawn module
------------------------- -------------------------

View File

@ -52,6 +52,14 @@ ahriman.models.internal\_status module
:no-undoc-members: :no-undoc-members:
:show-inheritance: :show-inheritance:
ahriman.models.log\_record\_id module
-------------------------------------
.. automodule:: ahriman.models.log_record_id
:members:
:no-undoc-members:
:show-inheritance:
ahriman.models.migration module ahriman.models.migration module
------------------------------- -------------------------------

View File

@ -4,6 +4,14 @@ ahriman.web.views.status package
Submodules Submodules
---------- ----------
ahriman.web.views.status.logs module
------------------------------------
.. automodule:: ahriman.web.views.status.logs
:members:
:no-undoc-members:
:show-inheritance:
ahriman.web.views.status.package module ahriman.web.views.status.package module
--------------------------------------- ---------------------------------------

View File

@ -40,7 +40,6 @@
</div> </div>
<table id="packages" class="table table-striped table-hover" <table id="packages" class="table table-striped table-hover"
data-click-to-select="true"
data-export-options='{"fileName": "packages"}' data-export-options='{"fileName": "packages"}'
data-page-list="[10, 25, 50, 100, all]" data-page-list="[10, 25, 50, 100, all]"
data-page-size="10" data-page-size="10"
@ -76,14 +75,14 @@
<div class="container"> <div class="container">
<footer class="d-flex flex-wrap justify-content-between align-items-center border-top"> <footer class="d-flex flex-wrap justify-content-between align-items-center border-top">
<ul class="nav"> <ul class="nav">
<li><a class="nav-link" href="https://github.com/arcan1s/ahriman" title="sources">ahriman</a></li> <li><a class="nav-link" href="https://github.com/arcan1s/ahriman" title="sources"><i class="bi bi-github"></i> ahriman</a></li>
<li><a class="nav-link" href="https://github.com/arcan1s/ahriman/releases" title="releases list">releases</a></li> <li><a class="nav-link" href="https://github.com/arcan1s/ahriman/releases" title="releases list">releases</a></li>
<li><a class="nav-link" href="https://github.com/arcan1s/ahriman/issues" title="issues tracker">report a bug</a></li> <li><a class="nav-link" href="https://github.com/arcan1s/ahriman/issues" title="issues tracker">report a bug</a></li>
</ul> </ul>
{% if index_url is not none %} {% if index_url is not none %}
<ul class="nav"> <ul class="nav">
<li><a class="nav-link" href="{{ index_url }}" title="repo index">repo index</a></li> <li><a class="nav-link" href="{{ index_url }}" title="repo index"><i class="bi bi-house"></i> repo index</a></li>
</ul> </ul>
{% endif %} {% endif %}
@ -92,7 +91,7 @@
{{ auth.control|safe }} {{ auth.control|safe }}
{% else %} {% else %}
<form action="/api/v1/logout" method="post"> <form action="/api/v1/logout" method="post">
<button class="btn btn-link" style="text-decoration: none">logout ({{ auth.username }})</button> <button class="btn btn-link" style="text-decoration: none"><i class="bi bi-box-arrow-right"></i> logout ({{ auth.username }})</button>
</form> </form>
{% endif %} {% endif %}
{% endif %} {% endif %}
@ -110,6 +109,8 @@
{% include "build-status/package-add-modal.jinja2" %} {% include "build-status/package-add-modal.jinja2" %}
{% include "build-status/package-info-modal.jinja2" %}
{% include "build-status/table.jinja2" %} {% include "build-status/table.jinja2" %}
</body> </body>

View File

@ -1,16 +1,16 @@
<div id="failed-form" tabindex="-1" role="dialog" class="modal fade"> <div id="failed-form" tabindex="-1" role="dialog" class="modal fade">
<div class="modal-dialog" role="document"> <div class="modal-dialog" role="document">
<div class="modal-content"> <div class="modal-content">
<div class="modal-header bg-danger"> <div class="modal-header bg-danger text-white">
<h4 class="modal-title">failed</h4> <h4 id="error-title" class="modal-title"></h4>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button> <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
</div> </div>
<div class="modal-body"> <div class="modal-body">
<p>Packages update has failed.</p> <p id="error-description"></p>
<p id="error-details"></p> <p id="error-details"></p>
</div> </div>
<div class="modal-footer"> <div class="modal-footer">
<button type="button" class="btn btn-primary" data-bs-dismiss="modal">close</button> <button type="button" class="btn btn-primary" data-bs-dismiss="modal"><i class="bi bi-x"></i> close</button>
</div> </div>
</div> </div>
</div> </div>
@ -18,10 +18,14 @@
<script> <script>
const failedForm = $("#failed-form"); const failedForm = $("#failed-form");
const errorDescription = $("#error-description");
const errorDetails = $("#error-details"); const errorDetails = $("#error-details");
const errorTitle = $("#error-title");
failedForm.on("hidden.bs.modal", () => { reload(); }); failedForm.on("hidden.bs.modal", () => { reload(); });
function showFailure(details) { function showFailure(title, description, details) {
errorTitle.text(title);
errorDescription.text(description);
errorDetails.text(details); errorDetails.text(details);
failedForm.modal("show"); failedForm.modal("show");
} }

View File

@ -3,7 +3,7 @@
<div class="modal-content"> <div class="modal-content">
<form action="/api/v1/login" method="post"> <form action="/api/v1/login" method="post">
<div class="modal-header"> <div class="modal-header">
<h4 class="modal-title">login</h4> <h4 class="modal-title">Login</h4>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button> <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
</div> </div>
<div class="modal-body"> <div class="modal-body">
@ -21,7 +21,7 @@
</div> </div>
</div> </div>
<div class="modal-footer"> <div class="modal-footer">
<button class="btn btn-primary">login</button> <button class="btn btn-primary"><i class="bi bi-person"></i> login</button>
</div> </div>
</form> </form>
</div> </div>

View File

@ -2,7 +2,7 @@
<div class="modal-dialog" role="document"> <div class="modal-dialog" role="document">
<div class="modal-content"> <div class="modal-content">
<div class="modal-header"> <div class="modal-header">
<h4 class="modal-title">add new packages</h4> <h4 class="modal-title">Add new packages</h4>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button> <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
</div> </div>
<div class="modal-body"> <div class="modal-body">
@ -15,9 +15,9 @@
</div> </div>
</div> </div>
<div class="modal-footer"> <div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">close</button> <button type="button" class="btn btn-primary" data-bs-dismiss="modal" onclick="addPackages()"><i class="bi bi-play"></i> add</button>
<button type="button" class="btn btn-success" data-bs-dismiss="modal" onclick="requestPackages()">request</button> <button type="button" class="btn btn-success" data-bs-dismiss="modal" onclick="requestPackages()"><i class="bi bi-plus"></i> request</button>
<button type="button" class="btn btn-primary" data-bs-dismiss="modal" onclick="addPackages()">add</button> <button type="button" class="btn btn-secondary" data-bs-dismiss="modal"><i class="bi bi-x"></i> close</button>
</div> </div>
</div> </div>
</div> </div>

View File

@ -0,0 +1,71 @@
<div id="package-info-form" tabindex="-1" role="dialog" class="modal fade">
<div class="modal-dialog modal-xl" role="document">
<div class="modal-content">
<div id="package-info-modal-header" class="modal-header">
<h4 id="package-info" class="modal-title"></h4>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
</div>
<div class="modal-body">
<pre class="pre-scrollable language-logs"><code id="package-info-logs" class="language-logs"></code><button id="copy-btn" type="button" class="btn language-logs" onclick="copyLogs()"><i class="bi bi-clipboard"></i> copy</button></pre>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" onclick="showLogs()"><i class="bi bi-arrow-clockwise"></i> reload</button>
<button type="button" class="btn btn-primary" data-bs-dismiss="modal"><i class="bi bi-x"></i> close</button>
</div>
</div>
</div>
</div>
<script>
const packageInfo = $("#package-info");
const packageInfoForm = $("#package-info-form");
const packageInfoHeader = $("#package-info-modal-header");
const packageInfoLogs = $("#package-info-logs");
const packageInfoLogsCopyButton = $("#copy-btn");
async function copyLogs() {
const logs = packageInfoLogs.text();
await navigator.clipboard.writeText(logs);
packageInfoLogsCopyButton.html("<i class=\"bi bi-clipboard-check\"></i> copied");
setTimeout(()=> {
packageInfoLogsCopyButton.html("<i class=\"bi bi-clipboard\"></i> copy");
}, 2000);
}
function showLogs(package) {
const isPackageBaseSet = package !== undefined;
if (isPackageBaseSet)
packageInfoForm.data("package", package); // set package base as currently used
else
package = packageInfoForm.data("package"); // read package base from the current window attribute
const headerClass = status => {
if (status === "pending") return ["bg-warning"];
if (status === "building") return ["bg-warning"];
if (status === "failed") return ["bg-danger", "text-white"];
if (status === "success") return ["bg-success", "text-white"];
return ["bg-secondary", "text-white"];
};
$.ajax({
url: `/api/v1/packages/${package}/logs`,
type: "GET",
dataType: "json",
success: response => {
packageInfo.text(`${response.package_base} ${response.status.status} at ${new Date(1000 * response.status.timestamp).toISOString()}`);
packageInfoLogs.text(response.logs);
packageInfoHeader.removeClass();
packageInfoHeader.addClass("modal-header");
headerClass(response.status.status).forEach((clz) => packageInfoHeader.addClass(clz));
if (isPackageBaseSet) packageInfoForm.modal("show"); // we don't need to show window again
},
error: (jqXHR, _, errorThrown) => {
// show failed modal in case if first time loading
if (isPackageBaseSet) showFailure("Load failure", `Could not load package ${package} logs:`, errorThrown);
},
});
}
</script>

View File

@ -1,16 +1,16 @@
<div id="success-form" tabindex="-1" role="dialog" class="modal fade"> <div id="success-form" tabindex="-1" role="dialog" class="modal fade">
<div class="modal-dialog" role="document"> <div class="modal-dialog" role="document">
<div class="modal-content"> <div class="modal-content">
<div class="modal-header bg-success"> <div class="modal-header bg-success text-white">
<h4 class="modal-title">success</h4> <h4 id="success-title" class="modal-title"></h4>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button> <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
</div> </div>
<div class="modal-body"> <div class="modal-body">
<p>Packages update has been run.</p> <p id="success-description"></p>
<ul id="success-details"></ul> <ul id="success-details"></ul>
</div> </div>
<div class="modal-footer"> <div class="modal-footer">
<button type="button" class="btn btn-primary" data-bs-dismiss="modal">close</button> <button type="button" class="btn btn-primary" data-bs-dismiss="modal"><i class="bi bi-x"></i> close</button>
</div> </div>
</div> </div>
</div> </div>
@ -18,10 +18,14 @@
<script> <script>
const successForm = $("#success-form"); const successForm = $("#success-form");
const successDescription = $("#success-description");
const successDetails = $("#success-details"); const successDetails = $("#success-details");
const successTitle = $("#success-title");
successForm.on("hidden.bs.modal", () => { reload(); }); successForm.on("hidden.bs.modal", () => { reload(); });
function showSuccess(details) { function showSuccess(title, description, details) {
successTitle.text(title);
successDescription.text(description);
successDetails.empty().append(details); successDetails.empty().append(details);
successForm.modal("show"); successForm.modal("show");
} }

View File

@ -8,6 +8,7 @@
() => { () => {
removeButton.prop("disabled", !table.bootstrapTable("getSelections").length); removeButton.prop("disabled", !table.bootstrapTable("getSelections").length);
}); });
table.on("click-row.bs.table", (_, row) => { showLogs(row.id); });
const architectureBadge = $("#badge-architecture"); const architectureBadge = $("#badge-architecture");
const repositoryBadge = $("#badge-repository"); const repositoryBadge = $("#badge-repository");
@ -26,9 +27,11 @@
li.innerText = pkg; li.innerText = pkg;
return li; return li;
}); });
showSuccess(details); showSuccess("Success", `Package action at ${uri} has been run on:`, details);
},
error: (jqXHR, _, errorThrown) => {
showFailure("Action failed", `Package action request at ${uri} on ${packages} has failed:`, errorThrown);
}, },
error: (jqXHR, _, errorThrown) => { showFailure(errorThrown); },
}); });
} }
@ -38,7 +41,11 @@
function removePackages() { doPackageAction("/api/v1/service/remove", getSelection()); } function removePackages() { doPackageAction("/api/v1/service/remove", getSelection()); }
function updatePackages() { doPackageAction("/api/v1/service/add", getSelection()); } function updatePackages() {
const currentSelection = getSelection();
const url = currentSelection.length === 0 ? "/api/v1/service/update" : "/api/v1/service/add";
doPackageAction(url, getSelection());
}
function hideControls(hidden) { function hideControls(hidden) {
addButton.attr("hidden", hidden); addButton.attr("hidden", hidden);
@ -95,7 +102,7 @@
table.bootstrapTable("hideLoading"); table.bootstrapTable("hideLoading");
} else { } else {
// other errors // other errors
showFailure(errorThrown); showFailure("Load failure", "Could not load list of packages:", errorThrown);
} }
hideControls(true); hideControls(true);
}, },

View File

@ -19,10 +19,11 @@
<p>This repository is signed with <a href="https://pgp.mit.edu/pks/lookup?search=0x{{ pgp_key }}&fingerprint=on&op=index" title="key search">{{ pgp_key }}</a> by default.</p> <p>This repository is signed with <a href="https://pgp.mit.edu/pks/lookup?search=0x{{ pgp_key }}&fingerprint=on&op=index" title="key search">{{ pgp_key }}</a> by default.</p>
{% endif %} {% endif %}
<pre>$ cat /etc/pacman.conf <p>In order to use this repository edit your <code>/etc/pacman.conf</code> as following:</p>
[{{ repository }}]
<pre class="language-ini"><code id="pacman-conf" class="language-ini">[{{ repository }}]
Server = {{ link_path }} Server = {{ link_path }}
SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly</pre> SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly</code><button id="copy-btn" type="button" class="btn language-ini" onclick="copyPacmanConf()"><i class="bi bi-clipboard"></i> copy</button></pre>
</div> </div>
<div class="container"> <div class="container">
@ -83,17 +84,32 @@ SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Pa
<footer class="d-flex flex-wrap justify-content-between align-items-center border-top"> <footer class="d-flex flex-wrap justify-content-between align-items-center border-top">
<ul class="nav"> <ul class="nav">
{% if homepage is not none %} {% if homepage is not none %}
<li><a class="nav-link" href="{{ homepage }}" title="homepage">homepage</a></li> <li><a class="nav-link" href="{{ homepage }}" title="homepage"><i class="bi bi-house"></i> homepage</a></li>
{% endif %} {% endif %}
</ul> </ul>
<ul class="nav"> <ul class="nav">
<li><a class="nav-link" href="https://github.com/arcan1s/ahriman" title="sources">ahriman</a></li> <li><a class="nav-link" href="https://github.com/arcan1s/ahriman" title="sources"><i class="bi bi-github"></i> ahriman</a></li>
</ul> </ul>
</footer> </footer>
</div> </div>
{% include "utils/bootstrap-scripts.jinja2" %} {% include "utils/bootstrap-scripts.jinja2" %}
<script>
const pacmanConf = $("#pacman-conf");
const pacmanConfCopyButton = $("#copy-btn");
async function copyPacmanConf() {
const conf = pacmanConf.text();
await navigator.clipboard.writeText(conf);
pacmanConfCopyButton.html("<i class=\"bi bi-clipboard-check\"></i> copied");
setTimeout(() => {
pacmanConfCopyButton.html("<i class=\"bi bi-clipboard\"></i> copy");
}, 2000);
}
</script>
</body> </body>
</html> </html>

View File

@ -4,9 +4,10 @@
<script src="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.min.js"></script> <script src="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-ka7Sk0Gln4gmtz2MlQnikT1wXgYsOg+OMhuP+IlRH9sENBO0LRn5q+8nbTov4+1p" crossorigin="anonymous"></script> <script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.6/dist/umd/popper.min.js" integrity="sha384-oBqDVmMz9ATKxIep9tiCxS/Z9fNfEXiDAYTujMAeBAsjFuCZSmKbSSUnQlmh/jp3" crossorigin="anonymous"></script>
<script src="https://unpkg.com/bootstrap-table@1.20.2/dist/bootstrap-table.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.2/dist/js/bootstrap.min.js" integrity="sha384-IDwe1+LCz02ROU9k972gdyvl+AESN10+x7tBKgc9I5HFtuNz0wWnPclzo6p9vxnk" crossorigin="anonymous"></script>
<script src="https://unpkg.com/bootstrap-table@1.21.1/dist/bootstrap-table.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.20.2/dist/extensions/export/bootstrap-table-export.min.js"></script> <script src="https://unpkg.com/bootstrap-table@1.21.1/dist/extensions/export/bootstrap-table-export.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.20.2/dist/extensions/resizable/bootstrap-table-resizable.js"></script> <script src="https://unpkg.com/bootstrap-table@1.21.1/dist/extensions/resizable/bootstrap-table-resizable.js"></script>

View File

@ -1,9 +1,23 @@
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous" type="text/css"> <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.2/dist/css/bootstrap.min.css" integrity="sha384-Zenh87qX5JnK2Jl0vWa8Ck2rdkQ2Bzep5IDxbcnCeuOxjzrPF/et3URy9Bv1WTRi" crossorigin="anonymous" type="text/css">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.8.3/font/bootstrap-icons.css" type="text/css"> <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.2/font/bootstrap-icons.css" type="text/css">
<link rel="stylesheet" href="https://unpkg.com/bootstrap-table@1.20.2/dist/bootstrap-table.min.css" type="text/css"> <link rel="stylesheet" href="https://unpkg.com/bootstrap-table@1.21.1/dist/bootstrap-table.min.css" type="text/css">
<link href="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.css" rel="stylesheet"> <link href="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.css" rel="stylesheet">
<style> <style>
.pre-scrollable {
max-height: 680px;
overflow-y: scroll;
}
pre[class*="language-"] {
position: relative;
}
pre[class*="language-"] button{
position: absolute;
top: 0px;
right: 5px;
}
</style> </style>

View File

@ -74,6 +74,7 @@ setup(
"package/share/ahriman/templates/build-status/failed-modal.jinja2", "package/share/ahriman/templates/build-status/failed-modal.jinja2",
"package/share/ahriman/templates/build-status/login-modal.jinja2", "package/share/ahriman/templates/build-status/login-modal.jinja2",
"package/share/ahriman/templates/build-status/package-add-modal.jinja2", "package/share/ahriman/templates/build-status/package-add-modal.jinja2",
"package/share/ahriman/templates/build-status/package-info-modal.jinja2",
"package/share/ahriman/templates/build-status/success-modal.jinja2", "package/share/ahriman/templates/build-status/success-modal.jinja2",
"package/share/ahriman/templates/build-status/table.jinja2", "package/share/ahriman/templates/build-status/table.jinja2",
]), ]),

View File

@ -19,7 +19,7 @@
# #
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository from ahriman.core.repository import Repository
@ -44,7 +44,8 @@ class ApplicationProperties(LazyLogging):
configuration(Configuration): configuration instance configuration(Configuration): configuration instance
report(bool): force enable or disable reporting report(bool): force enable or disable reporting
unsafe(bool): if set no user check will be performed before path creation unsafe(bool): if set no user check will be performed before path creation
refresh_pacman_database(int): pacman database syncronization level, ``0`` is disabled refresh_pacman_database(int, optional): pacman database syncronization level, ``0`` is disabled
(Default value = 0)
""" """
self.configuration = configuration self.configuration = configuration
self.architecture = architecture self.architecture = architecture

View File

@ -28,6 +28,7 @@ from typing import List, Type
from ahriman.application.lock import Lock from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError
from ahriman.core.log import Log
from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.repository_paths import RepositoryPaths
@ -94,7 +95,8 @@ class Handler:
bool: True on success, False otherwise bool: True on success, False otherwise
""" """
try: try:
configuration = Configuration.from_path(args.configuration, architecture, args.quiet) configuration = Configuration.from_path(args.configuration, architecture)
Log.load(configuration, quiet=args.quiet, report=args.report)
with Lock(args, architecture, configuration): with Lock(args, architecture, configuration):
cls.run(args, architecture, configuration, report=args.report, unsafe=args.unsafe) cls.run(args, architecture, configuration, report=args.report, unsafe=args.unsafe)
return True return True

View File

@ -28,7 +28,7 @@ from typing import Literal, Optional, Type
from ahriman import version from ahriman import version
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import DuplicateRunError from ahriman.core.exceptions import DuplicateRunError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client from ahriman.core.status.client import Client
from ahriman.core.util import check_user from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum from ahriman.models.build_status import BuildStatusEnum
@ -73,7 +73,7 @@ class Lock(LazyLogging):
self.unsafe = args.unsafe self.unsafe = args.unsafe
self.paths = configuration.repository_paths self.paths = configuration.repository_paths
self.reporter = Client.load(configuration) if args.report else Client() self.reporter = Client.load(configuration, report=args.report)
def __enter__(self) -> Lock: def __enter__(self) -> Lock:
""" """

View File

@ -24,7 +24,7 @@ from pyalpm import DB, Handle, Package, SIG_PACKAGE, error as PyalpmError # typ
from typing import Generator, Set from typing import Generator, Set
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.repository_paths import RepositoryPaths

View File

@ -22,7 +22,7 @@ from __future__ import annotations
from typing import Dict, List, Type from typing import Dict, List, Type
from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.pacman import Pacman
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.aur_package import AURPackage from ahriman.models.aur_package import AURPackage

View File

@ -21,7 +21,7 @@ from pathlib import Path
from typing import List from typing import List
from ahriman.core.exceptions import BuildError from ahriman.core.exceptions import BuildError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output from ahriman.core.util import check_output
from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.repository_paths import RepositoryPaths

View File

@ -23,7 +23,7 @@ from typing import Optional, Type
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.auth_settings import AuthSettings from ahriman.models.auth_settings import AuthSettings
from ahriman.models.user_access import UserAccess from ahriman.models.user_access import UserAccess
@ -62,7 +62,7 @@ class Auth(LazyLogging):
Returns: Returns:
str: login control as html code to insert str: login control as html code to insert
""" """
return """<button type="button" class="btn btn-link" data-bs-toggle="modal" data-bs-target="#loginForm" style="text-decoration: none">login</button>""" return """<button type="button" class="btn btn-link" data-bs-toggle="modal" data-bs-target="#loginForm" style="text-decoration: none"><i class="bi bi-box-arrow-in-right"></i> login</button>"""
@classmethod @classmethod
def load(cls: Type[Auth], configuration: Configuration, database: SQLite) -> Auth: def load(cls: Type[Auth], configuration: Configuration, database: SQLite) -> Auth:

View File

@ -69,7 +69,7 @@ class OAuth(Mapping):
Returns: Returns:
str: login control as html code to insert str: login control as html code to insert
""" """
return """<a class="nav-link" href="/api/v1/login" title="login via OAuth2">login</a>""" return """<a class="nav-link" href="/api/v1/login" title="login via OAuth2"><i class="bi bi-google"></i> login</a>"""
@staticmethod @staticmethod
def get_provider(name: str) -> Type[aioauth_client.OAuth2Client]: def get_provider(name: str) -> Type[aioauth_client.OAuth2Client]:

View File

@ -23,7 +23,7 @@ import shutil
from pathlib import Path from pathlib import Path
from typing import List, Optional from typing import List, Optional
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, walk from ahriman.core.util import check_output, walk
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
@ -174,7 +174,8 @@ class Sources(LazyLogging):
sources_dir(Path): local path to git repository sources_dir(Path): local path to git repository
remote(RemoteSource): remote target, branch and url remote(RemoteSource): remote target, branch and url
*pattern(str): glob patterns *pattern(str): glob patterns
commit_author(Optional[str]): commit author in form of git config (i.e. ``user <user@host>``) commit_author(Optional[str], optional): commit author in form of git config (i.e. ``user <user@host>``)
(Default value = None)
""" """
instance = Sources() instance = Sources()
instance.add(sources_dir, *pattern) instance.add(sources_dir, *pattern)
@ -188,7 +189,8 @@ class Sources(LazyLogging):
Args: Args:
sources_dir(Path): local path to git repository sources_dir(Path): local path to git repository
*pattern(str): glob patterns *pattern(str): glob patterns
intent_to_add(bool): record only the fact that it will be added later, acts as --intent-to-add git flag intent_to_add(bool, optional): record only the fact that it will be added later, acts as
--intent-to-add git flag (Default value = False)
""" """
# glob directory to find files which match the specified patterns # glob directory to find files which match the specified patterns
found_files: List[Path] = [] found_files: List[Path] = []
@ -208,9 +210,9 @@ class Sources(LazyLogging):
Args: Args:
sources_dir(Path): local path to git repository sources_dir(Path): local path to git repository
message(Optional[str]): optional commit message if any. If none set, message will be generated according to message(Optional[str], optional): optional commit message if any. If none set, message will be generated
the current timestamp according to the current timestamp (Default value = None)
author(Optional[str]): optional commit author if any author(Optional[str], optional): optional commit author if any (Default value = None)
""" """
if message is None: if message is None:
message = f"Autogenerated commit at {datetime.datetime.utcnow()}" message = f"Autogenerated commit at {datetime.datetime.utcnow()}"

View File

@ -24,7 +24,7 @@ from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.exceptions import BuildError from ahriman.core.exceptions import BuildError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output from ahriman.core.util import check_output
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.repository_paths import RepositoryPaths
@ -84,10 +84,12 @@ class Task(LazyLogging):
user=self.uid) user=self.uid)
# well it is not actually correct, but we can deal with it # well it is not actually correct, but we can deal with it
packages = Task._check_output("makepkg", "--packagelist", packages = Task._check_output(
exception=BuildError(self.package.base), "makepkg", "--packagelist",
cwd=sources_dir, exception=BuildError(self.package.base),
logger=self.logger).splitlines() cwd=sources_dir,
logger=self.logger
).splitlines()
return [Path(package) for package in packages] return [Path(package) for package in packages]
def init(self, sources_dir: Path, database: SQLite) -> None: def init(self, sources_dir: Path, database: SQLite) -> None:

View File

@ -20,10 +20,8 @@
from __future__ import annotations from __future__ import annotations
import configparser import configparser
import logging
import sys import sys
from logging.config import fileConfig
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Generator, List, Optional, Tuple, Type from typing import Any, Dict, Generator, List, Optional, Tuple, Type
@ -38,8 +36,6 @@ class Configuration(configparser.RawConfigParser):
Attributes: Attributes:
ARCHITECTURE_SPECIFIC_SECTIONS(List[str]): (class attribute) known sections which can be architecture specific. ARCHITECTURE_SPECIFIC_SECTIONS(List[str]): (class attribute) known sections which can be architecture specific.
Required by dump and merging functions Required by dump and merging functions
DEFAULT_LOG_FORMAT(str): (class attribute) default log format (in case of fallback)
DEFAULT_LOG_LEVEL(int): (class attribute) default log level (in case of fallback)
SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package
architecture(Optional[str]): repository architecture architecture(Optional[str]): repository architecture
path(Optional[Path]): path to root configuration file path(Optional[Path]): path to root configuration file
@ -64,9 +60,6 @@ class Configuration(configparser.RawConfigParser):
>>> path, architecture = configuration.check_loaded() >>> path, architecture = configuration.check_loaded()
""" """
DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d %(funcName)s]: %(message)s"
DEFAULT_LOG_LEVEL = logging.DEBUG
ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "sign", "web"] ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "sign", "web"]
SYSTEM_CONFIGURATION_PATH = Path(sys.prefix) / "share" / "ahriman" / "settings" / "ahriman.ini" SYSTEM_CONFIGURATION_PATH = Path(sys.prefix) / "share" / "ahriman" / "settings" / "ahriman.ini"
@ -75,8 +68,8 @@ class Configuration(configparser.RawConfigParser):
default constructor. In the most cases must not be called directly default constructor. In the most cases must not be called directly
Args: Args:
allow_no_value(bool): copies ``configparser.RawConfigParser`` behaviour. In case if it is set to ``True``, allow_no_value(bool, optional): copies ``configparser.RawConfigParser`` behaviour. In case if it is set
the keys without values will be allowed to ``True``, the keys without values will be allowed (Default value = False)
""" """
configparser.RawConfigParser.__init__(self, allow_no_value=allow_no_value, converters={ configparser.RawConfigParser.__init__(self, allow_no_value=allow_no_value, converters={
"list": self.__convert_list, "list": self.__convert_list,
@ -117,14 +110,13 @@ class Configuration(configparser.RawConfigParser):
return RepositoryPaths(self.getpath("repository", "root"), architecture) return RepositoryPaths(self.getpath("repository", "root"), architecture)
@classmethod @classmethod
def from_path(cls: Type[Configuration], path: Path, architecture: str, quiet: bool) -> Configuration: def from_path(cls: Type[Configuration], path: Path, architecture: str) -> Configuration:
""" """
constructor with full object initialization constructor with full object initialization
Args: Args:
path(Path): path to root configuration file path(Path): path to root configuration file
architecture(str): repository architecture architecture(str): repository architecture
quiet(bool): force disable any log messages
Returns: Returns:
Configuration: configuration instance Configuration: configuration instance
@ -132,7 +124,6 @@ class Configuration(configparser.RawConfigParser):
configuration = cls() configuration = cls()
configuration.load(path) configuration.load(path)
configuration.merge_sections(architecture) configuration.merge_sections(architecture)
configuration.load_logging(quiet)
return configuration return configuration
@staticmethod @staticmethod
@ -281,23 +272,6 @@ class Configuration(configparser.RawConfigParser):
except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError): except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError):
pass pass
def load_logging(self, quiet: bool) -> None:
"""
setup logging settings from configuration
Args:
quiet(bool): force disable any log messages
"""
try:
path = self.logging_path
fileConfig(path)
except Exception:
logging.basicConfig(filename=None, format=self.DEFAULT_LOG_FORMAT,
level=self.DEFAULT_LOG_LEVEL)
logging.exception("could not load logging from configuration, fallback to stderr")
if quiet:
logging.disable(logging.WARNING) # only print errors here
def merge_sections(self, architecture: str) -> None: def merge_sections(self, architecture: str) -> None:
""" """
merge architecture specific sections into main configuration merge architecture specific sections into main configuration

View File

@ -27,7 +27,7 @@ from typing import List, Type
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database.data import migrate_data from ahriman.core.database.data import migrate_data
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.migration import Migration from ahriman.models.migration import Migration
from ahriman.models.migration_result import MigrationResult from ahriman.models.migration_result import MigrationResult

View File

@ -0,0 +1,35 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ["steps"]
steps = [
"""
create table logs (
package_base text not null,
process_id integer not null,
created real not null,
record text
)
""",
"""
create index logs_package_base_process_id on logs (package_base, process_id)
""",
]

View File

@ -21,5 +21,6 @@ from ahriman.core.database.operations.operations import Operations
from ahriman.core.database.operations.auth_operations import AuthOperations from ahriman.core.database.operations.auth_operations import AuthOperations
from ahriman.core.database.operations.build_operations import BuildOperations from ahriman.core.database.operations.build_operations import BuildOperations
from ahriman.core.database.operations.logs_operations import LogsOperations
from ahriman.core.database.operations.package_operations import PackageOperations from ahriman.core.database.operations.package_operations import PackageOperations
from ahriman.core.database.operations.patch_operations import PatchOperations from ahriman.core.database.operations.patch_operations import PatchOperations

View File

@ -26,7 +26,7 @@ from ahriman.models.package import Package
class BuildOperations(Operations): class BuildOperations(Operations):
""" """
operations for main functions operations for build queue functions
""" """
def build_queue_clear(self, package_base: Optional[str]) -> None: def build_queue_clear(self, package_base: Optional[str]) -> None:

View File

@ -0,0 +1,102 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from typing import List, Optional
from ahriman.core.database.operations import Operations
from ahriman.core.util import pretty_datetime
from ahriman.models.log_record_id import LogRecordId
class LogsOperations(Operations):
"""
logs operations
"""
def logs_get(self, package_base: str) -> str:
"""
extract logs for specified package base
Args:
package_base(str): package base to extract logs
Return:
str: full package log
"""
def run(connection: Connection) -> List[str]:
return [
f"""[{pretty_datetime(row["created"])}] {row["record"]}"""
for row in connection.execute(
"""
select created, record from logs where package_base = :package_base
order by created asc
""",
{"package_base": package_base})
]
records = self.with_connection(run)
return "\n".join(records)
def logs_insert(self, log_record_id: LogRecordId, created: float, record: str) -> None:
"""
write new log record to database
Args:
log_record_id(LogRecordId): current log record id
created(float): log created timestamp from log record attribute
record(str): log record
"""
def run(connection: Connection) -> None:
connection.execute(
"""
insert into logs
(package_base, process_id, created, record)
values
(:package_base, :process_id, :created, :record)
""",
dict(
package_base=log_record_id.package_base,
process_id=log_record_id.process_id,
created=created,
record=record
)
)
return self.with_connection(run, commit=True)
def logs_remove(self, package_base: str, current_process_id: Optional[int]) -> None:
"""
remove log records for the specified package
Args:
package_base(str): package base to remove logs
current_process_id(Optional[int]): current process id. If set it will remove only logs belonging to another
process
"""
def run(connection: Connection) -> None:
connection.execute(
"""
delete from logs
where package_base = :package_base and (:process_id is null or process_id <> :process_id)
""",
{"package_base": package_base, "process_id": current_process_id}
)
return self.with_connection(run, commit=True)

View File

@ -22,7 +22,8 @@ import sqlite3
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Tuple, TypeVar, Callable from typing import Any, Dict, Tuple, TypeVar, Callable
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
T = TypeVar("T") T = TypeVar("T")

View File

@ -27,10 +27,11 @@ from typing import Type
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database.migrations import Migrations from ahriman.core.database.migrations import Migrations
from ahriman.core.database.operations import AuthOperations, BuildOperations, PackageOperations, PatchOperations from ahriman.core.database.operations import AuthOperations, BuildOperations, LogsOperations, PackageOperations, \
PatchOperations
class SQLite(AuthOperations, BuildOperations, PackageOperations, PatchOperations): class SQLite(AuthOperations, BuildOperations, LogsOperations, PackageOperations, PatchOperations):
""" """
wrapper for sqlite3 database wrapper for sqlite3 database

View File

@ -25,7 +25,7 @@ from tempfile import TemporaryDirectory
from ahriman.core.build_tools.sources import Sources from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import GitRemoteError from ahriman.core.exceptions import GitRemoteError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.util import walk from ahriman.core.util import walk
from ahriman.models.package_source import PackageSource from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource from ahriman.models.remote_source import RemoteSource

View File

@ -26,7 +26,7 @@ from typing import Generator
from ahriman.core.build_tools.sources import Sources from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import GitRemoteError from ahriman.core.exceptions import GitRemoteError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource from ahriman.models.remote_source import RemoteSource

View File

@ -0,0 +1,21 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.log.lazy_logging import LazyLogging
from ahriman.core.log.log import Log

View File

@ -0,0 +1,61 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import re
from aiohttp.abc import BaseRequest, StreamResponse
from aiohttp.web_log import AccessLogger
class FilteredAccessLogger(AccessLogger):
"""
access logger implementation with log filter enabled
Attributes:
LOG_PATH_REGEX(re.Pattern): (class attribute) regex for logs uri
"""
# official packages have only ``[A-Za-z0-9_.+-]`` regex
LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[A-Za-z0-9_.+%-]+/logs$")
@staticmethod
def is_logs_post(request: BaseRequest) -> bool:
"""
check if request looks lie logs posting
Args:
request(BaseRequest): http reqeust descriptor
Returns:
bool: True in case if request looks like logs positing and False otherwise
"""
return request.method == "POST" and FilteredAccessLogger.LOG_PATH_REGEX.match(request.path) is not None
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
"""
access log with enabled filter by request path
Args:
request(BaseRequest): http reqeust descriptor
response(StreamResponse): streaming response object
time(float):
"""
if self.is_logs_post(request):
return
AccessLogger.log(self, request, response, time)

View File

@ -0,0 +1,80 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import logging
from ahriman.core.configuration import Configuration
class HttpLogHandler(logging.Handler):
"""
handler for the http logging. Because default ``logging.handlers.HTTPHandler`` does not support cookies
authorization, we have to implement own handler which overrides the ``logging.handlers.HTTPHandler.emit`` method
Attributes:
reporter(Client): build status reporter instance
"""
def __init__(self, configuration: Configuration, *, report: bool) -> None:
"""
default constructor
Args:
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we don't really care about those parameters because they will be handled by the reporter
logging.Handler.__init__(self)
# client has to be importer here because of circular imports
from ahriman.core.status.client import Client
self.reporter = Client.load(configuration, report=report)
@classmethod
def load(cls, configuration: Configuration, *, report: bool) -> HttpLogHandler:
"""
install logger. This function creates handler instance and adds it to the handler list in case if no other
http handler found
Args:
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
root = logging.getLogger()
if (handler := next((handler for handler in root.handlers if isinstance(handler, cls)), None)) is not None:
return handler # there is already registered instance
handler = cls(configuration, report=report)
root.addHandler(handler)
return handler
def emit(self, record: logging.LogRecord) -> None:
"""
emit log records using reporter client
Args:
record(logging.LogRecord): log record to log
"""
try:
self.reporter.logs(record)
except Exception:
self.handleError(record)

View File

@ -17,9 +17,10 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
import contextlib
import logging import logging
from typing import Any from typing import Any, Generator
class LazyLogging: class LazyLogging:
@ -62,3 +63,47 @@ class LazyLogging:
clazz = self.__class__ clazz = self.__class__
prefix = "" if clazz.__module__ is None else f"{clazz.__module__}." prefix = "" if clazz.__module__ is None else f"{clazz.__module__}."
return f"{prefix}{clazz.__qualname__}" return f"{prefix}{clazz.__qualname__}"
@staticmethod
def _package_logger_reset() -> None:
"""
reset package logger to empty one
"""
logging.setLogRecordFactory(logging.LogRecord)
@staticmethod
def _package_logger_set(package_base: str) -> None:
"""
set package base as extra info to the logger
Args:
package_base(str): package base
"""
current_factory = logging.getLogRecordFactory()
def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord:
record = current_factory(*args, **kwargs)
record.package_base = package_base
return record
logging.setLogRecordFactory(package_record_factory)
@contextlib.contextmanager
def in_package_context(self, package_base: str) -> Generator[None, None, None]:
"""
execute function while setting package context
Args:
package_base(str): package base to set context in
Examples:
This function is designed to be called as context manager with ``package_base`` argument, e.g.:
>>> with self.in_package_context(package.base):
>>> build_package(package)
"""
try:
self._package_logger_set(package_base)
yield
finally:
self._package_logger_reset()

View File

@ -0,0 +1,61 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
from logging.config import fileConfig
from ahriman.core.configuration import Configuration
from ahriman.core.log.http_log_handler import HttpLogHandler
class Log:
"""
simple static method class which setups application loggers
Attributes:
DEFAULT_LOG_FORMAT(str): (class attribute) default log format (in case of fallback)
DEFAULT_LOG_LEVEL(int): (class attribute) default log level (in case of fallback)
"""
DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d %(funcName)s]: %(message)s"
DEFAULT_LOG_LEVEL = logging.DEBUG
@staticmethod
def load(configuration: Configuration, *, quiet: bool, report: bool) -> None:
"""
setup logging settings from configuration
Args:
configuration(Configuration): configuration instance
quiet(bool): force disable any log messages
report(bool): force enable or disable reporting
"""
try:
path = configuration.logging_path
fileConfig(path)
except Exception:
logging.basicConfig(filename=None, format=Log.DEFAULT_LOG_FORMAT,
level=Log.DEFAULT_LOG_LEVEL)
logging.exception("could not load logging from configuration, fallback to stderr")
HttpLogHandler.load(configuration, report=report)
if quiet:
logging.disable(logging.WARNING) # only print errors here

View File

@ -23,7 +23,7 @@ from typing import Iterable, Type
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ReportError from ahriman.core.exceptions import ReportError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.report_settings import ReportSettings from ahriman.models.report_settings import ReportSettings
from ahriman.models.result import Result from ahriman.models.result import Result

View File

@ -84,7 +84,8 @@ class Executor(Cleaner):
result = Result() result = Result()
for single in updates: for single in updates:
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (build_dir := Path(dir_name)): with self.in_package_context(single.base), \
TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (build_dir := Path(dir_name)):
try: try:
build_single(single, build_dir) build_single(single, build_dir)
result.add_success(single) result.add_success(single)
@ -110,6 +111,7 @@ class Executor(Cleaner):
self.paths.tree_clear(package_base) # remove all internal files self.paths.tree_clear(package_base) # remove all internal files
self.database.build_queue_clear(package_base) self.database.build_queue_clear(package_base)
self.database.patches_remove(package_base, []) self.database.patches_remove(package_base, [])
self.database.logs_remove(package_base, None)
self.reporter.remove(package_base) # we only update status page in case of base removal self.reporter.remove(package_base) # we only update status page in case of base removal
except Exception: except Exception:
self.logger.exception("could not remove base %s", package_base) self.logger.exception("could not remove base %s", package_base)
@ -153,21 +155,21 @@ class Executor(Cleaner):
Returns: Returns:
Result: path to repository database Result: path to repository database
""" """
def rename(archive: PackageDescription, base: str) -> None: def rename(archive: PackageDescription, package_base: str) -> None:
if archive.filename is None: if archive.filename is None:
self.logger.warning("received empty package name for base %s", base) self.logger.warning("received empty package name for base %s", package_base)
return # suppress type checking, it never can be none actually return # suppress type checking, it never can be none actually
if (safe := safe_filename(archive.filename)) != archive.filename: if (safe := safe_filename(archive.filename)) != archive.filename:
shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe) shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe)
archive.filename = safe archive.filename = safe
def update_single(name: Optional[str], base: str) -> None: def update_single(name: Optional[str], package_base: str) -> None:
if name is None: if name is None:
self.logger.warning("received empty package name for base %s", base) self.logger.warning("received empty package name for base %s", package_base)
return # suppress type checking, it never can be none actually return # suppress type checking, it never can be none actually
# in theory, it might be NOT packages directory, but we suppose it is # in theory, it might be NOT packages directory, but we suppose it is
full_path = self.paths.packages / name full_path = self.paths.packages / name
files = self.sign.process_sign_package(full_path, base) files = self.sign.process_sign_package(full_path, package_base)
for src in files: for src in files:
dst = self.paths.repository / safe_filename(src.name) dst = self.paths.repository / safe_filename(src.name)
shutil.move(src, dst) shutil.move(src, dst)
@ -180,24 +182,25 @@ class Executor(Cleaner):
result = Result() result = Result()
for local in updates: for local in updates:
try: with self.in_package_context(local.base):
for description in local.packages.values(): try:
rename(description, local.base) for description in local.packages.values():
update_single(description.filename, local.base) rename(description, local.base)
self.reporter.set_success(local) update_single(description.filename, local.base)
result.add_success(local) self.reporter.set_success(local)
result.add_success(local)
current_package_archives = { current_package_archives = {
package package
for current in current_packages for current in current_packages
if current.base == local.base if current.base == local.base
for package in current.packages for package in current.packages
} }
removed_packages.extend(current_package_archives.difference(local.packages)) removed_packages.extend(current_package_archives.difference(local.packages))
except Exception: except Exception:
self.reporter.set_failed(local.base) self.reporter.set_failed(local.base)
result.add_failed(local) result.add_failed(local)
self.logger.exception("could not process %s", local.base) self.logger.exception("could not process %s", local.base)
self.clear_packages() self.clear_packages()
self.process_remove(removed_packages) self.process_remove(removed_packages)

View File

@ -22,7 +22,7 @@ from ahriman.core.alpm.repo import Repo
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnsafeRunError from ahriman.core.exceptions import UnsafeRunError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.sign.gpg import GPG from ahriman.core.sign.gpg import GPG
from ahriman.core.status.client import Client from ahriman.core.status.client import Client
from ahriman.core.triggers import TriggerLoader from ahriman.core.triggers import TriggerLoader
@ -58,7 +58,8 @@ class RepositoryProperties(LazyLogging):
database(SQLite): database instance database(SQLite): database instance
report(bool): force enable or disable reporting report(bool): force enable or disable reporting
unsafe(bool): if set no user check will be performed before path creation unsafe(bool): if set no user check will be performed before path creation
refresh_pacman_database(int): pacman database syncronization level, ``0`` is disabled refresh_pacman_database(int, optional): pacman database syncronization level, ``0`` is disabled
(Default value = 0)
""" """
self.architecture = architecture self.architecture = architecture
self.configuration = configuration self.configuration = configuration
@ -77,5 +78,5 @@ class RepositoryProperties(LazyLogging):
self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database) self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database)
self.sign = GPG(architecture, configuration) self.sign = GPG(architecture, configuration)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client.load(configuration) if report else Client() self.reporter = Client.load(configuration, report=report)
self.triggers = TriggerLoader(architecture, configuration) self.triggers = TriggerLoader(architecture, configuration)

View File

@ -56,26 +56,26 @@ class UpdateHandler(Cleaner):
result: List[Package] = [] result: List[Package] = []
for local in self.packages(): for local in self.packages():
if local.base in self.ignore_list: with self.in_package_context(local.base):
continue if local.base in self.ignore_list:
if local.is_vcs and not vcs: continue
continue if local.is_vcs and not vcs:
if filter_packages and local.base not in filter_packages: continue
continue if filter_packages and local.base not in filter_packages:
source = local.remote.source if local.remote is not None else None continue
source = local.remote.source if local.remote is not None else None
try: try:
if source == PackageSource.Repository: if source == PackageSource.Repository:
remote = Package.from_official(local.base, self.pacman) remote = Package.from_official(local.base, self.pacman)
else: else:
remote = Package.from_aur(local.base, self.pacman) remote = Package.from_aur(local.base, self.pacman)
if local.is_outdated(remote, self.paths): if local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base) self.reporter.set_pending(local.base)
result.append(remote) result.append(remote)
except Exception: except Exception:
self.reporter.set_failed(local.base) self.reporter.set_failed(local.base)
self.logger.exception("could not load remote package %s", local.base) self.logger.exception("could not load remote package %s", local.base)
continue
return result return result
@ -89,20 +89,21 @@ class UpdateHandler(Cleaner):
result: List[Package] = [] result: List[Package] = []
packages = {local.base: local for local in self.packages()} packages = {local.base: local for local in self.packages()}
for dirname in self.paths.cache.iterdir(): for cache_dir in self.paths.cache.iterdir():
try: with self.in_package_context(cache_dir.name):
Sources.fetch(dirname, remote=None) try:
remote = Package.from_build(dirname) Sources.fetch(cache_dir, remote=None)
remote = Package.from_build(cache_dir)
local = packages.get(remote.base) local = packages.get(remote.base)
if local is None: if local is None:
self.reporter.set_unknown(remote) self.reporter.set_unknown(remote)
result.append(remote) result.append(remote)
elif local.is_outdated(remote, self.paths): elif local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base) self.reporter.set_pending(local.base)
result.append(remote) result.append(remote)
except Exception: except Exception:
self.logger.exception("could not process package at %s", dirname) self.logger.exception("could not process package at %s", cache_dir)
return result return result

View File

@ -24,7 +24,7 @@ from typing import List, Optional, Set, Tuple
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import BuildError from ahriman.core.exceptions import BuildError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, exception_response_text from ahriman.core.util import check_output, exception_response_text
from ahriman.models.sign_settings import SignSettings from ahriman.models.sign_settings import SignSettings
@ -157,20 +157,20 @@ class GPG(LazyLogging):
logger=self.logger) logger=self.logger)
return [path, path.parent / f"{path.name}.sig"] return [path, path.parent / f"{path.name}.sig"]
def process_sign_package(self, path: Path, base: str) -> List[Path]: def process_sign_package(self, path: Path, package_base: str) -> List[Path]:
""" """
sign package if required by configuration sign package if required by configuration
Args: Args:
path(Path): path to file to sign path(Path): path to file to sign
base(str): package base required to check for key overrides package_base(str): package base required to check for key overrides
Returns: Returns:
List[Path]: list of generated files including original file List[Path]: list of generated files including original file
""" """
if SignSettings.Packages not in self.targets: if SignSettings.Packages not in self.targets:
return [path] return [path]
key = self.configuration.get("sign", f"key_{base}", fallback=self.default_key) key = self.configuration.get("sign", f"key_{package_base}", fallback=self.default_key)
if key is None: if key is None:
self.logger.error("no default key set, skip package %s sign", path) self.logger.error("no default key set, skip package %s sign", path)
return [path] return [path]

View File

@ -27,7 +27,7 @@ from threading import Lock, Thread
from typing import Callable, Dict, Iterable, Tuple from typing import Callable, Dict, Iterable, Tuple
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.package_source import PackageSource from ahriman.models.package_source import PackageSource

View File

@ -19,6 +19,8 @@
# #
from __future__ import annotations from __future__ import annotations
import logging
from typing import List, Optional, Tuple, Type from typing import List, Optional, Tuple, Type
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
@ -33,19 +35,24 @@ class Client:
""" """
@classmethod @classmethod
def load(cls: Type[Client], configuration: Configuration) -> Client: def load(cls: Type[Client], configuration: Configuration, *, report: bool) -> Client:
""" """
load client from settings load client from settings
Args: Args:
configuration(Configuration): configuration instance configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
Returns: Returns:
Client: client according to current settings Client: client according to current settings
""" """
if not report:
return cls()
address = configuration.get("web", "address", fallback=None) address = configuration.get("web", "address", fallback=None)
host = configuration.get("web", "host", fallback=None) host = configuration.get("web", "host", fallback=None)
port = configuration.getint("web", "port", fallback=None) port = configuration.getint("web", "port", fallback=None)
if address or (host and port): if address or (host and port):
from ahriman.core.status.web_client import WebClient from ahriman.core.status.web_client import WebClient
return WebClient(configuration) return WebClient(configuration)
@ -60,17 +67,17 @@ class Client:
status(BuildStatusEnum): current package build status status(BuildStatusEnum): current package build status
""" """
def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: def get(self, package_base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
""" """
get package status get package status
Args: Args:
base(Optional[str]): package base to get package_base(Optional[str]): package base to get
Returns: Returns:
List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found
""" """
del base del package_base
return [] return []
def get_internal(self) -> InternalStatus: def get_internal(self) -> InternalStatus:
@ -82,20 +89,28 @@ class Client:
""" """
return InternalStatus(status=BuildStatus()) return InternalStatus(status=BuildStatus())
def remove(self, base: str) -> None: def logs(self, record: logging.LogRecord) -> None:
"""
post log record
Args:
record(logging.LogRecord): log record to post to api
"""
def remove(self, package_base: str) -> None:
""" """
remove packages from watcher remove packages from watcher
Args: Args:
base(str): package base to remove package_base(str): package base to remove
""" """
def update(self, base: str, status: BuildStatusEnum) -> None: def update(self, package_base: str, status: BuildStatusEnum) -> None:
""" """
update package build status. Unlike ``add`` it does not update package properties update package build status. Unlike ``add`` it does not update package properties
Args: Args:
base(str): package base to update package_base(str): package base to update
status(BuildStatusEnum): current package build status status(BuildStatusEnum): current package build status
""" """
@ -107,32 +122,32 @@ class Client:
status(BuildStatusEnum): current ahriman status status(BuildStatusEnum): current ahriman status
""" """
def set_building(self, base: str) -> None: def set_building(self, package_base: str) -> None:
""" """
set package status to building set package status to building
Args: Args:
base(str): package base to update package_base(str): package base to update
""" """
return self.update(base, BuildStatusEnum.Building) return self.update(package_base, BuildStatusEnum.Building)
def set_failed(self, base: str) -> None: def set_failed(self, package_base: str) -> None:
""" """
set package status to failed set package status to failed
Args: Args:
base(str): package base to update package_base(str): package base to update
""" """
return self.update(base, BuildStatusEnum.Failed) return self.update(package_base, BuildStatusEnum.Failed)
def set_pending(self, base: str) -> None: def set_pending(self, package_base: str) -> None:
""" """
set package status to pending set package status to pending
Args: Args:
base(str): package base to update package_base(str): package base to update
""" """
return self.update(base, BuildStatusEnum.Pending) return self.update(package_base, BuildStatusEnum.Pending)
def set_success(self, package: Package) -> None: def set_success(self, package: Package) -> None:
""" """

View File

@ -17,14 +17,17 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
import os
from typing import Dict, List, Optional, Tuple from typing import Dict, List, Optional, Tuple
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnknownPackageError from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository from ahriman.core.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
@ -57,6 +60,9 @@ class Watcher(LazyLogging):
self.known: Dict[str, Tuple[Package, BuildStatus]] = {} self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
self.status = BuildStatus() self.status = BuildStatus()
# special variables for updating logs
self._last_log_record_id = LogRecordId("", os.getpid())
@property @property
def packages(self) -> List[Tuple[Package, BuildStatus]]: def packages(self) -> List[Tuple[Package, BuildStatus]]:
""" """
@ -67,12 +73,12 @@ class Watcher(LazyLogging):
""" """
return list(self.known.values()) return list(self.known.values())
def get(self, base: str) -> Tuple[Package, BuildStatus]: def get(self, package_base: str) -> Tuple[Package, BuildStatus]:
""" """
get current package base build status get current package base build status
Args: Args:
base(str): package base package_base(str): package base
Returns: Returns:
Tuple[Package, BuildStatus]: package and its status Tuple[Package, BuildStatus]: package and its status
@ -81,9 +87,21 @@ class Watcher(LazyLogging):
UnknownPackage: if no package found UnknownPackage: if no package found
""" """
try: try:
return self.known[base] return self.known[package_base]
except KeyError: except KeyError:
raise UnknownPackageError(base) raise UnknownPackageError(package_base)
def get_logs(self, package_base: str) -> str:
"""
extract logs for the package base
Args:
package_base(str): package base
Returns:
str: package logs
"""
return self.database.logs_get(package_base)
def load(self) -> None: def load(self) -> None:
""" """
@ -110,6 +128,17 @@ class Watcher(LazyLogging):
""" """
self.known.pop(package_base, None) self.known.pop(package_base, None)
self.database.package_remove(package_base) self.database.package_remove(package_base)
self.remove_logs(package_base, None)
def remove_logs(self, package_base: str, current_process_id: Optional[int]) -> None:
"""
remove package related logs
Args:
package_base(str): package base
current_process_id(int): current process id
"""
self.database.logs_remove(package_base, current_process_id)
def update(self, package_base: str, status: BuildStatusEnum, package: Optional[Package]) -> None: def update(self, package_base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
""" """
@ -132,6 +161,21 @@ class Watcher(LazyLogging):
self.known[package_base] = (package, full_status) self.known[package_base] = (package, full_status)
self.database.package_update(package, full_status) self.database.package_update(package, full_status)
def update_logs(self, log_record_id: LogRecordId, created: float, record: str) -> None:
"""
make new log record into database
Args:
log_record_id(LogRecordId): log record id
created(float): log created record
record(str): log record
"""
if self._last_log_record_id != log_record_id:
# there is new log record, so we remove old ones
self.remove_logs(log_record_id.package_base, log_record_id.process_id)
self._last_log_record_id = log_record_id
self.database.logs_insert(log_record_id, created, record)
def update_self(self, status: BuildStatusEnum) -> None: def update_self(self, status: BuildStatusEnum) -> None:
""" """
update service status update service status

View File

@ -17,12 +17,13 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
import logging
import requests import requests
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client from ahriman.core.status.client import Client
from ahriman.core.util import exception_response_text from ahriman.core.util import exception_response_text
from ahriman.models.build_status import BuildStatusEnum, BuildStatus from ahriman.models.build_status import BuildStatusEnum, BuildStatus
@ -114,17 +115,29 @@ class WebClient(Client, LazyLogging):
except Exception: except Exception:
self.logger.exception("could not login as %s", self.user) self.logger.exception("could not login as %s", self.user)
def _package_url(self, base: str = "") -> str: def _logs_url(self, package_base: str) -> str:
"""
get url for the logs api
Args:
package_base(str): package base
Returns:
str: full url for web service for logs
"""
return f"{self.address}/api/v1/packages/{package_base}/logs"
def _package_url(self, package_base: str = "") -> str:
""" """
url generator url generator
Args: Args:
base(str, optional): package base to generate url (Default value = "") package_base(str, optional): package base to generate url (Default value = "")
Returns: Returns:
str: full url of web service for specific package base str: full url of web service for specific package base
""" """
return f"{self.address}/api/v1/packages/{base}" return f"{self.address}/api/v1/packages/{package_base}"
def add(self, package: Package, status: BuildStatusEnum) -> None: def add(self, package: Package, status: BuildStatusEnum) -> None:
""" """
@ -147,18 +160,18 @@ class WebClient(Client, LazyLogging):
except Exception: except Exception:
self.logger.exception("could not add %s", package.base) self.logger.exception("could not add %s", package.base)
def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: def get(self, package_base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
""" """
get package status get package status
Args: Args:
base(Optional[str]): package base to get package_base(Optional[str]): package base to get
Returns: Returns:
List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found
""" """
try: try:
response = self.__session.get(self._package_url(base or "")) response = self.__session.get(self._package_url(package_base or ""))
response.raise_for_status() response.raise_for_status()
status_json = response.json() status_json = response.json()
@ -167,9 +180,9 @@ class WebClient(Client, LazyLogging):
for package in status_json for package in status_json
] ]
except requests.HTTPError as e: except requests.HTTPError as e:
self.logger.exception("could not get %s: %s", base, exception_response_text(e)) self.logger.exception("could not get %s: %s", package_base, exception_response_text(e))
except Exception: except Exception:
self.logger.exception("could not get %s", base) self.logger.exception("could not get %s", package_base)
return [] return []
def get_internal(self) -> InternalStatus: def get_internal(self) -> InternalStatus:
@ -191,38 +204,59 @@ class WebClient(Client, LazyLogging):
self.logger.exception("could not get web service status") self.logger.exception("could not get web service status")
return InternalStatus(status=BuildStatus()) return InternalStatus(status=BuildStatus())
def remove(self, base: str) -> None: def logs(self, record: logging.LogRecord) -> None:
"""
post log record
Args:
record(logging.LogRecord): log record to post to api
"""
package_base = getattr(record, "package_base", None)
if package_base is None:
return # in case if no package base supplised we need just skip log message
payload = {
"created": record.created,
"message": record.getMessage(),
"process_id": record.process,
}
# in this method exception has to be handled outside in logger handler
response = self.__session.post(self._logs_url(package_base), json=payload)
response.raise_for_status()
def remove(self, package_base: str) -> None:
""" """
remove packages from watcher remove packages from watcher
Args: Args:
base(str): basename to remove package_base(str): basename to remove
""" """
try: try:
response = self.__session.delete(self._package_url(base)) response = self.__session.delete(self._package_url(package_base))
response.raise_for_status() response.raise_for_status()
except requests.HTTPError as e: except requests.HTTPError as e:
self.logger.exception("could not delete %s: %s", base, exception_response_text(e)) self.logger.exception("could not delete %s: %s", package_base, exception_response_text(e))
except Exception: except Exception:
self.logger.exception("could not delete %s", base) self.logger.exception("could not delete %s", package_base)
def update(self, base: str, status: BuildStatusEnum) -> None: def update(self, package_base: str, status: BuildStatusEnum) -> None:
""" """
update package build status. Unlike ``add`` it does not update package properties update package build status. Unlike ``add`` it does not update package properties
Args: Args:
base(str): package base to update package_base(str): package base to update
status(BuildStatusEnum): current package build status status(BuildStatusEnum): current package build status
""" """
payload = {"status": status.value} payload = {"status": status.value}
try: try:
response = self.__session.post(self._package_url(base), json=payload) response = self.__session.post(self._package_url(package_base), json=payload)
response.raise_for_status() response.raise_for_status()
except requests.HTTPError as e: except requests.HTTPError as e:
self.logger.exception("could not update %s: %s", base, exception_response_text(e)) self.logger.exception("could not update %s: %s", package_base, exception_response_text(e))
except Exception: except Exception:
self.logger.exception("could not update %s", base) self.logger.exception("could not update %s", package_base)
def update_self(self, status: BuildStatusEnum) -> None: def update_self(self, status: BuildStatusEnum) -> None:
""" """

View File

@ -20,7 +20,7 @@
from typing import Iterable from typing import Iterable
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.result import Result from ahriman.models.result import Result

View File

@ -27,7 +27,7 @@ from typing import Generator, Iterable
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExtensionError from ahriman.core.exceptions import ExtensionError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.triggers import Trigger from ahriman.core.triggers import Trigger
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.result import Result from ahriman.models.result import Result

View File

@ -24,7 +24,7 @@ from typing import Iterable, Type
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SynchronizationError from ahriman.core.exceptions import SynchronizationError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.upload_settings import UploadSettings from ahriman.models.upload_settings import UploadSettings

View File

@ -44,7 +44,8 @@ def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optiona
Args: Args:
*args(str): command line arguments *args(str): command line arguments
exception(Optional[Exception]): exception which has to be reraised instead of default subprocess exception exception(Optional[Exception], optional): exception which has to be reraised instead of default subprocess
exception (Default value = None)
cwd(Optional[Path], optional): current working directory (Default value = None) cwd(Optional[Path], optional): current working directory (Default value = None)
input_data(Optional[str], optional): data which will be written to command stdin (Default value = None) input_data(Optional[str], optional): data which will be written to command stdin (Default value = None)
logger(Optional[Logger], optional): logger to log command result if required (Default value = None) logger(Optional[Logger], optional): logger to log command result if required (Default value = None)

View File

@ -0,0 +1,34 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
@dataclass(frozen=True)
class LogRecordId:
"""
log record process identifier
Attributes:
package_base(str): package base for which log record belongs
process_id(int): process id from which log record was emitted
"""
package_base: str
process_id: int

View File

@ -30,7 +30,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Type
from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb
from ahriman.core.exceptions import PackageInfoError from ahriman.core.exceptions import PackageInfoError
from ahriman.core.lazy_logging import LazyLogging from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, full_version from ahriman.core.util import check_output, full_version
from ahriman.models.package_description import PackageDescription from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource from ahriman.models.package_source import PackageSource
@ -218,7 +218,7 @@ class Package(LazyLogging):
Args: Args:
name(str): package name (either base or normal name) name(str): package name (either base or normal name)
pacman(Pacman): alpm wrapper instance pacman(Pacman): alpm wrapper instance
use_syncdb(bool): use pacman databases instead of official repositories RPC (Default value = True) use_syncdb(bool, optional): use pacman databases instead of official repositories RPC (Default value = True)
Returns: Returns:
Package: package properties Package: package properties
@ -365,7 +365,8 @@ class Package(LazyLogging):
Args: Args:
remote(Package): package properties from remote source remote(Package): package properties from remote source
paths(RepositoryPaths): repository paths instance. Required for VCS packages cache paths(RepositoryPaths): repository paths instance. Required for VCS packages cache
calculate_version(bool, optional): expand version to actual value (by calculating git versions) (Default value = True) calculate_version(bool, optional): expand version to actual value (by calculating git versions)
(Default value = True)
Returns: Returns:
bool: True if the package is out-of-dated and False otherwise bool: True if the package is out-of-dated and False otherwise

View File

@ -25,6 +25,7 @@ from ahriman.web.views.service.add import AddView
from ahriman.web.views.service.remove import RemoveView from ahriman.web.views.service.remove import RemoveView
from ahriman.web.views.service.request import RequestView from ahriman.web.views.service.request import RequestView
from ahriman.web.views.service.search import SearchView from ahriman.web.views.service.search import SearchView
from ahriman.web.views.status.logs import LogsView
from ahriman.web.views.status.package import PackageView from ahriman.web.views.status.package import PackageView
from ahriman.web.views.status.packages import PackagesView from ahriman.web.views.status.packages import PackagesView
from ahriman.web.views.status.status import StatusView from ahriman.web.views.status.status import StatusView
@ -61,6 +62,10 @@ def setup_routes(application: Application, static_path: Path) -> None:
* ``GET /api/v1/package/:base`` get package base status * ``GET /api/v1/package/:base`` get package base status
* ``POST /api/v1/package/:base`` update package base status * ``POST /api/v1/package/:base`` update package base status
* ``DELETE /api/v1/packages/{package}/logs`` delete package related logs
* ``GET /api/v1/packages/{package}/logs`` create log record for the package
* ``POST /api/v1/packages/{package}/logs`` get last package logs
* ``GET /api/v1/status`` get service status itself * ``GET /api/v1/status`` get service status itself
* ``POST /api/v1/status`` update service status itself * ``POST /api/v1/status`` update service status itself
@ -94,6 +99,10 @@ def setup_routes(application: Application, static_path: Path) -> None:
application.router.add_get("/api/v1/packages/{package}", PackageView, allow_head=True) application.router.add_get("/api/v1/packages/{package}", PackageView, allow_head=True)
application.router.add_post("/api/v1/packages/{package}", PackageView) application.router.add_post("/api/v1/packages/{package}", PackageView)
application.router.add_delete("/api/v1/packages/{package}/logs", LogsView)
application.router.add_get("/api/v1/packages/{package}/logs", LogsView, allow_head=True)
application.router.add_post("/api/v1/packages/{package}/logs", LogsView)
application.router.add_get("/api/v1/status", StatusView, allow_head=True) application.router.add_get("/api/v1/status", StatusView, allow_head=True)
application.router.add_post("/api/v1/status", StatusView) application.router.add_post("/api/v1/status", StatusView)

View File

@ -0,0 +1,105 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import HTTPBadRequest, HTTPNoContent, Response, json_response
from aiohttp.web_exceptions import HTTPNotFound
from ahriman.core.exceptions import UnknownPackageError
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.user_access import UserAccess
from ahriman.web.views.base import BaseView
class LogsView(BaseView):
"""
package logs web view
Attributes:
DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
HEAD_PERMISSION(UserAccess): (class attribute) head permissions of self
POST_PERMISSION(UserAccess): (class attribute) post permissions of self
"""
DELETE_PERMISSION = POST_PERMISSION = UserAccess.Full
GET_PERMISSION = HEAD_PERMISSION = UserAccess.Reporter
async def delete(self) -> None:
"""
delete package logs
Raises:
HTTPNoContent: on success response
"""
package_base = self.request.match_info["package"]
self.service.remove_logs(package_base, None)
raise HTTPNoContent()
async def get(self) -> Response:
"""
get last package logs
Returns:
Response: 200 with package logs on success
"""
package_base = self.request.match_info["package"]
try:
_, status = self.service.get(package_base)
except UnknownPackageError:
raise HTTPNotFound()
logs = self.service.get_logs(package_base)
response = {
"package_base": package_base,
"status": status.view(),
"logs": logs
}
return json_response(response)
async def post(self) -> None:
"""
create new package log record
JSON body must be supplied, the following model is used::
{
"created": 42.001, # log record created timestamp
"message": "log message", # log record
"process_id": 42 # process id from which log record was emitted
}
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
package_base = self.request.match_info["package"]
data = await self.extract_data()
try:
created = data["created"]
record = data["message"]
process_id = data["process_id"]
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.service.update_logs(LogRecordId(package_base, process_id), created, record)
raise HTTPNoContent()

View File

@ -40,6 +40,18 @@ class PackageView(BaseView):
DELETE_PERMISSION = POST_PERMISSION = UserAccess.Full DELETE_PERMISSION = POST_PERMISSION = UserAccess.Full
GET_PERMISSION = HEAD_PERMISSION = UserAccess.Read GET_PERMISSION = HEAD_PERMISSION = UserAccess.Read
async def delete(self) -> None:
"""
delete package base from status page
Raises:
HTTPNoContent: on success response
"""
package_base = self.request.match_info["package"]
self.service.remove(package_base)
raise HTTPNoContent()
async def get(self) -> Response: async def get(self) -> Response:
""" """
get current package base status get current package base status
@ -50,10 +62,10 @@ class PackageView(BaseView):
Raises: Raises:
HTTPNotFound: if no package was found HTTPNotFound: if no package was found
""" """
base = self.request.match_info["package"] package_base = self.request.match_info["package"]
try: try:
package, status = self.service.get(base) package, status = self.service.get(package_base)
except UnknownPackageError: except UnknownPackageError:
raise HTTPNotFound() raise HTTPNotFound()
@ -65,18 +77,6 @@ class PackageView(BaseView):
] ]
return json_response(response) return json_response(response)
async def delete(self) -> None:
"""
delete package base from status page
Raises:
HTTPNoContent: on success response
"""
base = self.request.match_info["package"]
self.service.remove(base)
raise HTTPNoContent()
async def post(self) -> None: async def post(self) -> None:
""" """
update package build status update package build status
@ -93,7 +93,7 @@ class PackageView(BaseView):
HTTPBadRequest: if bad data is supplied HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response HTTPNoContent: in case of success response
""" """
base = self.request.match_info["package"] package_base = self.request.match_info["package"]
data = await self.extract_data() data = await self.extract_data()
try: try:
@ -103,8 +103,8 @@ class PackageView(BaseView):
raise HTTPBadRequest(reason=str(e)) raise HTTPBadRequest(reason=str(e))
try: try:
self.service.update(base, status, package) self.service.update(package_base, status, package)
except UnknownPackageError: except UnknownPackageError:
raise HTTPBadRequest(reason=f"Package {base} is unknown, but no package body set") raise HTTPBadRequest(reason=f"Package {package_base} is unknown, but no package body set")
raise HTTPNoContent() raise HTTPNoContent()

View File

@ -27,6 +27,7 @@ from ahriman.core.auth import Auth
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.exceptions import InitializeError from ahriman.core.exceptions import InitializeError
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
from ahriman.core.spawn import Spawn from ahriman.core.spawn import Spawn
from ahriman.core.status.watcher import Watcher from ahriman.core.status.watcher import Watcher
from ahriman.web.middlewares.exception_handler import exception_handler from ahriman.web.middlewares.exception_handler import exception_handler
@ -79,7 +80,7 @@ def run_server(application: web.Application) -> None:
port = configuration.getint("web", "port") port = configuration.getint("web", "port")
web.run_app(application, host=host, port=port, handle_signals=False, web.run_app(application, host=host, port=port, handle_signals=False,
access_log=logging.getLogger("http")) access_log=logging.getLogger("http"), access_log_class=FilteredAccessLogger)
def setup_service(architecture: str, configuration: Configuration, spawner: Spawn) -> web.Application: def setup_service(architecture: str, configuration: Configuration, spawner: Spawn) -> web.Application:

View File

@ -51,18 +51,22 @@ def test_architectures_extract_specified(args: argparse.Namespace) -> None:
assert Handler.architectures_extract(args) == sorted(set(architectures)) assert Handler.architectures_extract(args) == sorted(set(architectures))
def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None: def test_call(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
""" """
must call inside lock must call inside lock
""" """
args.configuration = Path("") args.configuration = Path("")
args.quiet = False args.quiet = False
args.report = False
mocker.patch("ahriman.application.handlers.Handler.run") mocker.patch("ahriman.application.handlers.Handler.run")
mocker.patch("ahriman.core.configuration.Configuration.from_path") configuration_mock = mocker.patch("ahriman.core.configuration.Configuration.from_path", return_value=configuration)
log_load_mock = mocker.patch("ahriman.core.log.Log.load")
enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__") enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__")
exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__") exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__")
assert Handler.call(args, "x86_64") assert Handler.call(args, "x86_64")
configuration_mock.assert_called_once_with(args.configuration, "x86_64")
log_load_mock.assert_called_once_with(configuration, quiet=args.quiet, report=args.report)
enter_mock.assert_called_once_with() enter_mock.assert_called_once_with()
exit_mock.assert_called_once_with(None, None, None) exit_mock.assert_called_once_with(None, None, None)

View File

@ -120,7 +120,7 @@ def test_imply_with_report(args: argparse.Namespace, configuration: Configuratio
load_mock = mocker.patch("ahriman.core.status.client.Client.load") load_mock = mocker.patch("ahriman.core.status.client.Client.load")
Status.run(args, "x86_64", configuration, report=False, unsafe=False) Status.run(args, "x86_64", configuration, report=False, unsafe=False)
load_mock.assert_called_once_with(configuration) load_mock.assert_called_once_with(configuration, report=True)
def test_disallow_auto_architecture_run() -> None: def test_disallow_auto_architecture_run() -> None:

View File

@ -75,7 +75,7 @@ def test_imply_with_report(args: argparse.Namespace, configuration: Configuratio
load_mock = mocker.patch("ahriman.core.status.client.Client.load") load_mock = mocker.patch("ahriman.core.status.client.Client.load")
StatusUpdate.run(args, "x86_64", configuration, report=False, unsafe=False) StatusUpdate.run(args, "x86_64", configuration, report=False, unsafe=False)
load_mock.assert_called_once_with(configuration) load_mock.assert_called_once_with(configuration, report=True)
def test_disallow_auto_architecture_run() -> None: def test_disallow_auto_architecture_run() -> None:

View File

@ -215,7 +215,7 @@ def configuration(resource_path_root: Path) -> Configuration:
Configuration: configuration test instance Configuration: configuration test instance
""" """
path = resource_path_root / "core" / "ahriman.ini" path = resource_path_root / "core" / "ahriman.ini"
return Configuration.from_path(path=path, architecture="x86_64", quiet=False) return Configuration.from_path(path=path, architecture="x86_64")
@pytest.fixture @pytest.fixture

View File

@ -1,3 +1,4 @@
import logging
import pytest import pytest
from ahriman.core.alpm.repo import Repo from ahriman.core.alpm.repo import Repo
@ -36,6 +37,17 @@ def leaf_python_schedule(package_python_schedule: Package) -> Leaf:
return Leaf(package_python_schedule, set()) return Leaf(package_python_schedule, set())
@pytest.fixture
def log_record() -> logging.LogRecord:
"""
fixture for log record object
Returns:
logging.LogRecord: log record test instance
"""
return logging.LogRecord("record", logging.INFO, "path", 42, "log message", args=(), exc_info=None)
@pytest.fixture @pytest.fixture
def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Repo: def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Repo:
""" """

View File

@ -1,7 +1,7 @@
from ahriman.core.database.migrations.m002_user_access import steps from ahriman.core.database.migrations.m002_user_access import steps
def test_migration_package_source() -> None: def test_migration_user_access() -> None:
""" """
migration must not be empty migration must not be empty
""" """

View File

@ -1,7 +1,7 @@
from ahriman.core.database.migrations.m003_patch_variables import steps from ahriman.core.database.migrations.m003_patch_variables import steps
def test_migration_package_source() -> None: def test_migration_patches() -> None:
""" """
migration must not be empty migration must not be empty
""" """

View File

@ -0,0 +1,8 @@
from ahriman.core.database.migrations.m004_logs import steps
def test_migration_logs() -> None:
"""
migration must not be empty
"""
assert steps

View File

@ -35,7 +35,7 @@ def test_build_queue_insert_get(database: SQLite, package_ahriman: Package) -> N
def test_build_queue_insert(database: SQLite, package_ahriman: Package) -> None: def test_build_queue_insert(database: SQLite, package_ahriman: Package) -> None:
""" """
must update user in the database must update build queue in the database
""" """
database.build_queue_insert(package_ahriman) database.build_queue_insert(package_ahriman)
assert database.build_queue_get() == [package_ahriman] assert database.build_queue_get() == [package_ahriman]

View File

@ -0,0 +1,39 @@
from ahriman.core.database import SQLite
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
def test_logs_insert_remove_process(database: SQLite, package_ahriman: Package,
package_python_schedule: Package) -> None:
"""
must clear process specific package logs
"""
database.logs_insert(LogRecordId(package_ahriman.base, 1), 42.0, "message 1")
database.logs_insert(LogRecordId(package_ahriman.base, 2), 43.0, "message 2")
database.logs_insert(LogRecordId(package_python_schedule.base, 1), 42.0, "message 3")
database.logs_remove(package_ahriman.base, 1)
assert database.logs_get(package_ahriman.base) == "[1970-01-01 00:00:42] message 1"
assert database.logs_get(package_python_schedule.base) == "[1970-01-01 00:00:42] message 3"
def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must clear full package logs
"""
database.logs_insert(LogRecordId(package_ahriman.base, 1), 42.0, "message 1")
database.logs_insert(LogRecordId(package_ahriman.base, 2), 43.0, "message 2")
database.logs_insert(LogRecordId(package_python_schedule.base, 1), 42.0, "message 3")
database.logs_remove(package_ahriman.base, None)
assert not database.logs_get(package_ahriman.base)
assert database.logs_get(package_python_schedule.base) == "[1970-01-01 00:00:42] message 3"
def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None:
"""
must insert and get package logs
"""
database.logs_insert(LogRecordId(package_ahriman.base, 1), 43.0, "message 2")
database.logs_insert(LogRecordId(package_ahriman.base, 1), 42.0, "message 1")
assert database.logs_get(package_ahriman.base) == "[1970-01-01 00:00:42] message 1\n[1970-01-01 00:00:43] message 2"

View File

@ -0,0 +1,16 @@
import logging
import pytest
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
@pytest.fixture
def filtered_access_logger() -> FilteredAccessLogger:
"""
fixture for custom access logger
Returns:
FilteredAccessLogger: custom access logger test instance
"""
logger = logging.getLogger()
return FilteredAccessLogger(logger)

View File

@ -0,0 +1,71 @@
from pytest_mock import MockerFixture
from unittest.mock import MagicMock
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
def test_is_logs_post() -> None:
"""
must correctly define if request belongs to logs posting
"""
request = MagicMock()
request.method = "POST"
request.path = "/api/v1/packages/ahriman/logs"
assert FilteredAccessLogger.is_logs_post(request)
request.method = "POST"
request.path = "/api/v1/packages/linux-headers/logs"
assert FilteredAccessLogger.is_logs_post(request)
request.method = "POST"
request.path = "/api/v1/packages/memtest86+/logs"
assert FilteredAccessLogger.is_logs_post(request)
request.method = "POST"
request.path = "/api/v1/packages/memtest86%2B/logs"
assert FilteredAccessLogger.is_logs_post(request)
request.method = "POST"
request.path = "/api/v1/packages/python2.7/logs"
assert FilteredAccessLogger.is_logs_post(request)
request.method = "GET"
request.path = "/api/v1/packages/ahriman/logs"
assert not FilteredAccessLogger.is_logs_post(request)
request.method = "POST"
request.path = "/api/v1/packages/ahriman"
assert not FilteredAccessLogger.is_logs_post(request)
request.method = "POST"
request.path = "/api/v1/packages/ahriman/logs/random/path/after"
assert not FilteredAccessLogger.is_logs_post(request)
def test_log(filtered_access_logger: FilteredAccessLogger, mocker: MockerFixture) -> None:
"""
must emit log record
"""
request_mock = MagicMock()
response_mock = MagicMock()
is_log_path_mock = mocker.patch("ahriman.core.log.filtered_access_logger.FilteredAccessLogger.is_logs_post",
return_value=False)
log_mock = mocker.patch("aiohttp.web_log.AccessLogger.log")
filtered_access_logger.log(request_mock, response_mock, 0.001)
is_log_path_mock.assert_called_once_with(request_mock)
log_mock.assert_called_once_with(filtered_access_logger, request_mock, response_mock, 0.001)
def test_log_filter_logs(filtered_access_logger: FilteredAccessLogger, mocker: MockerFixture) -> None:
"""
must skip log record in case if it is from logs posting
"""
request_mock = MagicMock()
response_mock = MagicMock()
mocker.patch("ahriman.core.log.filtered_access_logger.FilteredAccessLogger.is_logs_post", return_value=True)
log_mock = mocker.patch("aiohttp.web_log.AccessLogger.log")
filtered_access_logger.log(request_mock, response_mock, 0.001)
log_mock.assert_not_called()

View File

@ -0,0 +1,56 @@
import logging
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.log.http_log_handler import HttpLogHandler
def test_load(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load handler
"""
# because of test cases we need to reset handler list
root = logging.getLogger()
current_handler = next((handler for handler in root.handlers if isinstance(handler, HttpLogHandler)), None)
root.removeHandler(current_handler)
add_mock = mocker.patch("logging.Logger.addHandler")
load_mock = mocker.patch("ahriman.core.status.client.Client.load")
handler = HttpLogHandler.load(configuration, report=False)
assert handler
add_mock.assert_called_once_with(handler)
load_mock.assert_called_once_with(configuration, report=False)
def test_load_exist(configuration: Configuration) -> None:
"""
must not load handler if already set
"""
handler = HttpLogHandler.load(configuration, report=False)
new_handler = HttpLogHandler.load(configuration, report=False)
assert handler is new_handler
def test_emit(configuration: Configuration, log_record: logging.LogRecord, mocker: MockerFixture) -> None:
"""
must emit log record to reporter
"""
log_mock = mocker.patch("ahriman.core.status.client.Client.logs")
handler = HttpLogHandler(configuration, report=False)
handler.emit(log_record)
log_mock.assert_called_once_with(log_record)
def test_emit_failed(configuration: Configuration, log_record: logging.LogRecord, mocker: MockerFixture) -> None:
"""
must call handle error on exception
"""
mocker.patch("ahriman.core.status.client.Client.logs", side_effect=Exception())
handle_error_mock = mocker.patch("logging.Handler.handleError")
handler = HttpLogHandler(configuration, report=False)
handler.emit(log_record)
handle_error_mock.assert_called_once_with(log_record)

View File

@ -0,0 +1,76 @@
import logging
import pytest
from pytest_mock import MockerFixture
from ahriman.core.alpm.repo import Repo
from ahriman.core.database import SQLite
from ahriman.models.package import Package
def test_logger(database: SQLite) -> None:
"""
must set logger attribute
"""
assert database.logger
assert database.logger.name == "ahriman.core.database.sqlite.SQLite"
def test_logger_attribute_error(database: SQLite) -> None:
"""
must raise AttributeError in case if no attribute found
"""
with pytest.raises(AttributeError):
database.loggerrrr
def test_logger_name(database: SQLite, repo: Repo) -> None:
"""
must correctly generate logger name
"""
assert database.logger_name == "ahriman.core.database.sqlite.SQLite"
assert repo.logger_name == "ahriman.core.alpm.repo.Repo"
def test_package_logger_set_reset(database: SQLite) -> None:
"""
must set and reset package base attribute
"""
package_base = "package base"
database._package_logger_set(package_base)
record = logging.makeLogRecord({})
assert record.package_base == package_base
database._package_logger_reset()
record = logging.makeLogRecord({})
with pytest.raises(AttributeError):
record.package_base
def test_in_package_context(database: SQLite, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must set package log context
"""
set_mock = mocker.patch("ahriman.core.log.LazyLogging._package_logger_set")
reset_mock = mocker.patch("ahriman.core.log.LazyLogging._package_logger_reset")
with database.in_package_context(package_ahriman.base):
pass
set_mock.assert_called_once_with(package_ahriman.base)
reset_mock.assert_called_once_with()
def test_in_package_context_failed(database: SQLite, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must reset package context even if exception occurs
"""
mocker.patch("ahriman.core.log.LazyLogging._package_logger_set")
reset_mock = mocker.patch("ahriman.core.log.LazyLogging._package_logger_reset")
with pytest.raises(Exception):
with database.in_package_context(package_ahriman.base):
raise Exception()
reset_mock.assert_called_once_with()

View File

@ -0,0 +1,35 @@
import logging
from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.log import Log
def test_load(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load logging
"""
logging_mock = mocker.patch("ahriman.core.log.log.fileConfig")
http_log_mock = mocker.patch("ahriman.core.log.http_log_handler.HttpLogHandler.load")
Log.load(configuration, quiet=False, report=False)
logging_mock.assert_called_once_with(configuration.logging_path)
http_log_mock.assert_called_once_with(configuration, report=False)
def test_load_fallback(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must fallback to stderr without errors
"""
mocker.patch("ahriman.core.log.log.fileConfig", side_effect=PermissionError())
Log.load(configuration, quiet=False, report=False)
def test_load_quiet(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must disable logging in case if quiet flag set
"""
disable_mock = mocker.patch("logging.disable")
Log.load(configuration, quiet=True, report=False)
disable_mock.assert_called_once_with(logging.WARNING)

View File

@ -63,6 +63,7 @@ def test_process_remove_base(executor: Executor, package_ahriman: Package, mocke
repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove")
build_queue_mock = mocker.patch("ahriman.core.database.SQLite.build_queue_clear") build_queue_mock = mocker.patch("ahriman.core.database.SQLite.build_queue_clear")
patches_mock = mocker.patch("ahriman.core.database.SQLite.patches_remove") patches_mock = mocker.patch("ahriman.core.database.SQLite.patches_remove")
logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_remove")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.remove") status_client_mock = mocker.patch("ahriman.core.status.client.Client.remove")
executor.process_remove([package_ahriman.base]) executor.process_remove([package_ahriman.base])
@ -73,6 +74,7 @@ def test_process_remove_base(executor: Executor, package_ahriman: Package, mocke
tree_clear_mock.assert_called_once_with(package_ahriman.base) tree_clear_mock.assert_called_once_with(package_ahriman.base)
build_queue_mock.assert_called_once_with(package_ahriman.base) build_queue_mock.assert_called_once_with(package_ahriman.base)
patches_mock.assert_called_once_with(package_ahriman.base, []) patches_mock.assert_called_once_with(package_ahriman.base, [])
logs_mock.assert_called_once_with(package_ahriman.base, None)
status_client_mock.assert_called_once_with(package_ahriman.base) status_client_mock.assert_called_once_with(package_ahriman.base)

View File

@ -4,7 +4,6 @@ from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnsafeRunError from ahriman.core.exceptions import UnsafeRunError
from ahriman.core.repository.repository_properties import RepositoryProperties from ahriman.core.repository.repository_properties import RepositoryProperties
from ahriman.core.status.web_client import WebClient
def test_create_tree_on_load(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None: def test_create_tree_on_load(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None:
@ -27,26 +26,3 @@ def test_create_tree_on_load_unsafe(configuration: Configuration, database: SQLi
RepositoryProperties("x86_64", configuration, database, report=False, unsafe=False) RepositoryProperties("x86_64", configuration, database, report=False, unsafe=False)
tree_create_mock.assert_not_called() tree_create_mock.assert_not_called()
def test_create_dummy_report_client(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None:
"""
must create dummy report client if report is disabled
"""
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
load_mock = mocker.patch("ahriman.core.status.client.Client.load")
properties = RepositoryProperties("x86_64", configuration, database, report=False, unsafe=False)
load_mock.assert_not_called()
assert not isinstance(properties.reporter, WebClient)
def test_create_full_report_client(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None:
"""
must create load report client if report is enabled
"""
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
load_mock = mocker.patch("ahriman.core.status.client.Client.load")
RepositoryProperties("x86_64", configuration, database, report=True, unsafe=True)
load_mock.assert_called_once_with(configuration)

View File

@ -1,5 +1,6 @@
import pytest import pytest
from pathlib import Path
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
from ahriman.core.repository.update_handler import UpdateHandler from ahriman.core.repository.update_handler import UpdateHandler
@ -103,15 +104,15 @@ def test_updates_local(update_handler: UpdateHandler, package_ahriman: Package,
must check for updates for locally stored packages must check for updates for locally stored packages
""" """
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman]) mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.base]) mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)])
mocker.patch("ahriman.models.package.Package.is_outdated", return_value=True) mocker.patch("ahriman.models.package.Package.is_outdated", return_value=True)
fetch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.fetch") fetch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.fetch")
package_load_mock = mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) package_load_mock = mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman)
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_pending") status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_pending")
assert update_handler.updates_local() == [package_ahriman] assert update_handler.updates_local() == [package_ahriman]
fetch_mock.assert_called_once_with(package_ahriman.base, remote=None) fetch_mock.assert_called_once_with(Path(package_ahriman.base), remote=None)
package_load_mock.assert_called_once_with(package_ahriman.base) package_load_mock.assert_called_once_with(Path(package_ahriman.base))
status_client_mock.assert_called_once_with(package_ahriman.base) status_client_mock.assert_called_once_with(package_ahriman.base)
@ -120,7 +121,7 @@ def test_updates_local_unknown(update_handler: UpdateHandler, package_ahriman: P
must return unknown package as out-dated must return unknown package as out-dated
""" """
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[]) mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[])
mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.base]) mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)])
mocker.patch("ahriman.models.package.Package.is_outdated", return_value=True) mocker.patch("ahriman.models.package.Package.is_outdated", return_value=True)
mocker.patch("ahriman.core.build_tools.sources.Sources.fetch") mocker.patch("ahriman.core.build_tools.sources.Sources.fetch")
mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman)
@ -136,7 +137,7 @@ def test_updates_local_with_failures(update_handler: UpdateHandler, package_ahri
must process local through the packages with failure must process local through the packages with failure
""" """
mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages") mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages")
mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.base]) mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)])
mocker.patch("ahriman.core.build_tools.sources.Sources.fetch", side_effect=Exception()) mocker.patch("ahriman.core.build_tools.sources.Sources.fetch", side_effect=Exception())
assert not update_handler.updates_local() assert not update_handler.updates_local()

View File

@ -1,3 +1,5 @@
import logging
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
@ -12,7 +14,16 @@ def test_load_dummy_client(configuration: Configuration) -> None:
""" """
must load dummy client if no settings set must load dummy client if no settings set
""" """
assert isinstance(Client.load(configuration), Client) assert not isinstance(Client.load(configuration, report=True), WebClient)
def test_load_dummy_client_disabled(configuration: Configuration) -> None:
"""
must load dummy client if report is set to False
"""
configuration.set_option("web", "host", "localhost")
configuration.set_option("web", "port", "8080")
assert not isinstance(Client.load(configuration, report=False), WebClient)
def test_load_full_client(configuration: Configuration) -> None: def test_load_full_client(configuration: Configuration) -> None:
@ -21,7 +32,7 @@ def test_load_full_client(configuration: Configuration) -> None:
""" """
configuration.set_option("web", "host", "localhost") configuration.set_option("web", "host", "localhost")
configuration.set_option("web", "port", "8080") configuration.set_option("web", "port", "8080")
assert isinstance(Client.load(configuration), WebClient) assert isinstance(Client.load(configuration, report=True), WebClient)
def test_load_full_client_from_address(configuration: Configuration) -> None: def test_load_full_client_from_address(configuration: Configuration) -> None:
@ -29,7 +40,7 @@ def test_load_full_client_from_address(configuration: Configuration) -> None:
must load full client by using address must load full client by using address
""" """
configuration.set_option("web", "address", "http://localhost:8080") configuration.set_option("web", "address", "http://localhost:8080")
assert isinstance(Client.load(configuration), WebClient) assert isinstance(Client.load(configuration, report=True), WebClient)
def test_add(client: Client, package_ahriman: Package) -> None: def test_add(client: Client, package_ahriman: Package) -> None:
@ -57,6 +68,13 @@ def test_get_internal(client: Client) -> None:
assert actual == expected assert actual == expected
def test_log(client: Client, log_record: logging.LogRecord) -> None:
"""
must process log record without errors
"""
client.logs(log_record)
def test_remove(client: Client, package_ahriman: Package) -> None: def test_remove(client: Client, package_ahriman: Package) -> None:
""" """
must process remove without errors must process remove without errors

View File

@ -8,6 +8,7 @@ from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.status.watcher import Watcher from ahriman.core.status.watcher import Watcher
from ahriman.core.status.web_client import WebClient from ahriman.core.status.web_client import WebClient
from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
@ -18,10 +19,7 @@ def test_force_no_report(configuration: Configuration, database: SQLite, mocker:
configuration.set_option("web", "port", "8080") configuration.set_option("web", "port", "8080")
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
load_mock = mocker.patch("ahriman.core.status.client.Client.load")
watcher = Watcher("x86_64", configuration, database) watcher = Watcher("x86_64", configuration, database)
load_mock.assert_not_called()
assert not isinstance(watcher.repository.reporter, WebClient) assert not isinstance(watcher.repository.reporter, WebClient)
@ -43,6 +41,15 @@ def test_get_failed(watcher: Watcher, package_ahriman: Package) -> None:
watcher.get(package_ahriman.base) watcher.get(package_ahriman.base)
def test_get_logs(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must return package logs
"""
logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_get")
watcher.get_logs(package_ahriman.base)
logs_mock.assert_called_once_with(package_ahriman.base)
def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
""" """
must correctly load packages must correctly load packages
@ -76,11 +83,22 @@ def test_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixtur
must remove package base must remove package base
""" """
cache_mock = mocker.patch("ahriman.core.database.SQLite.package_remove") cache_mock = mocker.patch("ahriman.core.database.SQLite.package_remove")
logs_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs")
watcher.known = {package_ahriman.base: (package_ahriman, BuildStatus())} watcher.known = {package_ahriman.base: (package_ahriman, BuildStatus())}
watcher.remove(package_ahriman.base) watcher.remove(package_ahriman.base)
assert not watcher.known assert not watcher.known
cache_mock.assert_called_once_with(package_ahriman.base) cache_mock.assert_called_once_with(package_ahriman.base)
logs_mock.assert_called_once_with(package_ahriman.base, None)
def test_remove_logs(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must remove package logs
"""
logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_remove")
watcher.remove_logs(package_ahriman.base, 42)
logs_mock.assert_called_once_with(package_ahriman.base, 42)
def test_remove_unknown(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: def test_remove_unknown(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
@ -128,6 +146,38 @@ def test_update_unknown(watcher: Watcher, package_ahriman: Package) -> None:
watcher.update(package_ahriman.base, BuildStatusEnum.Unknown, None) watcher.update(package_ahriman.base, BuildStatusEnum.Unknown, None)
def test_update_logs_new(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must create package logs record for new package
"""
delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs")
insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert")
log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id)
assert watcher._last_log_record_id != log_record_id
watcher.update_logs(log_record_id, 42.01, "log record")
delete_mock.assert_called_once_with(package_ahriman.base, log_record_id.process_id)
insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
assert watcher._last_log_record_id == log_record_id
def test_update_logs_update(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must create package logs record for current package
"""
delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs")
insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert")
log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id)
watcher._last_log_record_id = log_record_id
watcher.update_logs(log_record_id, 42.01, "log record")
delete_mock.assert_not_called()
insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
def test_update_self(watcher: Watcher) -> None: def test_update_self(watcher: Watcher) -> None:
""" """
must update service status must update service status

View File

@ -1,4 +1,5 @@
import json import json
import logging
import pytest import pytest
import requests import requests
@ -13,6 +14,14 @@ from ahriman.models.package import Package
from ahriman.models.user import User from ahriman.models.user import User
def test_status_url(web_client: WebClient) -> None:
"""
must generate login url correctly
"""
assert web_client._login_url.startswith(web_client.address)
assert web_client._login_url.endswith("/api/v1/login")
def test_status_url(web_client: WebClient) -> None: def test_status_url(web_client: WebClient) -> None:
""" """
must generate package status url correctly must generate package status url correctly
@ -75,9 +84,17 @@ def test_login_skip(web_client: WebClient, mocker: MockerFixture) -> None:
requests_mock.assert_not_called() requests_mock.assert_not_called()
def test_logs_url(web_client: WebClient, package_ahriman: Package) -> None:
"""
must generate logs url correctly
"""
assert web_client._logs_url(package_ahriman.base).startswith(web_client.address)
assert web_client._logs_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}/logs")
def test_package_url(web_client: WebClient, package_ahriman: Package) -> None: def test_package_url(web_client: WebClient, package_ahriman: Package) -> None:
""" """
must generate package status correctly must generate package status url correctly
""" """
assert web_client._package_url(package_ahriman.base).startswith(web_client.address) assert web_client._package_url(package_ahriman.base).startswith(web_client.address)
assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}") assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}")
@ -192,6 +209,43 @@ def test_get_internal_failed_http_error(web_client: WebClient, mocker: MockerFix
assert web_client.get_internal().architecture is None assert web_client.get_internal().architecture is None
def test_logs(web_client: WebClient, log_record: logging.LogRecord, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must process log record
"""
requests_mock = mocker.patch("requests.Session.post")
log_record.package_base = package_ahriman.base
payload = {
"created": log_record.created,
"message": log_record.getMessage(),
"process_id": log_record.process,
}
web_client.logs(log_record)
requests_mock.assert_called_once_with(pytest.helpers.anyvar(str, True), json=payload)
def test_log_failed(web_client: WebClient, log_record: logging.LogRecord, package_ahriman: Package,
mocker: MockerFixture) -> None:
"""
must pass exception during log post
"""
mocker.patch("requests.Session.post", side_effect=Exception())
log_record.package_base = package_ahriman.base
with pytest.raises(Exception):
web_client.logs(log_record)
def test_log_skip(web_client: WebClient, log_record: logging.LogRecord, mocker: MockerFixture) -> None:
"""
must skip log record posting if no package base set
"""
requests_mock = mocker.patch("requests.Session.post")
web_client.logs(log_record)
requests_mock.assert_not_called()
def test_remove(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: def test_remove(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
""" """
must process package removal must process package removal

View File

@ -1,5 +1,4 @@
import configparser import configparser
import logging
import pytest import pytest
from pathlib import Path from pathlib import Path
@ -25,14 +24,12 @@ def test_from_path(mocker: MockerFixture) -> None:
mocker.patch("pathlib.Path.is_file", return_value=True) mocker.patch("pathlib.Path.is_file", return_value=True)
read_mock = mocker.patch("ahriman.core.configuration.Configuration.read") read_mock = mocker.patch("ahriman.core.configuration.Configuration.read")
load_includes_mock = mocker.patch("ahriman.core.configuration.Configuration.load_includes") load_includes_mock = mocker.patch("ahriman.core.configuration.Configuration.load_includes")
load_logging_mock = mocker.patch("ahriman.core.configuration.Configuration.load_logging")
path = Path("path") path = Path("path")
configuration = Configuration.from_path(path, "x86_64", True) configuration = Configuration.from_path(path, "x86_64")
assert configuration.path == path assert configuration.path == path
read_mock.assert_called_once_with(path) read_mock.assert_called_once_with(path)
load_includes_mock.assert_called_once_with() load_includes_mock.assert_called_once_with()
load_logging_mock.assert_called_once_with(True)
def test_from_path_file_missing(mocker: MockerFixture) -> None: def test_from_path_file_missing(mocker: MockerFixture) -> None:
@ -41,10 +38,9 @@ def test_from_path_file_missing(mocker: MockerFixture) -> None:
""" """
mocker.patch("pathlib.Path.is_file", return_value=False) mocker.patch("pathlib.Path.is_file", return_value=False)
mocker.patch("ahriman.core.configuration.Configuration.load_includes") mocker.patch("ahriman.core.configuration.Configuration.load_includes")
mocker.patch("ahriman.core.configuration.Configuration.load_logging")
read_mock = mocker.patch("ahriman.core.configuration.Configuration.read") read_mock = mocker.patch("ahriman.core.configuration.Configuration.read")
configuration = Configuration.from_path(Path("path"), "x86_64", True) configuration = Configuration.from_path(Path("path"), "x86_64")
read_mock.assert_called_once_with(configuration.SYSTEM_CONFIGURATION_PATH) read_mock.assert_called_once_with(configuration.SYSTEM_CONFIGURATION_PATH)
@ -263,23 +259,6 @@ def test_load_includes_no_section(configuration: Configuration) -> None:
configuration.load_includes() configuration.load_includes()
def test_load_logging_fallback(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must fallback to stderr without errors
"""
mocker.patch("ahriman.core.configuration.fileConfig", side_effect=PermissionError())
configuration.load_logging(quiet=False)
def test_load_logging_quiet(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must disable logging in case if quiet flag set
"""
disable_mock = mocker.patch("logging.disable")
configuration.load_logging(quiet=True)
disable_mock.assert_called_once_with(logging.WARNING)
def test_merge_sections_missing(configuration: Configuration) -> None: def test_merge_sections_missing(configuration: Configuration) -> None:
""" """
must merge create section if not exists must merge create section if not exists

View File

@ -1,28 +0,0 @@
import pytest
from ahriman.core.alpm.repo import Repo
from ahriman.core.database import SQLite
def test_logger(database: SQLite) -> None:
"""
must set logger attribute
"""
assert database.logger
assert database.logger.name == "ahriman.core.database.sqlite.SQLite"
def test_logger_attribute_error(database: SQLite) -> None:
"""
must raise AttributeError in case if no attribute found
"""
with pytest.raises(AttributeError):
database.loggerrrr
def test_logger_name(database: SQLite, repo: Repo) -> None:
"""
must correctly generate logger name
"""
assert database.logger_name == "ahriman.core.database.sqlite.SQLite"
assert repo.logger_name == "ahriman.core.alpm.repo.Repo"

View File

@ -327,6 +327,7 @@ def test_walk(resource_path_root: Path) -> None:
resource_path_root / "web" / "templates" / "build-status" / "failed-modal.jinja2", resource_path_root / "web" / "templates" / "build-status" / "failed-modal.jinja2",
resource_path_root / "web" / "templates" / "build-status" / "login-modal.jinja2", resource_path_root / "web" / "templates" / "build-status" / "login-modal.jinja2",
resource_path_root / "web" / "templates" / "build-status" / "package-add-modal.jinja2", resource_path_root / "web" / "templates" / "build-status" / "package-add-modal.jinja2",
resource_path_root / "web" / "templates" / "build-status" / "package-info-modal.jinja2",
resource_path_root / "web" / "templates" / "build-status" / "success-modal.jinja2", resource_path_root / "web" / "templates" / "build-status" / "success-modal.jinja2",
resource_path_root / "web" / "templates" / "build-status" / "table.jinja2", resource_path_root / "web" / "templates" / "build-status" / "table.jinja2",
resource_path_root / "web" / "templates" / "static" / "favicon.ico", resource_path_root / "web" / "templates" / "static" / "favicon.ico",

View File

@ -4,6 +4,7 @@ from aiohttp import web
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
from ahriman.core.exceptions import InitializeError from ahriman.core.exceptions import InitializeError
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
from ahriman.core.status.watcher import Watcher from ahriman.core.status.watcher import Watcher
from ahriman.web.web import on_shutdown, on_startup, run_server from ahriman.web.web import on_shutdown, on_startup, run_server
@ -48,8 +49,10 @@ def test_run(application: web.Application, mocker: MockerFixture) -> None:
run_application_mock = mocker.patch("aiohttp.web.run_app") run_application_mock = mocker.patch("aiohttp.web.run_app")
run_server(application) run_server(application)
run_application_mock.assert_called_once_with(application, host="127.0.0.1", port=port, run_application_mock.assert_called_once_with(
handle_signals=False, access_log=pytest.helpers.anyvar(int)) application, host="127.0.0.1", port=port, handle_signals=False,
access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger
)
def test_run_with_auth(application_with_auth: web.Application, mocker: MockerFixture) -> None: def test_run_with_auth(application_with_auth: web.Application, mocker: MockerFixture) -> None:
@ -61,8 +64,10 @@ def test_run_with_auth(application_with_auth: web.Application, mocker: MockerFix
run_application_mock = mocker.patch("aiohttp.web.run_app") run_application_mock = mocker.patch("aiohttp.web.run_app")
run_server(application_with_auth) run_server(application_with_auth)
run_application_mock.assert_called_once_with(application_with_auth, host="127.0.0.1", port=port, run_application_mock.assert_called_once_with(
handle_signals=False, access_log=pytest.helpers.anyvar(int)) application_with_auth, host="127.0.0.1", port=port, handle_signals=False,
access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger
)
def test_run_with_debug(application_with_debug: web.Application, mocker: MockerFixture) -> None: def test_run_with_debug(application_with_debug: web.Application, mocker: MockerFixture) -> None:
@ -74,5 +79,7 @@ def test_run_with_debug(application_with_debug: web.Application, mocker: MockerF
run_application_mock = mocker.patch("aiohttp.web.run_app") run_application_mock = mocker.patch("aiohttp.web.run_app")
run_server(application_with_debug) run_server(application_with_debug)
run_application_mock.assert_called_once_with(application_with_debug, host="127.0.0.1", port=port, run_application_mock.assert_called_once_with(
handle_signals=False, access_log=pytest.helpers.anyvar(int)) application_with_debug, host="127.0.0.1", port=port, handle_signals=False,
access_log=pytest.helpers.anyvar(int), access_log_class=FilteredAccessLogger
)

View File

@ -0,0 +1,94 @@
import pytest
from aiohttp.test_utils import TestClient
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.user_access import UserAccess
from ahriman.web.views.status.logs import LogsView
async def test_get_permission() -> None:
"""
must return correct permission for the request
"""
for method in ("GET", "HEAD"):
request = pytest.helpers.request("", "", method)
assert await LogsView.get_permission(request) == UserAccess.Reporter
for method in ("DELETE", "POST"):
request = pytest.helpers.request("", "", method)
assert await LogsView.get_permission(request) == UserAccess.Full
async def test_delete(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must delete logs for package
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
await client.post(f"/api/v1/packages/{package_python_schedule.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()})
await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
json={"created": 42.0, "message": "message", "process_id": 42})
await client.post(f"/api/v1/packages/{package_python_schedule.base}/logs",
json={"created": 42.0, "message": "message", "process_id": 42})
response = await client.delete(f"/api/v1/packages/{package_ahriman.base}/logs")
assert response.status == 204
response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs")
logs = await response.json()
assert not logs["logs"]
response = await client.get(f"/api/v1/packages/{package_python_schedule.base}/logs")
logs = await response.json()
assert logs["logs"]
async def test_get(client: TestClient, package_ahriman: Package) -> None:
"""
must get logs for package
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
json={"created": 42.0, "message": "message", "process_id": 42})
response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs")
assert response.status == 200
logs = await response.json()
assert logs["logs"] == "[1970-01-01 00:00:42] message"
async def test_get_not_foud(client: TestClient, package_ahriman: Package) -> None:
"""
must return not found for missing package
"""
response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs")
assert response.status == 404
async def test_post(client: TestClient, package_ahriman: Package) -> None:
"""
must create logs record
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
post_response = await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
json={"created": 42.0, "message": "message", "process_id": 42})
assert post_response.status == 204
response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs")
logs = await response.json()
assert logs["logs"] == "[1970-01-01 00:00:42] message"
async def test_post_exception(client: TestClient, package_ahriman: Package) -> None:
"""
must raise exception on invalid payload
"""
post_response = await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", json={})
assert post_response.status == 400

View File

@ -20,31 +20,6 @@ async def test_get_permission() -> None:
assert await PackageView.get_permission(request) == UserAccess.Full assert await PackageView.get_permission(request) == UserAccess.Full
async def test_get(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must return status for specific package
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
await client.post(f"/api/v1/packages/{package_python_schedule.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()})
response = await client.get(f"/api/v1/packages/{package_ahriman.base}")
assert response.ok
packages = [Package.from_json(item["package"]) for item in await response.json()]
assert packages
assert {package.base for package in packages} == {package_ahriman.base}
async def test_get_not_found(client: TestClient, package_ahriman: Package) -> None:
"""
must return Not Found for unknown package
"""
response = await client.get(f"/api/v1/packages/{package_ahriman.base}")
assert response.status == 404
async def test_delete(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None: async def test_delete(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None:
""" """
must delete single base must delete single base
@ -81,6 +56,31 @@ async def test_delete_unknown(client: TestClient, package_ahriman: Package, pack
assert response.ok assert response.ok
async def test_get(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must return status for specific package
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
await client.post(f"/api/v1/packages/{package_python_schedule.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()})
response = await client.get(f"/api/v1/packages/{package_ahriman.base}")
assert response.ok
packages = [Package.from_json(item["package"]) for item in await response.json()]
assert packages
assert {package.base for package in packages} == {package_ahriman.base}
async def test_get_not_found(client: TestClient, package_ahriman: Package) -> None:
"""
must return Not Found for unknown package
"""
response = await client.get(f"/api/v1/packages/{package_ahriman.base}")
assert response.status == 404
async def test_post(client: TestClient, package_ahriman: Package) -> None: async def test_post(client: TestClient, package_ahriman: Package) -> None:
""" """
must update package status must update package status