Skip to content

GitLab

Commit dfa03469 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Release 'supernumerarycolumns'

parents 078faddb bcc165fc
Hide whitespace changes
Inline Side-by-side
Showing with 340 additions and 156 deletions
CHANGELOG.md
View file @ dfa03469
# 0.3.0
- Repair tabular file structure prior to look for value errors
# 0.2.22
- Fix crash on home page if non existant repo
......
requirements.txt
View file @ dfa03469
......@@ -9,4 +9,4 @@ python-dotenv==0.10.1
requests==2.22.0
toml==0.10.0
tabulator==1.21.0
validata_core==0.4.1
validata_core==0.5.0
setup.py
View file @ dfa03469
......@@ -13,7 +13,7 @@ with readme_filepath.open('rt', encoding='utf-8') as fd:
setup(
name='validata_ui',
version='0.2.22',
version='0.3.0',
description='Validata Web UI',
long_description=LONG_DESCRIPTION,
......@@ -70,7 +70,7 @@ setup(
'tabulator',
'opendataschema >= 0.5.5, < 0.6',
'validata_core >= 0.4.1, < 0.5',
'validata_core >= 0.5.0, < 0.6',
],
)
validata_ui/static/validata.css
View file @ dfa03469
......@@ -21,3 +21,28 @@ body {
bottom: 0;
width: 100%;
}
/* Pop-over management */
.popover .popover-body h2 {
font-size: 1.2em;
font-weight: bold;
}
/* CSS trick to allow th to be styled with danger, warning, ...
even if already using class thead-light class */
.table .thead-light th.table-danger {
background-color: #f5c6cb;
}
.table .thead-light th.table-warning {
background-color: #ffeeba;
}
/* Remove margin bottom for repair action items */
div.actions ul li p {
margin-bottom: 0;
}
/* Force align=center for <th> */
body th {
text-align: center;
}
validata_ui/static/validata_popover.js 0 → 100644
View file @ dfa03469
// Popovers that stays while on popover bubble
// https://stackoverflow.com/questions/15989591/how-can-i-keep-bootstrap-popover-alive-while-the-popover-is-being-hovered
$('[data-toggle="popover"]')
.popover({
trigger: "manual",
html: true,
animation: false,
placement: "auto"
})
.on("mouseenter", function() {
var _this = this
$(this).popover("show")
$(".popover").on("mouseleave", function() {
$(_this).popover("hide")
})
})
.on("mouseleave", function() {
var _this = this
setTimeout(function() {
if (!$(".popover:hover").length) {
$(_this).popover("hide")
}
}, 100)
})
// Open links found in error description in a new window/tab
$("body").on("mouseover", "div.popover a", function() {
$(this).prop("title", "Ouvrir dans une nouvelle fenêtre")
})
$("body").on("click", "div.popover a", function() {
var url = $(this).prop("href")
window.open(url, "_blank")
return false
})
validata_ui/templates/validation_macros.html
View file @ dfa03469
{% macro preview(source_data) %}
<p>
Prévisualisation de {{ source_data.preview_rows_nb }} ligne{% if source_data.preview_rows_nb > 1 %}s{% endif %} sur {{ source_data.rows_nb }} au total :
</p>
<div class="table-responsive">
<table class="table table-striped table-bordered table-sm table-hover">
{% macro thead(report, with_lineno_col=False) %}
<thead class="thead-light">
<tr>
{% for col in source_data.header %}
<th scope="col">{{ col }}</th>
{% endfor %}
{% if with_lineno_col %}
<th scope="col">1</th>
{% endif %}
{% for h in report.table.headers %}
<th
scope="col"
{% if report.table.cols_alert[loop.index - 1] != "" %}class="{{ report.table.cols_alert[loop.index - 1] }}"{% endif %}
data-toggle="popover"
title="{{ report.table.headers_title[loop.index - 1]}}"
data-content="{{ report.table.headers_description[loop.index - 1] | commonmark2html | escape}}"
>{{ h }}</th>
{% endfor %}
</tr>
</thead>
{% endmacro %}
{% macro preview(report, source_data) %}
<div class="table-responsive-sm">
<table class="table-striped table-bordered table-sm table-hover">
{{ thead(report) }}
<tbody>
{% for row in source_data.preview_rows %}
<tr>
{% for val in row %}
<td>{{ val }}</td>
<td
{% if report.table.cols_alert[loop.index - 1] != "" %}class="{{ report.table.cols_alert[loop.index - 1] }}"{% endif %}
>{{ val }}</td>
{% endfor %}
</tr>
{% endfor %}
......@@ -70,13 +82,7 @@
{% macro body_errors_screen(report, source_data) %}
<div class="table-responsive-sm">
<table class="table-sm table-bordered table-striped table-hover">
<thead class="thead-light">
<th scope="col">1</th>
{% for h in source_data.header %}
<th scope="col" data-toggle="popover" title="{{ report.table.headers_title[loop.index - 1]}}" data-content="{{ report.table.headers_description[loop.index - 1] | commonmark2html | escape}}">{{
h }}</th>
{% endfor %}
</thead>
{{ thead(report, True) }}
<tbody>
{% for row in report.table.errors.body_by_rows %}
<tr>
......@@ -114,7 +120,7 @@
{% set structure_errors = report.table['error-stats']['structure-errors'] %}
{% if structure_errors['count'] != 0 %}
<p>Erreur de structure ({{ structure_errors['count'] }}) :</p>
<p>Erreurs de structure ({{ structure_errors['count'] }}) :</p>
<ul>
{% for item in structure_errors['count-by-code'] %}
<li>{{ item[0] }} ({{ item[1] }})</li>
......@@ -124,17 +130,16 @@
<p>Aucune erreur de structure.</p>
{% endif %}
{% if report.table.do_display_body_errors %}
{% set value_errors = report.table['error-stats']['value-errors'] %}
{% if value_errors['count'] > 0 %}
<p>Erreur de contenu ({{ value_errors['count'] }} sur {{ value_errors['rows-count'] }} ligne{% if value_errors['rows-count'] > 1 %}s{% endif %}) :</p>
<p>Erreurs de contenu ({{ value_errors['count'] }} sur {{ value_errors['rows-count'] }} ligne{% if value_errors['rows-count'] > 1 %}s{% endif %}) :</p>
<ul>
{% for item in value_errors['count-by-code'] %}
<li>{{ item[0] }} ({{ item[1] }})</li>
{% endfor %}
</ul>
{% endif %}
{% else %}
<p>Aucune erreur de contenu</p>
{% endif %}
{% endmacro %}
validata_ui/templates/validation_report.html
View file @ dfa03469
......@@ -8,14 +8,13 @@
{% block head %}
{{ super() }}
<style>
.popover .popover-body h2 {
font-size: 1.2em;
font-weight: bold;
}
#table-errors td h2 {
font-size: 1em;
font-weight: bold;
}
ul.structure_errors li p {
margin-bottom: 0;
}
@media print {
.hidden-print {
......@@ -38,8 +37,8 @@
</p>
{% endif %}
{# Schema info #}
<div class="row my-4">
{# Schema info #}
<div class="col-lg-6 mb-4">
<div class="card">
<div class="card-body">
......@@ -47,6 +46,7 @@
</div>
</div>
</div>
{# Validation stats #}
<div class="col-lg-6">
<div class="card">
<div class="card-body">
......@@ -87,104 +87,90 @@
</div>
</div>
</div>
</div>
{% if report.error_count > 0 %}
{% if report.table.errors.structure %}
<h3 class="my-4">Erreurs de structure</h3>
{# Non-column errors #}
{% for err in report.table.errors.structure %}
{% if not err.in_column_comp %}
{{ err.content | safe }}
{% endif %}
{% endfor %}
{# Column errors #}
{% for err in report.table.errors.structure %}
{% if err.in_column_comp %}
{{ err.content | safe }}
{% endif %}
{% endfor %}
{# Afficher un tableau de comparaison des colonnes #}
{% if report.table.column_comparison_needed %}
<table class="table table-bordered table-sm table-striped table-hover">
<thead class="thead-light">
<tr>
<th scope="col">Colonnes du fichier</th>
<th scope="col">Colonnes attendues</th>
</tr>
</thead>
<tbody>
{% for elt in report.table.column_comparison_info.table %}
<tr{% if elt[2] == 'ko' %} class="table-danger"{% endif %}>
<td>{{ elt[0] }}</td>
<td>{{ elt[1] }}</td>
</tr>
<div class="mx-4">
{% if report.table.errors.structure %}
<h3 class="my-4">Erreurs de structure ({{ report.table['error-stats']['structure-errors']['count'] }})</h3>
<p>Prévisualisation du fichier source avec structure invalide (en-têtes)</p>
{# source header display #}
<div class="mb-4">
<div class="table-responsive">
<table class="table table-striped table-bordered table-sm table-hover">
<thead class="thead-light">
<tr>
{% for h, err in source_data.source_header_info %}
<th{% if err %} class="table-danger"{% endif %}>{{ h }}</th>
{% endfor %}
</tbody>
</table>
{% if report.table.column_comparison_info.has_missing %}
<p>Attention : toutes les colonnes attendues doivent être présentes dans le fichier à valider.<p>
{% endif %}
{% if report.table.column_comparison_info.has_case_errors %}
<p>Attention : la casse (minuscules/majuscules) doit être respectée dans les noms de colonne.<p>
{% endif %}
{% endif %}
{% endif %}
</tr>
</thead>
</table>
</div>
</div>
{% endif %}
</div>
{% if report.table.do_display_body_errors %} {# We do display body errors! #}
<div class="mx-4">
{% if report.table.errors.body %}
<h3 class="my-4">Erreurs de contenu</h3>
{{ macros.body_errors(report, source_data, print_mode) }}
{# Display body errors #}
<div class="mx-4">
<h3 class="my-4">
{% if report.table.errors.body %}
{% set value_errors = report.table['error-stats']['value-errors'] %}
Erreurs de contenu ({{ value_errors['count'] }} sur {{ value_errors['rows-count'] }} ligne{% if value_errors['rows-count'] > 1 %}s{% endif %})
{% else %}
{% if report.table.errors.structure %}
Aucune erreur de contenu
{% else %}
{{ macros.preview(source_data) }}
Aucune erreur détectée
{% endif %}
{% endif %}
</h3>
<p>Prévisualisation du fichier source
{% if report.repair_actions %}avec structure réparée{% endif %}
{% if not report.table.errors.body %}
(affichage de {{ source_data.preview_rows_nb }}
ligne{% if source_data.preview_rows_nb > 1 %}s{% endif %}
sur {{ source_data.rows_nb }} au total)
{% endif %}
</p>
{% if report.repair_actions %}
<div id="repair_actions" class="my-4 actions">
<div class="col-lg-6 alert alert-warning">
<p>
La structure de votre fichier a été automatiquement remaniée
<a id="repair_details_link" class="float-right" href="#" title="Voir les détails">détails</a>
</p>
<div id="repair_details_panel" class="d-none">
<ul>
{% for action in report.repair_actions %}
<li>{{ action | commonmark2html | safe}}</li>
{% endfor %}
</ul>
<p>
Il vous appartient d'apporter ces modifications sur votre fichier pour rendre sa structure
valide selon le schéma.
</p>
</div>
</div>
</div>
{% else %} {# do not display errors #}
<p>Veuillez corriger ces erreurs pour visualiser les éventuelles erreurs de contenu.</p>
{% endif %}
{% endif %}
{% if report.table.errors.body %}
{{ macros.body_errors(report, source_data, print_mode) }}
{% else %}
{{ macros.preview(report, source_data) }}
{% endif %}
</div>
{% endblock %}
{% block page_scripts %}
<script src="{{ url_for('static', filename='validata_popover.js') }}"></script>
<script>
// Popovers that stays while on popover bubble
// https://stackoverflow.com/questions/15989591/how-can-i-keep-bootstrap-popover-alive-while-the-popover-is-being-hovered
$('[data-toggle="popover"]').popover({
trigger: 'manual',
html: true,
animation: false,
placement: 'auto'
// Show detail on 'détails' link click
$('#repair_details_link').click(function(e) {
e.preventDefault();
$(this).addClass("d-none");
$("#repair_details_panel").addClass('d-block');
})
.on('mouseenter', function () {
var _this = this;
$(this).popover('show');
$('.popover').on('mouseleave', function () {
$(_this).popover('hide');
});
}).on('mouseleave', function () {
var _this = this;
setTimeout(function () {
if (!$('.popover:hover').length) {
$(_this).popover('hide');
}
}, 100);
});
// Open links found in error description in a new window/tab
$('body').on('mouseover', 'div.popover a', function() {
$(this).prop('title', 'Ouvrir dans une nouvelle fenêtre');
});
$('body').on('click', 'div.popover a', function() {
var url = $(this).prop('href');
window.open(url, '_blank')
return false;
});
</script>
{% endblock %}
validata_ui/views.py
View file @ dfa03469
......@@ -3,7 +3,6 @@
"""
import copy
import io
import itertools
import json
import logging
import subprocess
......@@ -22,7 +21,7 @@ from flask import abort, make_response, redirect, render_template, request, url_
import tabulator
from opendataschema import GitSchemaReference, by_commit_date
from validata_core import messages
from validata_core import messages, repair
from . import app, config, schema_catalog_registry, tableschema_from_url
from .ui_util import flash_error, flash_warning
......@@ -101,7 +100,8 @@ class SchemaInstance:
self.url = schema_reference.get_schema_url(ref=self.ref)
else:
abort(400, "L'un des paramètres est nécessaire : 'schema_name', 'schema_url'")
flash_error("Erreur dans la récupération des informations de schéma")
abort(redirect(url_for('home')))
try:
self.schema = tableschema_from_url(self.url)
......@@ -132,7 +132,7 @@ class SchemaInstance:
return None
def extract_source_data(source: ValidataResource, preview_rows_nb=5):
def extract_source_data(source: ValidataResource, schema_descriptor, preview_rows_nb=5):
""" Computes table preview """
def stringify(val):
......@@ -144,7 +144,19 @@ def extract_source_data(source: ValidataResource, preview_rows_nb=5):
nb_rows = 0
tabulator_source, tabulator_options = source.build_tabulator_stream_args()
# Gets original source, only to get headers
source_header = None
with tabulator.Stream(tabulator_source, **tabulator_options) as stream:
for row in stream:
if source_header is None:
source_header = ['' if v is None else v for v in row]
break
# Repair source
tabulator_source, tabulator_options = source.build_tabulator_stream_args()
fixed_source, repair_report = repair(tabulator_source, schema_descriptor, **tabulator_options)
with tabulator.Stream(fixed_source, {**tabulator_options, 'scheme': 'stream', 'format': 'inline'}) as stream:
for row in stream:
if header is None:
header = ['' if v is None else v for v in row]
......@@ -152,11 +164,22 @@ def extract_source_data(source: ValidataResource, preview_rows_nb=5):
rows.append(list(map(stringify, row)))
nb_rows += 1
preview_rows_nb = min(preview_rows_nb, nb_rows)
return {'header': header,
'rows_nb': nb_rows,
'data_rows': rows,
'preview_rows_nb': preview_rows_nb,
'preview_rows': rows[:preview_rows_nb]}
# Computes original_headers display
if any([err.code == 'wrong-headers-order' for err in repair_report]):
source_header_info = [(h, True) for h in source_header]
else:
schema_field_names = [f['name'] for f in schema_descriptor.get('fields') or []]
source_header_info = [(h, not h or h not in schema_field_names) for h in source_header]
return {
'source_header_info': source_header_info,
'header': header,
'rows_nb': nb_rows,
'data_rows': rows,
'preview_rows_nb': preview_rows_nb,
'preview_rows': rows[:preview_rows_nb]
}
def improve_errors(errors):
......@@ -194,6 +217,132 @@ def improve_errors(errors):
return list(map(improve_err, errors))
def compute_repair_actions(structure_errors):
"""Turn structure errors into repair action informations
"""
def handle_blank_headers(error_list, position_code, action_list, func=None, singular_msg_tpl="", plural_msg_tpl=""):
"""Factors code for blank-header errors
Warning: error_list parameter is modified in place
"""
blank_headers = [err for err in error_list
if err['code'] == 'blank-header' and err['message-data'].get('position') == position_code]
if blank_headers:
if func is None:
blank_headers_nb = len(blank_headers)
if blank_headers_nb == 1:
action_list.append(singular_msg_tpl)
else:
action_list.append(plural_msg_tpl.format(blank_headers_nb))
else:
func(action_list, blank_headers, singular_msg_tpl, plural_msg_tpl)
for err in blank_headers:
error_list.remove(err)
def handle_extra_duplicate_and_missing_errs(error_list, err_code, action_list, singular_msg_tpl, plural_msg_tpl):
"""Factors code for missing headers, extra headers and duplicate headers
Warning: error_list parameter is modified in place
"""
header_errors = [err for err in error_list if err['code'] == err_code]
col_names = ["`{}`".format(err['message-data']['column-name']) for err in header_errors]
if header_errors:
if len(header_errors) == 1:
action_list.append(singular_msg_tpl.format(col_names[0]))
else:
action_list.append(plural_msg_tpl.format(', '.join(col_names)))
for err in header_errors:
error_list.remove(err)
# No error, no info!
if not structure_errors:
return []
# keep a list of processed errors
pending_error_list = structure_errors.copy()
# action informations
action_list = []
# Leading blank headers
handle_blank_headers(pending_error_list, 'leading', action_list,
singular_msg_tpl='1 colonne sans en-tête avant les données a été supprimée',
plural_msg_tpl='{} colonnes sans en-tête avant les données ont été supprimées')
# inside empty header
def handle_in_blank_headers(action_list, error_list, singular_msg_tpl, plural_msg_tpl):
def add_msg(action_list, columns_nb, before, after, singular_msg_tpl, plural_msg_tpl):
if columns_nb == 1:
action_list.append(singular_msg_tpl.format(before, after))
else:
action_list.append(plural_msg_tpl.format(columns_nb, before, after))
before, after = None, None
columns_nb = 0
for err in sorted(error_list, key=lambda elt: elt['message-data']['column-number']):
before_header_name = err['message-data']['before-header-name']
after_header_name = err['message-data']['after-header-name']
if before_header_name == before and after_header_name == after:
columns_nb += 1
else:
if before is not None:
add_msg(action_list, columns_nb, before, after, singular_msg_tpl, plural_msg_tpl)
before = before_header_name
after = after_header_name
columns_nb = 1
add_msg(action_list, columns_nb, before, after, singular_msg_tpl, plural_msg_tpl)
handle_blank_headers(pending_error_list, 'in', action_list,
func=handle_in_blank_headers,
singular_msg_tpl='1 colonne sans en-tête (située entre les colonnes **{}** et **{}**) a été supprimée',
plural_msg_tpl='{} colonnes sans en-tête (situées entre les colonnes **{}** et **{}**) ont été supprimées')
# trailing empty headers
handle_blank_headers(pending_error_list, 'trailing', action_list,
singular_msg_tpl='1 colonne sans en-tête après les données a été supprimée',
plural_msg_tpl='{} colonnes sans en-tête après les données ont été supprimées')
# wrong-headers-order
wrong_headers_order = [err for err in pending_error_list if err['code'] == 'wrong-headers-order']
if wrong_headers_order:
actual_order = wrong_headers_order[0]['message-data']['actual-order']
wanted_order = wrong_headers_order[0]['message-data']['wanted-order']
def field_list_to_str(field_list):
return ', '.join(["**{}**".format(f) for f in field_list])
action_list.append("L'ordre des colonnes du fichier a été rétabli (de {} à {})".format(
field_list_to_str(actual_order), field_list_to_str(wanted_order)
))
pending_error_list = [err for err in pending_error_list if err not in wrong_headers_order]
# extra-headers
handle_extra_duplicate_and_missing_errs(pending_error_list,
'extra-header', action_list,
"La colonne {} inconnue du schéma a été rejetée après les données",
"Les colonnes {} inconnues du schéma ont été rejetées après les données")
# duplicate-header
handle_extra_duplicate_and_missing_errs(pending_error_list,
'duplicate-header', action_list,
"La colonne {} déjà rencontrée dans le fichier a été rejetée après les données",
"Les colonnes {} déjà rencontrées dans le fichier ont été rejetées après les données")
# missing-header
handle_extra_duplicate_and_missing_errs(pending_error_list,
'missing-header', action_list,
"La colonne {} absente du fichier a été ajoutée avec un contenu vide",
"Les colonnes {} absentes du fichier ont été ajoutées avec un contenu vide")
# unhandled errors (it may normally not happened)
for err in pending_error_list:
action_list.append('err: [{}] {}'.format(err['code'], err['message']))
return action_list
def create_validata_ui_report(validata_core_report, schema_dict):
""" Creates an error report easier to handle and display in templates:
- only one table
......@@ -221,11 +370,16 @@ def create_validata_ui_report(validata_core_report, schema_dict):
report['table']['col_count'] = len(headers)
# Computes column info