Commit 0c0c95c2 authored by Pierre Dittgen's avatar Pierre Dittgen

Customizing error messages (not finished)

parent e6c455f6
#!/usr/bin/env python3
import re
from datetime import datetime
import ujson as json
FRENCH_DATE_RE = re.compile(r'^[0-3]\d/[0-1]\d/[12]\d{3}$')
DATETIME_RE = re.compile(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$')
def update_msg(err, msg):
""" Handy method to update err message and return err """
err['message'] = msg
return err
# Core goodtables checks
def blank_row(err, headers, schema):
""" blank-row error """
return update_msg(err, 'La ligne est vide')
def duplicate_row(err, headers, schema):
""" duplicate-row error """
msg_prefix = 'La ligne est identique '
row_numbers = err['message-data']['row_numbers']
if not ',' in row_numbers:
msg = msg_prefix + "à la ligne {}".format(row_numbers)
else:
idx = row_numbers.rfind(',')
param_str = row_numbers[:idx] + ' et' + row_numbers[idx+1:]
msg = msg_prefix + "aux lignes {}".format(param_str)
return update_msg(err, msg)
def enumerable_constraint(err, headers, schema):
""" enumerable-constraint """
constraint_values = json.loads(err['message-data']['constraint'].replace("'", '"'))
ok_values = ['"{}"'.format(val) for val in constraint_values]
if len(ok_values) == 1:
return update_msg(err, 'Valeur incorrecte. La valeur autorisée est : {}'.format(ok_values[0]))
else:
ok_values_str = ', '.join(ok_values[:-2] + [ok_values[-2] + ' et ' + ok_values[-1]])
return update_msg(err, 'Valeur incorrecte. Les valeurs autorisées sont : {}'.format(ok_values_str))
def maximum_constraint(err, headers, schema):
""" maximum-constraint """
max_value = err['message-data']['constraint']
return update_msg(err, 'Valeur incorrecte. La valeur ne doit pas dépasser {}'.format(max_value))
def pattern_constraint(err, headers, schema):
""" pattern-constraint """
column_number = err['column-number']
field = schema['fields'][column_number]
col_name = field['name']
addon_info_list = []
if 'description' in field:
addon_info_list.append('Pour rappel, la description de cette colonne est « {} »'.format(field['description']))
if 'example' in field:
addon_info_list.append('Exemple(s) de valeur correcte : {}'.format(field['example']))
addon_info = '<br/>' + '<br/>'.join(addon_info_list) if addon_info_list else ''
return update_msg(err, 'Valeur incorrecte. La valeur ne respecte pas le format attendu pour la colonne « {} ».{}'.format(col_name, addon_info))
def required_constraint(err, headers, schema):
""" required-constraint error """
col_name = ''
col_nb = err['column-number']
if col_nb <= len(headers):
col_name = ' "{}"'.format(headers[col_nb - 1])
return update_msg(err, 'Le contenu de la colonne{} est obligatoire\n'.format(col_name)
+ 'Merci d\'indiquer une valeur.')
def type_or_format_error(err, headers, schema):
""" type-or-format-value """
err_type = err['message-data']['field_type']
err_value = err['message-data']['value']
# Date
if err_type == 'date':
# Checks if date is dd/mm/yyyy
dm = FRENCH_DATE_RE.match(err_value)
if dm:
iso_date = datetime.strptime(err_value, '%d/%m/%Y').strftime('%Y-%m-%d')
return update_msg(err, "Le format de la date n'est correct.\nLa forme attendue est \"{}\"".format(iso_date))
# Checks if date is yyyy-mm-ddThh:MM:ss
# print('DATE TIME ? [{}]'.format(err_value))
dm = DATETIME_RE.match(err_value)
if dm:
iso_date = err_value[:err_value.find('T')]
return update_msg(err, "Le format de la date n'est correct.\nLa forme attendue est \"{}\"".format(iso_date))
# Default date err msg
return update_msg(err, 'Le format de la date est incorrect')
# Number
elif err_type == 'number':
if ',' in err_value:
en_number = err_value.replace(',', '.')
return update_msg(err, "Le format du nombre est incorrect.\nLa forme attendue est \"{}\"".format(en_number))
# Boolean
# elif err_type == 'boolean':
# Default msg
return update_msg(err, 'XXXX {} XXXX'.format(err_type))
# Validata custom checks
def french_siret_value(err, headers, schema):
""" french-siret-value error """
return update_msg(err, 'Le numéro SIRET est non valide')
# Validata pre-checks
def invalid_column_delimiter(err, headers, schema):
""" invalid-column-delimiter """
md = err['message-data']
return update_msg(err, 'Le fichier CSV utilise le délimiteur de colonne « {} » au lieu du délimiteur attendu « {} ».'.format(md['detected'], md['expected'])
+ "<br/>Pour vous permettre de continuer la validation, un remplacement automatique a été réalisé.")
def missing_headers(err, headers, schema):
""" missing-headers """
cols = err['message-data']['headers']
if len(cols) == 1:
return update_msg(err, "La colonne \"{}\" n'a pas été trouvée dans le fichier".format(cols[0]))
else:
cols_ul = '<ul>' + '\n'.join(['<li>{}</li>'.format(col) for col in cols]) + '</ul>'
fields_nb = len(schema.get('fields', []))
addon_info = 'Utilisez-vous le bon schéma ?' if len(cols) == fields_nb else ''
return update_msg(err, "Les colonnes suivantes n'ont pas été trouvées dans le fichier : {}{}".format(cols_ul, addon_info))
def extra_headers(err, headers, schema):
""" extra-headers """
cols = err['message-data']['headers']
if len(cols) == 1:
return update_msg(err, "La colonne \"{}\" est inconnue dans le schéma".format(cols[0]))
else:
cols_ul = '<ul>' + '\n'.join(['<li>{}</li>'.format(col) for col in cols]) + '</ul>'
addon_info = 'Utilisez-vous le bon schéma ?' if len(cols) == len(headers) else ''
return update_msg(err, "Les colonnes suivantes sont inconnues dans le schéma : {}{}".format(cols_ul, addon_info))
def wrong_headers_order(err, headers, schema):
""" wrong-headers-order """
fields = [f['name'] for f in schema.get('fields', [])]
assert len(headers) == len(fields), 'Wrong column order between two lists of different lengths'
msgs = []
for i, (header, field) in enumerate(zip(headers, fields)):
if header == field:
continue
msgs.append('la colonne {} devrait être « {} » (au lieu de « {} »)'.format(i+1, field, header))
errors_str = '<ul>\n' + '\n'.join(['<li>{}</li>\n'.format(msg) for msg in msgs]) + '\n</ul>'
return update_msg(err, "Les colonnes du tableau ne sont pas dans l'ordre attendu :\n{}".format(errors_str))
......@@ -36,7 +36,6 @@
{% else %}
<h2>La table est invalide</h2>
<p>{{ report.error_count }} erreur(s) détectée(s)</p>
{% if source_type == 'url' %}
<button class="btn btn-primary" id="btn-reload">Relancer la validation</button>
......@@ -46,11 +45,13 @@
{% if report.table.errors.structure %}
<div>
<h3>Problèmes de structure</h3>
<ul>
{% for err in report.table.errors.structure %}
<li>[{{ err.code }}] {{ err.message }}</li>
{% endfor %}
</ul>
{{ report.table.errors.structure|length }} erreur(s) détectée(s)
{% for err in report.table.errors.structure %}
<div class="alert alert-danger">
{{ err.message | safe}}
</div>
{% endfor %}
</div>
{% endif %}
......@@ -58,10 +59,10 @@
{% if report.table.errors.body %}
<div>
<h3>Erreurs de valeurs</h3>
<h3>Problèmes de contenu</h3>
{{ report.table.errors.body|length }} erreur(s) détectée(s)
<div class="table-responsive-sm">
<table class="table-sm table-bordered">
<table class="table-sm table-bordered table-striped table-hover">
<thead class="thead-light">
<th scope="col">1</th>
{% for h in report.table.headers %}
......@@ -85,7 +86,7 @@
{% endif %}
{% for d in source_data.data_rows[row.row_id - 2] %}
{% if loop.index in row.errors %}
<td class="table-danger" data-toggle="tooltip" title="[{{row.errors[loop.index].code}}] {{ row.errors[loop.index].message }}">
<td class="table-danger" data-toggle="tooltip" title="{{ row.errors[loop.index].message }}">
{% elif 'row' in row.errors %}
<td class="table-danger" data-toggle="tooltip" title="{{ row.errors.row.message }}">
{% else %}
......@@ -126,4 +127,4 @@
})
})
</script>
{% endblock %}
{% endblock %}
\ No newline at end of file
......@@ -16,7 +16,7 @@
{% if val_info.author or val_info.contributor %}
<p class="text">
{% if val_info.author %}
Auteur : {{ val_info.author }}
Auteur : {{ val_info.author | safe }}
{% endif %}
{% if val_info.contributor %}
<br />Contributeur(s) : {{ val_info.contributor }}
......
......@@ -76,11 +76,11 @@ class ValidatorHelper:
return {**d1, 'code': schema_code, **d2}
@classmethod
def schema_fields(cls, schema_code):
""" Return schema fields from schema code """
def schema(cls, schema_code):
""" Return schema from schema code """
if not cls.schema_exist(schema_code):
return None
return cls.schema_dict[schema_code]['schema']['fields']
return cls.schema_dict[schema_code]['schema']
@classmethod
def schema_info_list(cls):
......
......@@ -12,6 +12,7 @@ from validata_validate import csv_helpers
from validata_ui_next import app
from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning, ValidataSource
from validata_ui_next.validate_helper import ValidatorHelper
from validata_ui_next import error_messages
from flask import Flask, jsonify, redirect, render_template, request, url_for
import tabulator
......@@ -82,40 +83,51 @@ ERR_CODE_TO_CONTEXT = dict([
# TODO: get it from validata_validate
('invalid-column-delimiter', 'table'),
('missing-headers', 'table'),
('wrong-headers-order', 'table'),
('extra-headers', 'table'),
# Custom checks fall in default case: body
])
ERROR_MESSAGE_FUNC = {
def improve_messages(errors):
# Core checks
'blank-row': error_messages.blank_row,
'duplicate-row': error_messages.duplicate_row,
'enumerable-constraint': error_messages.enumerable_constraint,
'maximum-constraint': error_messages.maximum_constraint,
'required-constraint': error_messages.required_constraint,
'type-or-format-error': error_messages.type_or_format_error,
'pattern-constraint': error_messages.pattern_constraint,
# Validata pre-checks
'extra-headers': error_messages.extra_headers,
'invalid-column-delimiter': error_messages.invalid_column_delimiter,
'wrong-headers-order': error_messages.wrong_headers_order,
'missing-headers': error_messages.missing_headers,
# Validata custom checks
'french-siret-value': error_messages.french_siret_value,
}
def improve_messages(errors, headers, schema):
""" Translates and improve error messages """
def update_message(error, new_message):
""" set a new error message """
def error_message_default_func(error, headers, schema):
""" Sets a new better error message """
data_str = json.dumps(error)
error['message'] = '[{}] {} ({})'.format(error['code'], error.get('message', ''), data_str)
return error
# Stores previous message if exists
if 'message' in error:
error['_original_message'] = error['message']
error['message'] = new_message
improved_errors = []
for error in errors:
if error['code'] == 'blank-row':
update_message(error, 'la ligne est vide')
elif error['code'] == 'duplicate-row':
msg_prefix = 'la ligne est identique '
row_numbers = error['message-data']['row_numbers']
if len(row_numbers) == 1:
msg = msg_prefix + "à la ligne {}".format(row_numbers)
else:
idx = row_numbers.rfind(',')
param_str = row_numbers[:idx] + ' et' + row_numbers[idx+1:]
msg = msg_prefix + "aux lignes {}".format(param_str)
update_message(error, msg)
improve_func = ERROR_MESSAGE_FUNC.get(error['code'], error_message_default_func)
improved_errors.append(improve_func(error, headers, schema))
# Return updated error list
return errors
return improved_errors
def contextualize(errors):
......@@ -124,7 +136,7 @@ def contextualize(errors):
return [{**err, 'context': ERR_CODE_TO_CONTEXT.get(err['code'], 'body')} for err in errors]
def create_validata_report(goodtables_report, schema_fields=[]):
def create_validata_report(goodtables_report, schema):
""" Creates an error report easier to handle and display in templates:
- only one table
- errors are contextualized
......@@ -151,15 +163,16 @@ def create_validata_report(goodtables_report, schema_fields=[]):
report['table']['col_count'] = len(headers)
# Headers title
schema_fields = schema.get('fields', [])
fields_dict = {f['name']: f['title'] for f in schema_fields}
report['table']['headers_title'] = [fields_dict.get(h, '?? colonne inconnue dans le schéma ??') for h in headers]
report['table']['headers_title'] = [fields_dict.get(h, r'/!\ colonne inconnue dans le schéma') for h in headers]
# Add context to errors
errors = contextualize(report['table']['errors'])
del report['table']['errors']
# Provide better (french) messages
errors = improve_messages(errors)
errors = improve_messages(errors, headers, schema)
# Count errors
report['error_count'] = len(errors)
......@@ -177,6 +190,8 @@ def create_validata_report(goodtables_report, schema_fields=[]):
rows = []
current_row_id = 0
for err in report['table']['errors']['body']:
if not 'row-number' in err:
print('ERR', err)
row_id = err['row-number']
del err['row-number']
del err['context']
......@@ -202,7 +217,7 @@ def validate(schema_code, source: ValidataSource):
source_data = extract_source_data(source)
validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema_fields(schema_code))
validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema(schema_code))
# return jsonify(validata_report)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment