Customizing error messages (not finished)

validata_ui_next/error_messages.py

+#!/usr/bin/env python3
+import re
+from datetime import datetime
+
+import ujson as json
+
+FRENCH_DATE_RE = re.compile(r'^[0-3]\d/[0-1]\d/[12]\d{3}$')
+DATETIME_RE = re.compile(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$')
+
+
+def update_msg(err, msg):
+    """ Handy method to update err message and return err """
+    err['message'] = msg
+    return err
+
+# Core goodtables checks
+
+
+def blank_row(err, headers, schema):
+    """ blank-row error """
+    return update_msg(err, 'La ligne est vide')
+
+
+def duplicate_row(err, headers, schema):
+    """ duplicate-row error """
+    msg_prefix = 'La ligne est identique '
+    row_numbers = err['message-data']['row_numbers']
+    if not ',' in row_numbers:
+        msg = msg_prefix + "à la ligne {}".format(row_numbers)
+    else:
+        idx = row_numbers.rfind(',')
+        param_str = row_numbers[:idx] + ' et' + row_numbers[idx+1:]
+        msg = msg_prefix + "aux lignes {}".format(param_str)
+    return update_msg(err, msg)
+
+
+def enumerable_constraint(err, headers, schema):
+    """ enumerable-constraint """
+    constraint_values = json.loads(err['message-data']['constraint'].replace("'", '"'))
+    ok_values = ['"{}"'.format(val) for val in constraint_values]
+    if len(ok_values) == 1:
+        return update_msg(err, 'Valeur incorrecte. La valeur autorisée est : {}'.format(ok_values[0]))
+    else:
+        ok_values_str = ', '.join(ok_values[:-2] + [ok_values[-2] + ' et ' + ok_values[-1]])
+        return update_msg(err, 'Valeur incorrecte. Les valeurs autorisées sont : {}'.format(ok_values_str))
+
+
+def maximum_constraint(err, headers, schema):
+    """ maximum-constraint """
+    max_value = err['message-data']['constraint']
+    return update_msg(err, 'Valeur incorrecte. La valeur ne doit pas dépasser {}'.format(max_value))
+
+
+def pattern_constraint(err, headers, schema):
+    """ pattern-constraint """
+    column_number = err['column-number']
+    field = schema['fields'][column_number]
+    col_name = field['name']
+    addon_info_list = []
+    if 'description' in field:
+        addon_info_list.append('Pour rappel, la description de cette colonne est « {} »'.format(field['description']))
+    if 'example' in field:
+        addon_info_list.append('Exemple(s) de valeur correcte : {}'.format(field['example']))
+    addon_info = '<br/>' + '<br/>'.join(addon_info_list) if addon_info_list else ''
+    return update_msg(err, 'Valeur incorrecte. La valeur ne respecte pas le format attendu pour la colonne « {} ».{}'.format(col_name, addon_info))
+
+
+def required_constraint(err, headers, schema):
+    """ required-constraint error """
+    col_name = ''
+    col_nb = err['column-number']
+    if col_nb <= len(headers):
+        col_name = ' "{}"'.format(headers[col_nb - 1])
+    return update_msg(err, 'Le contenu de la colonne{} est obligatoire\n'.format(col_name)
+                      + 'Merci d\'indiquer une valeur.')
+
+
+def type_or_format_error(err, headers, schema):
+    """ type-or-format-value """
+    err_type = err['message-data']['field_type']
+    err_value = err['message-data']['value']
+
+    # Date
+    if err_type == 'date':
+        # Checks if date is dd/mm/yyyy
+        dm = FRENCH_DATE_RE.match(err_value)
+        if dm:
+            iso_date = datetime.strptime(err_value, '%d/%m/%Y').strftime('%Y-%m-%d')
+            return update_msg(err, "Le format de la date n'est correct.\nLa forme attendue est \"{}\"".format(iso_date))
+
+        # Checks if date is yyyy-mm-ddThh:MM:ss
+        # print('DATE TIME ? [{}]'.format(err_value))
+        dm = DATETIME_RE.match(err_value)
+        if dm:
+            iso_date = err_value[:err_value.find('T')]
+            return update_msg(err, "Le format de la date n'est correct.\nLa forme attendue est \"{}\"".format(iso_date))
+
+        # Default date err msg
+        return update_msg(err, 'Le format de la date est incorrect')
+
+    # Number
+    elif err_type == 'number':
+        if ',' in err_value:
+            en_number = err_value.replace(',', '.')
+            return update_msg(err, "Le format du nombre est incorrect.\nLa forme attendue est \"{}\"".format(en_number))
+
+    # Boolean
+    # elif err_type == 'boolean':
+
+    # Default msg
+    return update_msg(err, 'XXXX {} XXXX'.format(err_type))
+
+# Validata custom checks
+
+
+def french_siret_value(err, headers, schema):
+    """ french-siret-value error """
+    return update_msg(err, 'Le numéro SIRET est non valide')
+
+# Validata pre-checks
+
+
+def invalid_column_delimiter(err, headers, schema):
+    """ invalid-column-delimiter """
+    md = err['message-data']
+    return update_msg(err, 'Le fichier CSV utilise le délimiteur de colonne « {} » au lieu du délimiteur attendu  « {} ».'.format(md['detected'], md['expected'])
+                      + "<br/>Pour vous permettre de continuer la validation, un remplacement automatique a été réalisé.")
+
+
+def missing_headers(err, headers, schema):
+    """ missing-headers """
+    cols = err['message-data']['headers']
+    if len(cols) == 1:
+        return update_msg(err, "La colonne \"{}\" n'a pas été trouvée dans le fichier".format(cols[0]))
+    else:
+        cols_ul = '<ul>' + '\n'.join(['<li>{}</li>'.format(col) for col in cols]) + '</ul>'
+        fields_nb = len(schema.get('fields', []))
+        addon_info = 'Utilisez-vous le bon schéma ?' if len(cols) == fields_nb else ''
+        return update_msg(err, "Les colonnes suivantes n'ont pas été trouvées dans le fichier : {}{}".format(cols_ul, addon_info))
+
+
+def extra_headers(err, headers, schema):
+    """ extra-headers """
+    cols = err['message-data']['headers']
+    if len(cols) == 1:
+        return update_msg(err, "La colonne \"{}\" est inconnue dans le schéma".format(cols[0]))
+    else:
+        cols_ul = '<ul>' + '\n'.join(['<li>{}</li>'.format(col) for col in cols]) + '</ul>'
+        addon_info = 'Utilisez-vous le bon schéma ?' if len(cols) == len(headers) else ''
+        return update_msg(err, "Les colonnes suivantes sont inconnues dans le schéma : {}{}".format(cols_ul, addon_info))
+
+
+def wrong_headers_order(err, headers, schema):
+    """ wrong-headers-order """
+    fields = [f['name'] for f in schema.get('fields', [])]
+    assert len(headers) == len(fields), 'Wrong column order between two lists of different lengths'
+    msgs = []
+    for i, (header, field) in enumerate(zip(headers, fields)):
+        if header == field:
+            continue
+        msgs.append('la colonne {} devrait être « {} » (au lieu de « {} »)'.format(i+1, field, header))
+    errors_str = '<ul>\n' + '\n'.join(['<li>{}</li>\n'.format(msg) for msg in msgs]) + '\n</ul>'
+    return update_msg(err, "Les colonnes du tableau ne sont pas dans l'ordre attendu :\n{}".format(errors_str))

validata_ui_next/templates/validation_report.html

@@ -36,7 +36,6 @@
 {% else %}
 
 <h2>La table est invalide</h2>
-<p>{{ report.error_count }} erreur(s) détectée(s)</p>
 
 {% if source_type == 'url' %}
 <button class="btn btn-primary" id="btn-reload">Relancer la validation</button>
@@ -46,11 +45,13 @@
 {% if report.table.errors.structure %}
 <div>
     <h3>Problèmes de structure</h3>
-    <ul>
-        {% for err in report.table.errors.structure %}
-        <li>[{{ err.code }}] {{ err.message }}</li>
-        {% endfor %}
-    </ul>
+    {{ report.table.errors.structure|length }} erreur(s) détectée(s)
+
+    {% for err in report.table.errors.structure %}
+    <div class="alert alert-danger">
+        {{ err.message | safe}}
+    </div>
+    {% endfor %}
 </div>
 {% endif %}
 
@@ -58,10 +59,10 @@
 
 {% if report.table.errors.body %}
 <div>
-    <h3>Erreurs de valeurs</h3>
-
+    <h3>Problèmes de contenu</h3>
+    {{ report.table.errors.body|length }} erreur(s) détectée(s)
     <div class="table-responsive-sm">
-        <table class="table-sm table-bordered">
+        <table class="table-sm table-bordered table-striped table-hover">
             <thead class="thead-light">
                 <th scope="col">1</th>
                 {% for h in report.table.headers %}
@@ -85,7 +86,7 @@
                     {% endif %}
                     {% for d in source_data.data_rows[row.row_id - 2] %}
                     {% if loop.index in row.errors %}
-                    <td class="table-danger" data-toggle="tooltip" title="[{{row.errors[loop.index].code}}] {{ row.errors[loop.index].message }}">
+                    <td class="table-danger" data-toggle="tooltip" title="{{ row.errors[loop.index].message }}">
                         {% elif 'row' in row.errors %}
                     <td class="table-danger" data-toggle="tooltip" title="{{ row.errors.row.message }}">
                         {% else %}
@@ -126,4 +127,4 @@
         })
     })
 </script>
-{% endblock %}
+{% endblock %}
\ No newline at end of file

validata_ui_next/templates/validator.html

@@ -16,7 +16,7 @@
         {% if val_info.author or val_info.contributor %}
         <p class="text">
             {% if val_info.author %}
-            Auteur : {{ val_info.author }}
+            Auteur : {{ val_info.author | safe }}
             {% endif %}
             {% if val_info.contributor %}
             <br />Contributeur(s) : {{ val_info.contributor }}

validata_ui_next/validate_helper.py

@@ -76,11 +76,11 @@ class ValidatorHelper:
         return {**d1, 'code': schema_code, **d2}
 
     @classmethod
-    def schema_fields(cls, schema_code):
-        """ Return schema fields from schema code """
+    def schema(cls, schema_code):
+        """ Return schema from schema code """
         if not cls.schema_exist(schema_code):
             return None
-        return cls.schema_dict[schema_code]['schema']['fields']
+        return cls.schema_dict[schema_code]['schema']
 
     @classmethod
     def schema_info_list(cls):

validata_ui_next/views.py

@@ -12,6 +12,7 @@ from validata_validate import csv_helpers
 from validata_ui_next import app
 from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning, ValidataSource
 from validata_ui_next.validate_helper import ValidatorHelper
+from validata_ui_next import error_messages
 
 from flask import Flask, jsonify, redirect, render_template, request, url_for
 import tabulator
@@ -82,40 +83,51 @@ ERR_CODE_TO_CONTEXT = dict([
     # TODO: get it from validata_validate
     ('invalid-column-delimiter', 'table'),
     ('missing-headers', 'table'),
+    ('wrong-headers-order', 'table'),
+    ('extra-headers', 'table'),
 
     # Custom checks fall in default case: body
 ])
 
+ERROR_MESSAGE_FUNC = {
 
-def improve_messages(errors):
+    # Core checks
+    'blank-row': error_messages.blank_row,
+    'duplicate-row': error_messages.duplicate_row,
+    'enumerable-constraint': error_messages.enumerable_constraint,
+    'maximum-constraint': error_messages.maximum_constraint,
+    'required-constraint': error_messages.required_constraint,
+    'type-or-format-error': error_messages.type_or_format_error,
+    'pattern-constraint': error_messages.pattern_constraint,
+
+    # Validata pre-checks
+    'extra-headers': error_messages.extra_headers,
+    'invalid-column-delimiter': error_messages.invalid_column_delimiter,
+    'wrong-headers-order': error_messages.wrong_headers_order,
+    'missing-headers': error_messages.missing_headers,
+
+    # Validata custom checks
+    'french-siret-value': error_messages.french_siret_value,
+}
+
+
+def improve_messages(errors, headers, schema):
     """ Translates and improve error messages """
 
-    def update_message(error, new_message):
-        """ set a new error message """
+    def error_message_default_func(error, headers, schema):
+        """ Sets a new better error message """
+        data_str = json.dumps(error)
+        error['message'] = '[{}] {} ({})'.format(error['code'], error.get('message', ''), data_str)
+        return error
 
-        # Stores previous message if exists
-        if 'message' in error:
-            error['_original_message'] = error['message']
-        error['message'] = new_message
+    improved_errors = []
 
     for error in errors:
 
-        if error['code'] == 'blank-row':
-            update_message(error, 'la ligne est vide')
-
-        elif error['code'] == 'duplicate-row':
-            msg_prefix = 'la ligne est identique '
-            row_numbers = error['message-data']['row_numbers']
-            if len(row_numbers) == 1:
-                msg = msg_prefix + "à la ligne {}".format(row_numbers)
-            else:
-                idx = row_numbers.rfind(',')
-                param_str = row_numbers[:idx] + ' et' + row_numbers[idx+1:]
-                msg = msg_prefix + "aux lignes {}".format(param_str)
-            update_message(error, msg)
+        improve_func = ERROR_MESSAGE_FUNC.get(error['code'], error_message_default_func)
+        improved_errors.append(improve_func(error, headers, schema))
 
-    # Return updated error list
-    return errors
+    return improved_errors
 
 
 def contextualize(errors):
@@ -124,7 +136,7 @@ def contextualize(errors):
     return [{**err, 'context': ERR_CODE_TO_CONTEXT.get(err['code'], 'body')} for err in errors]
 
 
-def create_validata_report(goodtables_report, schema_fields=[]):
+def create_validata_report(goodtables_report, schema):
     """ Creates an error report easier to handle and display in templates:
         - only one table
         - errors are contextualized
@@ -151,15 +163,16 @@ def create_validata_report(goodtables_report, schema_fields=[]):
     report['table']['col_count'] = len(headers)
 
     # Headers title
+    schema_fields = schema.get('fields', [])
     fields_dict = {f['name']: f['title'] for f in schema_fields}
-    report['table']['headers_title'] = [fields_dict.get(h, '?? colonne inconnue dans le schéma ??') for h in headers]
+    report['table']['headers_title'] = [fields_dict.get(h, r'/!\ colonne inconnue dans le schéma') for h in headers]
 
     # Add context to errors
     errors = contextualize(report['table']['errors'])
     del report['table']['errors']
 
     # Provide better (french) messages
-    errors = improve_messages(errors)
+    errors = improve_messages(errors, headers, schema)
 
     # Count errors
     report['error_count'] = len(errors)
@@ -177,6 +190,8 @@ def create_validata_report(goodtables_report, schema_fields=[]):
     rows = []
     current_row_id = 0
     for err in report['table']['errors']['body']:
+        if not 'row-number' in err:
+            print('ERR', err)
         row_id = err['row-number']
         del err['row-number']
         del err['context']
@@ -202,7 +217,7 @@ def validate(schema_code, source: ValidataSource):
 
     source_data = extract_source_data(source)
 
-    validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema_fields(schema_code))
+    validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema(schema_code))
 
     # return jsonify(validata_report)