views.py 9.28 KB
Newer Older
1 2 3 4
#!/usr/bin/env python3
"""
    Routes
"""
5
import copy
6 7 8
import json
import os
from collections import OrderedDict
9
from datetime import datetime
10
from io import BytesIO
11 12
from pathlib import Path

Pierre Dittgen's avatar
Pierre Dittgen committed
13
from flask import Flask, jsonify, redirect, render_template, request, url_for
14

15
import tabulator
16
from validata_ui import app, error_messages
17 18 19 20 21
from validata_ui.util import (ValidataSource, flash_error, flash_info,
                              flash_success, flash_warning)
from validata_ui.validate_helper import ValidatorHelper
from validata_validate import csv_helpers
from validata_validate.loaders import custom_loaders
22

23 24

def extract_source_data(source: ValidataSource, preview_rows_nb=5):
25
    """ Computes table preview """
26 27 28

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
29
        return '' if val is None else str(val)
30

31 32
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
33
    nb_rows = 0
34

35
    options = {}
36
    if source.format == "csv":
37 38
        options['delimiter'] = csv_helpers.detect_dialect(source.data, format=source.format, scheme=source.scheme,
                                                          custom_loaders=custom_loaders).delimiter
39
    with tabulator.Stream(source.data, format=source.format, scheme=source.scheme, custom_loaders=custom_loaders,
40
                          **options) as stream:
41 42
        for row in stream:
            if header is None:
43
                header = ['' if v is None else v for v in row]
44
            else:
45
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
46
                nb_rows += 1
47
    preview_rows_nb = min(preview_rows_nb, nb_rows)
48 49
    return {'header': header,
            'rows_nb': nb_rows,
50 51 52
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
53 54


55 56
def improve_messages(errors, headers, schema):
    """ Translates and improve error messages """
57

58 59 60 61 62
    def error_message_default_func(error, headers, schema):
        """ Sets a new better error message """
        error['title'] = error['code']
        error['content'] = error.get('message', 'pas d\'information complémentaire')
        return error
63

64
    improved_errors = []
Pierre Dittgen's avatar
Pierre Dittgen committed
65

66
    for error in errors:
Pierre Dittgen's avatar
Pierre Dittgen committed
67

68 69
        improve_func = ERROR_MESSAGE_FUNC.get(error['code'], error_message_default_func)
        improved_errors.append(improve_func(error, headers, schema))
Pierre Dittgen's avatar
Pierre Dittgen committed
70

71
    return improved_errors
Pierre Dittgen's avatar
Pierre Dittgen committed
72 73


74 75
def contextualize(errors):
    """ add context to errors """
76

77 78 79 80
    def add_context(err):
        """ Adds context info based on row-nb presence """
        context = 'body' if 'row-number' in err and not err['row-number'] is None else 'table'
        return {**err, 'context': context}
81

82
    return list(map(add_context, errors))
83 84


85
def create_validata_report(goodtables_report, schema):
86 87 88 89 90 91
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
92
        - error messages are improved
93 94 95 96 97 98 99 100 101 102 103
    """
    report = copy.deepcopy(goodtables_report)

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
104 105 106
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
107 108 109 110
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
111
    # Computes column info
112
    schema_fields = schema.get('fields', [])
Pierre Dittgen's avatar
Pierre Dittgen committed
113 114 115 116
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', '')) for f in schema_fields}
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
117

118 119
    # Add context to errors
    errors = contextualize(report['table']['errors'])
120 121
    del report['table']['errors']

122 123 124
    # Provide better (french) messages
    errors = improve_messages(errors, headers, schema)

125 126 127 128 129 130 131 132 133 134 135 136
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
        if err['context'] != 'body':
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
137
    # Checks if there are structure errors different to invalid-column-delimiter
138 139 140
    report['table']['display_body_errors'] = all(err['code'] == 'invalid-column-delimiter'
                                                 for err in report['table']['errors']['structure'])

Pierre Dittgen's avatar
Pierre Dittgen committed
141 142 143 144
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
145 146
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
147 148 149 150 151 152 153 154 155 156 157 158 159
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
160
    report['table']['errors']['by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
161 162 163 164

    return report


165
def validate(schema_code, source: ValidataSource):
166 167
    """ Validate source and display report """

Pierre Dittgen's avatar
Pierre Dittgen committed
168 169 170 171 172
    try:
        goodtables_report = ValidatorHelper.validate(schema_code, **source.get_goodtables_source())
    except tabulator.exceptions.FormatError:
        flash_error('Erreur : format de fichier non supporté')
        return redirect(url_for('scdl_validator', val_code=schema_code))
173 174

    source_data = extract_source_data(source)
175

176
    validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema(schema_code))
177

178
    # return jsonify(validata_report)
Pierre Dittgen's avatar
Pierre Dittgen committed
179

Pierre Dittgen's avatar
Pierre Dittgen committed
180 181
    # Complete report
    val_info = ValidatorHelper.schema_info(schema_code)
182
    return render_template('validation_report.html', title='Rapport de validation',
183
                           val_info=ValidatorHelper.schema_info(schema_code), report=validata_report,
184
                           validation_date=datetime.now().strftime('le %d/%m/%Y à %Hh%M'),
185
                           source=source, source_type=source.type, source_data=source_data,
186
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
Pierre Dittgen's avatar
Pierre Dittgen committed
187
                           breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
Pierre Dittgen's avatar
Pierre Dittgen committed
188 189
                                        {'url': url_for('scdl_validator', val_code=schema_code),
                                         'title': val_info['title']}])
190 191


192 193 194 195 196 197 198 199
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
    iob = BytesIO()
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


200 201 202 203 204 205
# Routes


@app.route('/')
def home():
    """ Home page """
Pierre Dittgen's avatar
Pierre Dittgen committed
206
    validators = ValidatorHelper.schema_info_list()
Pierre Dittgen's avatar
Pierre Dittgen committed
207
    flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu')
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
    return render_template('home.html', title='Accueil', validators=validators)


@app.route('/validators')
def validators():
    """ No validators page """
    return redirect(url_for('home'))


@app.route('/validators/<val_code>', methods=['GET', 'POST'])
def scdl_validator(val_code):
    """ Validator page """

    if not ValidatorHelper.schema_exist(val_code):
        flash_error('Validateur [{}] inconnu'.format(val_code))
        return redirect(url_for('home'))

    if request.method == 'GET':

Pierre Dittgen's avatar
Pierre Dittgen committed
227
        val_info = ValidatorHelper.schema_info(val_code)
228 229 230 231
        input_param = request.args.get('input')

        # First form display
        if input_param is None or input_param not in ('url', 'example'):
232
            return render_template('validation_form.html', title=val_info['title'],
233 234 235 236 237 238 239 240 241
                                   val_info=val_info,
                                   breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])

        # Process URL
        else:
            url = request.args.get('url')
            if url is None or url == '':
                flash_error("Vous n'avez pas indiqué d'url à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
242
            return validate(val_code, ValidataSource(url, url, 'url'))
243 244 245 246

    else:  # POST
        input_param = request.form.get('input')
        if input_param is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
247
            flash_error('Aucun fichier à valider')
248 249 250 251 252 253 254 255
            return redirect(url_for('scdl_validator', val_code=val_code))

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
256 257

            return validate(val_code, ValidataSource(bytes_data(f), f.filename, 'file'))
258 259

        return 'Bizarre, vous avez dit bizarre ?'