views.py 10.4 KB
Newer Older
1
2
3
4
#!/usr/bin/env python3
"""
    Routes
"""
5
import copy
6
7
8
9
10
import json
import os
from collections import OrderedDict
from pathlib import Path

11
from validata_validate import csv_helpers
12
from validata_validate.loaders import custom_loaders
13
from validata_ui_next import app
14
from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning, ValidataSource
15
from validata_ui_next.validate_helper import ValidatorHelper
16
from validata_ui_next import error_messages
17

Pierre Dittgen's avatar
Pierre Dittgen committed
18
from flask import Flask, jsonify, redirect, render_template, request, url_for
19
20
import tabulator

21
from io import BytesIO
22

23
24

def extract_source_data(source: ValidataSource, preview_rows_nb=5):
25
    """ Computes table preview """
26
27
28
29
30
31
32

    def stringify(val):
        """ Transform value into string """
        if val is None:
            return ''
        return str(val)

33
34
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
35
    nb_rows = 0
36
37
38

    delimiter = None
    if source.format == "csv":
39
40
41
42
        delimiter = csv_helpers.detect_dialect(source.data, format=source.format, scheme=source.scheme,
                                               custom_loaders=custom_loaders).delimiter
    with tabulator.Stream(source.data, format=source.format, scheme=source.scheme, custom_loaders=custom_loaders,
                          delimiter=delimiter) as stream:
43
44
45
46
        for row in stream:
            if header is None:
                header = row
            else:
47
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
48
                nb_rows += 1
49
    preview_rows_nb = min(preview_rows_nb, nb_rows)
50
51
    return {'header': header,
            'rows_nb': nb_rows,
52
53
54
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
55
56


57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# ERR_CODE_TO_CONTEXT = dict([
#     # TODO: gets it from spec.json
#     ('duplicate-header', 'head'),
#     ('extra-value', 'body'),
#     ('missing-value', 'body'),
#     ('source-error', 'table'),
#     ('schema-error', 'table'),
#     ('non-matching-header', 'head'),
#     ('blank-row', 'body'),
#     ('blank-header', 'head'),
#     ('enumerable-constraint', 'body'),
#     ('http-error', 'table'),
#     ('scheme-error', 'table'),
#     ('type-or-format-error', 'body'),
#     ('format-error', 'table'),
#     ('extra-header', 'head'),
#     ('pattern-constraint', 'body'),
#     ('required-constraint', 'body'),
#     ('missing-header', 'head'),
#     ('maximum-length-constraint', 'body'),
#     ('maximum-constraint', 'body'),
#     ('minimum-length-constraint', 'body'),
#     ('encoding-error', 'table'),
#     ('io-error', 'table'),
#     ('unique-constraint', 'body'),
#     ('duplicate-row', 'body'),
#     ('minimum-constraint', 'body'),

#     # TODO: get it from validata_validate
#     ('invalid-column-delimiter', 'table'),
#     ('missing-headers', 'table'),
#     ('wrong-headers-order', 'table'),
#     ('extra-headers', 'table'),

#     # Custom checks fall in default case: body
# ])
Pierre Dittgen's avatar
Pierre Dittgen committed
93

94
ERROR_MESSAGE_FUNC = {
Pierre Dittgen's avatar
Pierre Dittgen committed
95

96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
    # Core checks
    'blank-row': error_messages.blank_row,
    'duplicate-row': error_messages.duplicate_row,
    'enumerable-constraint': error_messages.enumerable_constraint,
    'maximum-constraint': error_messages.maximum_constraint,
    'required-constraint': error_messages.required_constraint,
    'type-or-format-error': error_messages.type_or_format_error,
    'pattern-constraint': error_messages.pattern_constraint,

    # Validata pre-checks
    'extra-headers': error_messages.extra_headers,
    'invalid-column-delimiter': error_messages.invalid_column_delimiter,
    'wrong-headers-order': error_messages.wrong_headers_order,
    'missing-headers': error_messages.missing_headers,

    # Validata custom checks
    'french-siret-value': error_messages.french_siret_value,
}


def improve_messages(errors, headers, schema):
117
118
    """ Translates and improve error messages """

119
120
121
122
123
    def error_message_default_func(error, headers, schema):
        """ Sets a new better error message """
        data_str = json.dumps(error)
        error['message'] = '[{}] {} ({})'.format(error['code'], error.get('message', ''), data_str)
        return error
124

125
    improved_errors = []
126
127
128

    for error in errors:

129
130
        improve_func = ERROR_MESSAGE_FUNC.get(error['code'], error_message_default_func)
        improved_errors.append(improve_func(error, headers, schema))
131

132
    return improved_errors
133
134


135
def contextualize(errors):
Pierre Dittgen's avatar
Pierre Dittgen committed
136
137
    """ add context to errors """

138
139
140
141
142
143
    def add_context(err):
        """ Adds context info based on row-nb presence """
        context = 'body' if 'row-number' in err else 'table'
        return {**err, 'context': context}

    return list(map(add_context, errors))
144
145


146
def create_validata_report(goodtables_report, schema):
147
148
149
150
151
152
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
153
        - error messages are improved
154
155
156
157
158
159
160
161
162
163
164
    """
    report = copy.deepcopy(goodtables_report)

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
165
166
167
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
168
169
170
171
172
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

    # Headers title
173
    schema_fields = schema.get('fields', [])
Pierre Dittgen's avatar
Pierre Dittgen committed
174
    fields_dict = {f['name']: f['title'] for f in schema_fields}
175
    report['table']['headers_title'] = [fields_dict.get(h, r'/!\ colonne inconnue dans le schéma') for h in headers]
176
177
178
179
180

    # Add context to errors
    errors = contextualize(report['table']['errors'])
    del report['table']['errors']

181
    # Provide better (french) messages
182
    errors = improve_messages(errors, headers, schema)
183

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
        if err['context'] != 'body':
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

    # and group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
200
201
        if not 'row-number' in err:
            print('ERR', err)
202
203
204
205
206
207
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})
208
209
210
211
212
213
214

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
215
    report['table']['errors']['by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
216
217
218
219

    return report


220
def validate(schema_code, source: ValidataSource):
221
222
    """ Validate source and display report """

223
224
225
    goodtables_report = ValidatorHelper.validate(schema_code, **source.get_goodtables_source())

    source_data = extract_source_data(source)
226

227
    validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema(schema_code))
228

229
    # return jsonify(validata_report)
Pierre Dittgen's avatar
Pierre Dittgen committed
230

Pierre Dittgen's avatar
Pierre Dittgen committed
231
232
    # Complete report
    val_info = ValidatorHelper.schema_info(schema_code)
233
    return render_template('validation_report.html', title='Rapport de validation',
234
                           val_info=ValidatorHelper.schema_info(schema_code), report=validata_report,
235
                           source=source, source_type=source.type, source_data=source_data,
236
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
Pierre Dittgen's avatar
Pierre Dittgen committed
237
238
                           breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
                                        {'url': url_for('scdl_validator', val_code=schema_code), 'title': val_info['title']}])
239
240


241
242
243
244
245
246
247
248
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
    iob = BytesIO()
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


249
250
251
252
253
254
# Routes


@app.route('/')
def home():
    """ Home page """
Pierre Dittgen's avatar
Pierre Dittgen committed
255
    validators = ValidatorHelper.schema_info_list()
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
    return render_template('home.html', title='Accueil', validators=validators)


@app.route('/about')
def about():
    """ Help -> About page """
    return render_template('about.html', title='À propos',
                           breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])


@app.route('/validators')
def validators():
    """ No validators page """
    return redirect(url_for('home'))


@app.route('/validators/<val_code>', methods=['GET', 'POST'])
def scdl_validator(val_code):
    """ Validator page """

    if not ValidatorHelper.schema_exist(val_code):
        flash_error('Validateur [{}] inconnu'.format(val_code))
        return redirect(url_for('home'))

    if request.method == 'GET':

Pierre Dittgen's avatar
Pierre Dittgen committed
282
        val_info = ValidatorHelper.schema_info(val_code)
283
284
285
286
287
288
289
290
291
292
293
294
295
296
        input_param = request.args.get('input')

        # First form display
        if input_param is None or input_param not in ('url', 'example'):
            return render_template('validator.html', title=val_info['title'],
                                   val_info=val_info,
                                   breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])

        # Process URL
        else:
            url = request.args.get('url')
            if url is None or url == '':
                flash_error("Vous n'avez pas indiqué d'url à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
297
            return validate(val_code, ValidataSource(url, url, 'url'))
298
299
300
301
302
303
304
305
306
307
308
309
310

    else:  # POST
        input_param = request.form.get('input')
        if input_param is None:
            flash_error('Source non définie')
            return redirect(url_for('scdl_validator', val_code=val_code))

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
311
312

            return validate(val_code, ValidataSource(bytes_data(f), f.filename, 'file'))
313
314

        return 'Bizarre, vous avez dit bizarre ?'