views.py 9.44 KB
Newer Older
1
2
3
4
#!/usr/bin/env python3
"""
    Routes
"""
5
import copy
6
import json
7
from datetime import datetime
8
from io import BytesIO
9

Pierre Dittgen's avatar
Pierre Dittgen committed
10
from commonmark import commonmark
11
from flask import redirect, render_template, request, url_for
12

13
import tabulator
14
import validata_ui.validata_util as vu
Pierre Dittgen's avatar
Pierre Dittgen committed
15
16
from validata_core import csv_helpers
from validata_core.loaders import custom_loaders
17
18
19
20
from validata_ui import app
from validata_ui.ui_util import (flash_error, flash_warning)
from validata_ui.validata_util import ValidataSource
from validata_ui.validate_helper import ValidatorHelper
21

22
23

def extract_source_data(source: ValidataSource, preview_rows_nb=5):
24
    """ Computes table preview """
25
26
27

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
28
        return '' if val is None else str(val)
29

30
31
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
32
    nb_rows = 0
33

34
    options = {}
35
    if source.format == "csv":
36
        options['delimiter'] = csv_helpers.detect_dialect(source.source, format=source.format, scheme=source.scheme,
37
                                                          custom_loaders=custom_loaders).delimiter
38
    with tabulator.Stream(source.source, format=source.format, scheme=source.scheme, custom_loaders=custom_loaders,
39
                          **options) as stream:
40
41
        for row in stream:
            if header is None:
42
                header = ['' if v is None else v for v in row]
43
            else:
44
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
45
                nb_rows += 1
46
    preview_rows_nb = min(preview_rows_nb, nb_rows)
47
48
    return {'header': header,
            'rows_nb': nb_rows,
49
50
51
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
52
53


Pierre Dittgen's avatar
Pierre Dittgen committed
54
55
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
56

Pierre Dittgen's avatar
Pierre Dittgen committed
57
58
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
59

Pierre Dittgen's avatar
Pierre Dittgen committed
60
61
62
63
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
64

Pierre Dittgen's avatar
Pierre Dittgen committed
65
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
66

Pierre Dittgen's avatar
Pierre Dittgen committed
67
68
69
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
70

Pierre Dittgen's avatar
Pierre Dittgen committed
71
72
73
74
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
75

Pierre Dittgen's avatar
Pierre Dittgen committed
76
77
78
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
79

Pierre Dittgen's avatar
Pierre Dittgen committed
80
81
82
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
83

Pierre Dittgen's avatar
Pierre Dittgen committed
84
        return {**err, **update_keys}
85

Pierre Dittgen's avatar
Pierre Dittgen committed
86
    return list(map(improve_err, errors))
87
88


89
def create_validata_report(goodtables_report, schema):
90
91
92
93
94
95
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
96
        - error messages are improved
97
98
99
100
101
102
103
104
105
106
107
    """
    report = copy.deepcopy(goodtables_report)

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
108
109
110
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
111
112
113
114
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
115
    # Computes column info
116
    schema_fields = schema.get('fields', [])
Pierre Dittgen's avatar
Pierre Dittgen committed
117
118
119
120
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', '')) for f in schema_fields}
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
121

122
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
123
124
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
125

126
127
128
129
130
131
132
133
134
135
136
137
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
        if err['context'] != 'body':
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
138
    # Checks if there are structure errors different to invalid-column-delimiter
139
140
141
    report['table']['display_body_errors'] = all(err['code'] == 'invalid-column-delimiter'
                                                 for err in report['table']['errors']['structure'])

Pierre Dittgen's avatar
Pierre Dittgen committed
142
143
144
145
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
146
147
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
148
149
150
151
152
153
154
155
156
157
158
159
160
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
161
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
162
163
164
165

    return report


166
def validate(schema_code, source: ValidataSource):
167
168
    """ Validate source and display report """

Pierre Dittgen's avatar
Pierre Dittgen committed
169
    try:
170
        goodtables_report = vu.validate(schema_code, **source.get_tabulator_params())
Pierre Dittgen's avatar
Pierre Dittgen committed
171
172
173
    except tabulator.exceptions.FormatError:
        flash_error('Erreur : format de fichier non supporté')
        return redirect(url_for('scdl_validator', val_code=schema_code))
174
175

    source_data = extract_source_data(source)
176

177
    validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema(schema_code))
178

179
    # return jsonify(validata_report)
Pierre Dittgen's avatar
Pierre Dittgen committed
180

Pierre Dittgen's avatar
Pierre Dittgen committed
181
182
    # Complete report
    val_info = ValidatorHelper.schema_info(schema_code)
183
    return render_template('validation_report.html', title='Rapport de validation',
184
                           val_info=ValidatorHelper.schema_info(schema_code), report=validata_report,
185
                           validation_date=datetime.now().strftime('le %d/%m/%Y à %Hh%M'),
186
                           source=source, source_type=source.type, source_data=source_data,
187
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
Pierre Dittgen's avatar
Pierre Dittgen committed
188
                           breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
Pierre Dittgen's avatar
Pierre Dittgen committed
189
190
                                        {'url': url_for('scdl_validator', val_code=schema_code),
                                         'title': val_info['title']}])
191
192


193
194
195
196
197
198
199
200
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
    iob = BytesIO()
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


201
202
203
204
205
206
# Routes


@app.route('/')
def home():
    """ Home page """
Pierre Dittgen's avatar
Pierre Dittgen committed
207
    validators = ValidatorHelper.schema_info_list()
Pierre Dittgen's avatar
Pierre Dittgen committed
208
    flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu')
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    return render_template('home.html', title='Accueil', validators=validators)


@app.route('/validators')
def validators():
    """ No validators page """
    return redirect(url_for('home'))


@app.route('/validators/<val_code>', methods=['GET', 'POST'])
def scdl_validator(val_code):
    """ Validator page """

    if not ValidatorHelper.schema_exist(val_code):
        flash_error('Validateur [{}] inconnu'.format(val_code))
        return redirect(url_for('home'))

    if request.method == 'GET':

Pierre Dittgen's avatar
Pierre Dittgen committed
228
        val_info = ValidatorHelper.schema_info(val_code)
229
230
231
232
        input_param = request.args.get('input')

        # First form display
        if input_param is None or input_param not in ('url', 'example'):
233
            return render_template('validation_form.html', title=val_info['title'],
234
235
236
237
238
                                   val_info=val_info,
                                   breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])

        # Process URL
        else:
239
240
            url_param = request.args.get('url')
            if url_param is None or url_param == '':
241
242
                flash_error("Vous n'avez pas indiqué d'url à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
243
            return validate(val_code, ValidataSource('url', url_param, url_param))
244
245
246
247

    else:  # POST
        input_param = request.form.get('input')
        if input_param is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
248
            flash_error('Aucun fichier à valider')
249
250
251
252
253
254
255
256
            return redirect(url_for('scdl_validator', val_code=val_code))

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
257

258
            return validate(val_code, ValidataSource('file', f.filename, bytes_data(f)))
259
260

        return 'Bizarre, vous avez dit bizarre ?'