views.py 9.59 KB
Newer Older
1 2 3 4
#!/usr/bin/env python3
"""
    Routes
"""
5
import copy
6
import json
7
from datetime import datetime
8
from io import BytesIO
9

Pierre Dittgen's avatar
Pierre Dittgen committed
10
from commonmark import commonmark
11
from flask import redirect, render_template, request, url_for
12

13
import tabulator
14
import validata_ui.validata_util as vu
Pierre Dittgen's avatar
Pierre Dittgen committed
15 16
from validata_core import csv_helpers
from validata_core.loaders import custom_loaders
17
from validata_ui import app
Pierre Dittgen's avatar
Pierre Dittgen committed
18
from validata_ui.ui_util import flash_error, flash_warning
19 20
from validata_ui.validata_util import ValidataSource
from validata_ui.validate_helper import ValidatorHelper
21

22 23

def extract_source_data(source: ValidataSource, preview_rows_nb=5):
24
    """ Computes table preview """
25 26 27

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
28
        return '' if val is None else str(val)
29

30 31
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
32
    nb_rows = 0
33

34
    options = {}
35
    if source.format == "csv":
36
        options['delimiter'] = csv_helpers.detect_dialect(source.source, format=source.format, scheme=source.scheme,
37
                                                          custom_loaders=custom_loaders).delimiter
38
    with tabulator.Stream(source.source, format=source.format, scheme=source.scheme, custom_loaders=custom_loaders,
39
                          **options) as stream:
40 41
        for row in stream:
            if header is None:
42
                header = ['' if v is None else v for v in row]
43
            else:
44
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
45
                nb_rows += 1
46
    preview_rows_nb = min(preview_rows_nb, nb_rows)
47 48
    return {'header': header,
            'rows_nb': nb_rows,
49 50 51
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
52 53


Pierre Dittgen's avatar
Pierre Dittgen committed
54 55
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
56

Pierre Dittgen's avatar
Pierre Dittgen committed
57 58
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
59

Pierre Dittgen's avatar
Pierre Dittgen committed
60 61 62 63
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
64

Pierre Dittgen's avatar
Pierre Dittgen committed
65
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
66

Pierre Dittgen's avatar
Pierre Dittgen committed
67 68 69
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
70

Pierre Dittgen's avatar
Pierre Dittgen committed
71 72 73 74
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
75

Pierre Dittgen's avatar
Pierre Dittgen committed
76 77 78
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
79

Pierre Dittgen's avatar
Pierre Dittgen committed
80 81 82
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
83

Pierre Dittgen's avatar
Pierre Dittgen committed
84
        return {**err, **update_keys}
85

Pierre Dittgen's avatar
Pierre Dittgen committed
86
    return list(map(improve_err, errors))
87 88


89
def create_validata_report(goodtables_report, schema):
90 91 92 93 94 95
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
96
        - error messages are improved
97 98 99 100 101 102 103 104 105 106 107
    """
    report = copy.deepcopy(goodtables_report)

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
108 109 110
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
111 112 113 114
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
115
    # Computes column info
116
    schema_fields = schema.get('fields', [])
Pierre Dittgen's avatar
Pierre Dittgen committed
117 118 119 120
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', '')) for f in schema_fields}
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
121

122
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
123 124
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
125

126 127 128 129 130 131 132 133 134 135 136 137
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
        if err['context'] != 'body':
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
138
    # Checks if there are structure errors different to invalid-column-delimiter
139 140 141
    report['table']['display_body_errors'] = all(err['code'] == 'invalid-column-delimiter'
                                                 for err in report['table']['errors']['structure'])

Pierre Dittgen's avatar
Pierre Dittgen committed
142 143 144 145
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
146 147
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
148 149 150 151 152 153 154 155 156 157 158 159 160
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
161
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
162 163 164 165

    return report


166
def validate(schema_code, source: ValidataSource):
167 168
    """ Validate source and display report """

169
    try:
170
        goodtables_report = vu.validate(schema_code, **source.get_tabulator_params())
171 172 173
    except tabulator.exceptions.FormatError:
        flash_error('Erreur : format de fichier non supporté')
        return redirect(url_for('scdl_validator', val_code=schema_code))
174 175

    source_data = extract_source_data(source)
176

Pierre Dittgen's avatar
Pierre Dittgen committed
177 178 179
    # handle report date
    date_str = goodtables_report['date']
    report_date = datetime.strptime(date_str[:date_str.find('.')], '%Y-%m-%dT%H:%M:%S')
180

Pierre Dittgen's avatar
Pierre Dittgen committed
181 182
    # Enhance goodtables_report
    validata_report = create_validata_report(goodtables_report, ValidatorHelper.schema(schema_code))
Pierre Dittgen's avatar
Pierre Dittgen committed
183

Pierre Dittgen's avatar
Pierre Dittgen committed
184
    # Display report to the user
Pierre Dittgen's avatar
Pierre Dittgen committed
185
    val_info = ValidatorHelper.schema_info(schema_code)
186
    return render_template('validation_report.html', title='Rapport de validation',
187
                           val_info=ValidatorHelper.schema_info(schema_code), report=validata_report,
Pierre Dittgen's avatar
Pierre Dittgen committed
188
                           validation_date=report_date.strftime('le %d/%m/%Y à %Hh%M'),
189
                           source=source, source_type=source.type, source_data=source_data,
190
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
Pierre Dittgen's avatar
Pierre Dittgen committed
191
                           breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
Pierre Dittgen's avatar
Pierre Dittgen committed
192 193
                                        {'url': url_for('scdl_validator', val_code=schema_code),
                                         'title': val_info['title']}])
194 195


196 197 198 199 200 201 202 203
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
    iob = BytesIO()
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


204 205 206 207 208 209
# Routes


@app.route('/')
def home():
    """ Home page """
Pierre Dittgen's avatar
Pierre Dittgen committed
210
    validators = ValidatorHelper.schema_info_list()
Pierre Dittgen's avatar
Pierre Dittgen committed
211
    flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu')
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
    return render_template('home.html', title='Accueil', validators=validators)


@app.route('/validators')
def validators():
    """ No validators page """
    return redirect(url_for('home'))


@app.route('/validators/<val_code>', methods=['GET', 'POST'])
def scdl_validator(val_code):
    """ Validator page """

    if not ValidatorHelper.schema_exist(val_code):
        flash_error('Validateur [{}] inconnu'.format(val_code))
        return redirect(url_for('home'))

    if request.method == 'GET':

Pierre Dittgen's avatar
Pierre Dittgen committed
231
        val_info = ValidatorHelper.schema_info(val_code)
232 233 234 235
        input_param = request.args.get('input')

        # First form display
        if input_param is None or input_param not in ('url', 'example'):
236
            return render_template('validation_form.html', title=val_info['title'],
237 238 239 240 241
                                   val_info=val_info,
                                   breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])

        # Process URL
        else:
242 243
            url_param = request.args.get('url')
            if url_param is None or url_param == '':
244 245
                flash_error("Vous n'avez pas indiqué d'url à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
246
            return validate(val_code, ValidataSource('url', url_param, url_param))
247 248 249 250

    else:  # POST
        input_param = request.form.get('input')
        if input_param is None:
251
            flash_error('Aucun fichier à valider')
252 253 254 255 256 257 258 259
            return redirect(url_for('scdl_validator', val_code=val_code))

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
                return redirect(url_for('scdl_validator', val_code=val_code))
260

261
            return validate(val_code, ValidataSource('file', f.filename, bytes_data(f)))
262 263

        return 'Bizarre, vous avez dit bizarre ?'