views.py 22.3 KB
Newer Older
1
2
3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
5
import io
Pierre Dittgen's avatar
Pierre Dittgen committed
6
import itertools
7
import json
Christophe Benz's avatar
Christophe Benz committed
8
import logging
9
import subprocess
Christophe Benz's avatar
Christophe Benz committed
10
import tempfile
11
from datetime import datetime
12
from operator import itemgetter
13
from pathlib import Path
14
from urllib.parse import quote_plus, urlencode
15

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
16
import requests
17
import tableschema
18
from backports.datetime_fromisoformat import MonkeyPatch
19
from commonmark import commonmark
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
20
from flask import make_response, redirect, render_template, request, url_for
21
from validata_core import compute_badge, messages
22

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
23
24
import tabulator

Pierre Dittgen's avatar
Pierre Dittgen committed
25
from . import app, config, ui_config, schema_catalog_map, schema_from_url
26
from .ui_util import flash_error, flash_warning
27
from .validata_util import ValidataResource, URLValidataResource, UploadedFileValidataResource
28

29
30
MonkeyPatch.patch_fromisoformat()

Christophe Benz's avatar
Christophe Benz committed
31
32
log = logging.getLogger(__name__)

33

34
35
36
class SchemaInstance():
    """Handly class to handle schema information"""

Pierre Dittgen's avatar
Pierre Dittgen committed
37
    def __init__(self, url=None, name=None, ref=None, spec=None, versions=None):
38
39
40
41
42
43
        """This function is not intended to be called directly
        but via from_parameters() static method!"""
        self.url = url
        self.name = name
        self.ref = ref
        self.spec = spec
Pierre Dittgen's avatar
Pierre Dittgen committed
44
        self.versions = versions
45
46

    @staticmethod
Pierre Dittgen's avatar
Pierre Dittgen committed
47
    def from_parameters(parameter_dict, schema_catalog_map):
48
49
        """Initializes schema instance from requests dict and tableschema catalog (for name ref)
        """
Pierre Dittgen's avatar
Pierre Dittgen committed
50
        schema_url, schema_name, schema_ref, versions = None, None, None, None
51
52
53
54
55
56
57
58
59
60

        # From schema_url
        if 'schema_url' in parameter_dict:
            schema_url = parameter_dict["schema_url"]

        # from schema_name (and schema_ref)
        elif 'schema_name' in parameter_dict:
            schema_name = parameter_dict['schema_name']
            schema_ref = parameter_dict.get('schema_ref')

Pierre Dittgen's avatar
Pierre Dittgen committed
61
62
63
64
65
66
67
68
69
70
71
72
            # Check schema name
            chunks = schema_name.split('.')
            if len(chunks) != 2:
                return None

            section_code, ref_name = chunks

            # Look for schema catalog first
            table_schema_catalog = schema_catalog_map.get(section_code)
            if table_schema_catalog is None:
                return None

73
            # Unknown schema name?
Pierre Dittgen's avatar
Pierre Dittgen committed
74
            table_schema_reference = table_schema_catalog.reference_by_name.get(ref_name)
75
76
77
            if table_schema_reference is None:
                return None

Pierre Dittgen's avatar
Pierre Dittgen committed
78
79
80
81
82
            # Git refs
            if table_schema_reference:
                versions = table_schema_reference.get_refs()
            options = {'ref': schema_ref} if schema_ref else {}
            schema_url = table_schema_reference.get_schema_url(**options)
83
84
85
86
87

        # else???
        else:
            return None

Pierre Dittgen's avatar
Pierre Dittgen committed
88
        return SchemaInstance(schema_url, schema_name, schema_ref, schema_from_url(schema_url), versions)
89
90
91
92
93
94
95
96
97
98
99
100

    def request_parameters(self):
        if self.name:
            return {
                'schema_name': self.name,
                'schema_ref': '' if self.ref is None else self.ref
            }
        return {
            'schema_url': self.url
        }


101
def extract_source_data(source: ValidataResource, preview_rows_nb=5):
102
    """ Computes table preview """
103
104
105

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
106
        return '' if val is None else str(val)
107

108
109
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
110
    nb_rows = 0
111

112
113
114
115
    # if source.format == "csv":
    #     options['delimiter'] = csv_helpers.detect_dialect(source.source, format=source.format, scheme=source.scheme,
    #                                                       custom_loaders=custom_loaders).delimiter
    tabulator_source, tabulator_options = source.build_tabulator_stream_args()
Pierre Dittgen's avatar
Pierre Dittgen committed
116
    with tabulator.Stream(tabulator_source, **tabulator_options) as stream:
117
118
        for row in stream:
            if header is None:
119
                header = ['' if v is None else v for v in row]
120
            else:
121
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
122
                nb_rows += 1
123
    preview_rows_nb = min(preview_rows_nb, nb_rows)
124
125
    return {'header': header,
            'rows_nb': nb_rows,
126
127
128
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
129
130


Pierre Dittgen's avatar
Pierre Dittgen committed
131
132
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
133

Pierre Dittgen's avatar
Pierre Dittgen committed
134
135
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
136

Pierre Dittgen's avatar
Pierre Dittgen committed
137
138
139
140
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
141

Pierre Dittgen's avatar
Pierre Dittgen committed
142
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
143

Pierre Dittgen's avatar
Pierre Dittgen committed
144
145
146
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
147

Pierre Dittgen's avatar
Pierre Dittgen committed
148
149
150
151
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
152

Pierre Dittgen's avatar
Pierre Dittgen committed
153
154
155
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
156

Pierre Dittgen's avatar
Pierre Dittgen committed
157
158
159
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
160

Pierre Dittgen's avatar
Pierre Dittgen committed
161
        return {**err, **update_keys}
162

Pierre Dittgen's avatar
Pierre Dittgen committed
163
    return list(map(improve_err, errors))
164
165


Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
166
def create_validata_ui_report(validata_core_report, schema_dict):
167
168
169
170
171
172
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
173
        - error messages are improved
174
    """
Pierre Dittgen's avatar
Pierre Dittgen committed
175
    report = copy.deepcopy(validata_core_report)
176
177
178
179
180
181
182
183
184

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
185
186
187
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
188
189
190
191
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
192
    # Computes column info
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
193
194
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', ''))
                   for f in schema_dict.get('fields', [])}
Pierre Dittgen's avatar
Pierre Dittgen committed
195
196
197
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
198

199
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
200
201
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
202

203
204
205
206
207
208
209
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
210
        if err['tag'] == 'structure':
211
212
213
214
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
215
    # Checks if there are structure errors different to invalid-column-delimiter
Pierre Dittgen's avatar
Pierre Dittgen committed
216
217
218
    structure_errors = report['table']['errors']['structure']
    report['table']['do_display_body_errors'] = len(structure_errors) == 0 or \
        all(err['code'] == 'invalid-column-delimiter' for err in structure_errors)
219

Pierre Dittgen's avatar
Pierre Dittgen committed
220
221
222
223
224
225
226
227
    # Checks if a column comparison is needed
    header_errors = ('missing-headers', 'extra-headers', 'wrong-headers-order')
    structure_errors = [{**err, 'in_column_comp': err['code'] in header_errors} for err in structure_errors]
    report['table']['errors']['structure'] = structure_errors
    column_comparison_needed = any(err['in_column_comp'] == True for err in structure_errors)
    column_comparison_table = []
    if column_comparison_needed:
        column_comparison_table = []
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
228
        field_names = [f['name'] for f in schema_dict.get('fields', [])]
229
        has_case_errors = False
Pierre Dittgen's avatar
Pierre Dittgen committed
230
231
        for t in itertools.zip_longest(headers, field_names, fillvalue=''):
            status = 'ok' if t[0] == t[1] else 'ko'
232
233
            if not has_case_errors and status == 'ko' and t[0].lower() == t[1].lower():
                has_case_errors = True
Pierre Dittgen's avatar
Pierre Dittgen committed
234
            column_comparison_table.append((*t, status))
235
236
237
238
239
        info = {}
        info['table'] = column_comparison_table
        info['has_missing'] = len(headers) < len(field_names)
        info['has_case_errors'] = has_case_errors
        report['table']['column_comparison_info'] = info
Pierre Dittgen's avatar
Pierre Dittgen committed
240
241
    report['table']['column_comparison_needed'] = column_comparison_needed

Pierre Dittgen's avatar
Pierre Dittgen committed
242
243
244
245
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
246
247
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
248
249
250
251
252
253
254
255
256
257
258
259
260
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
261
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
262

263
264
265
266
267
268
269
270
271
    # Sort by error names in statistics
    stats = report['table']['error-stats']
    code_title_map = messages.ERROR_MESSAGE_DEFAULT_TITLE
    for key in ('structure-errors', 'value-errors'):
        # convert dict into tuples with french title instead of error code
        # and sorts by title
        stats[key]['count-by-code'] = sorted([(code_title_map.get(k, k), v) for k, v in stats[key]['count-by-code'].items()],
                                             key=itemgetter(0))

Pierre Dittgen's avatar
Pierre Dittgen committed
272
273
274
    return report


Pierre Dittgen's avatar
Pierre Dittgen committed
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
    structure = badge['structure']
    body = badge.get('body')

    # Bad structure, stop here
    if structure == 'KO':
        return (
            'structure invalide', 'red')

    # No body error
    if body == 'OK':
        return ('structure invalide', 'orange') if structure == 'WARN' else ('valide', 'green')

    # else compute quality ratio percent
Christophe Benz's avatar
Christophe Benz committed
290
    p = (1 - badge['error-ratio']) * 100.0
Pierre Dittgen's avatar
Pierre Dittgen committed
291
292
293
294
295
296
297
298
    msg = 'cellules valides : {:.1f}%'.format(p)
    return (msg, 'red') if body == 'KO' else (msg, 'orange')


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
299
300
    return ('{}static/v1.svg?label=Validata&message={}&color={}'.format(
        config.SHIELDS_IO_BASE_URL, quote_plus(msg), color), msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
301
302


303
def validate(schema_instance: SchemaInstance, source: ValidataResource):
304
305
    """ Validate source and display report """

306
    # Validation is done through http call to validata-api
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
307
308
309
310
    if config.API_VALIDATE_ENDPOINT is None:
        flash_error("No Validate endpoint defined :-(")
        return redirect(url_for("custom_validator"))
    api_url = config.API_VALIDATE_ENDPOINT
311
312

    # Useful to receive response as JSON
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
313
314
    headers = {"Accept": "application/json"}

Pierre Dittgen's avatar
Pierre Dittgen committed
315
    try:
316
        if source.type == 'url':
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
317
            params = {
318
                "schema": schema_instance.url,
319
                "url": source.url,
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
320
321
322
323
            }
            req = requests.get(api_url, params=params, headers=headers)

        else:
324
            files = {'file': (source.filename, source.build_reader())}
325
            data = {"schema": schema_instance.url}
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
326
327
            req = requests.post(api_url, data=data, files=files, headers=headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
328
329
330
        # 400
        if req.status_code == 400:
            json_response = req.json()
331
332
333
334
            if json_response.get('message') == 'Missing headers':
                flash_error("Impossible d'extraire les en-têtes du fichier tabulaire")
                return redirect(compute_validation_form_url(schema_instance))

Pierre Dittgen's avatar
Pierre Dittgen committed
335
336
            flash_error("Une erreur est survenue durant la validation: {}"
                        .format(json_response.get('message')))
337
            return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
Pierre Dittgen committed
338

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
339
        if not req.ok:
Pierre Dittgen's avatar
Pierre Dittgen committed
340
            flash_error("Un erreur s'est produite côté serveur :-(")
341
            return redirect(compute_validation_form_url(schema_instance))
342

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
343
344
345
        json_response = req.json()
        validata_core_report = json_response['report']
        schema_dict = json_response['schema']
346

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
347
348
349
350
    except requests.ConnectionError as err:
        logging.exception(err)
        flash_error(str(err))
        return redirect(url_for('home'))
351

Pierre Dittgen's avatar
Pierre Dittgen committed
352
    # Computes badge from report and badge configuration
353
    badge = compute_badge(validata_core_report, config.BADGE_CONFIG)
Pierre Dittgen's avatar
Pierre Dittgen committed
354
355
    badge_url, badge_msg = get_badge_url_and_message(badge)

356
357
358
359
360
361
    source_errors = [err for err in validata_core_report['tables'][0]['errors'] if err['code'] == 'source-error']
    if source_errors:
        err = source_errors[0]
        msg = "l'encodage du fichier est invalide. Veuillez le corriger" if 'charmap' in err[
            'message'] else err['message']
        flash_error('Erreur de source : {}'.format(msg))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
362
        return redirect(url_for('custom_validator'))
363

364
    source_data = extract_source_data(source)
365

Pierre Dittgen's avatar
Pierre Dittgen committed
366
    # handle report date
367
    report_datetime = datetime.fromisoformat(validata_core_report['date']).astimezone()
368

Pierre Dittgen's avatar
Pierre Dittgen committed
369
    # Enhance validata_core_report
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
370
    validata_report = create_validata_ui_report(validata_core_report, schema_dict)
Pierre Dittgen's avatar
Pierre Dittgen committed
371

Pierre Dittgen's avatar
Pierre Dittgen committed
372
    # Display report to the user
373
    validator_form_url = compute_validation_form_url(schema_instance)
Pierre Dittgen's avatar
Pierre Dittgen committed
374
    schema_info, validator_title = compute_schema_info(schema_instance.spec, schema_instance.url)
375
    pdf_report_url = url_for('pdf_report')+'?'+urlencode(schema_instance.request_parameters())
376
    return render_template('validation_report.html', title='Rapport de validation',
377
378
                           schema_info=schema_info, report=validata_report,
                           pdf_report_url=pdf_report_url,
379
                           validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
Pierre Dittgen's avatar
Pierre Dittgen committed
380
                           source=source, source_data=source_data,
Pierre Dittgen's avatar
Pierre Dittgen committed
381
                           schema_current_version=schema_instance.ref,
382
                           print_mode=request.args.get('print', 'false') == 'true',
Pierre Dittgen's avatar
Pierre Dittgen committed
383
                           badge_url=badge_url, badge_msg=badge_msg,
384
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
Pierre Dittgen's avatar
Pierre Dittgen committed
385
                           breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
Pierre Dittgen's avatar
Pierre Dittgen committed
386
                                        {'url': validator_form_url, 'title': validator_title},
Pierre Dittgen's avatar
Pierre Dittgen committed
387
                                        ])
388
389


390
391
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
392
    iob = io.BytesIO()
393
394
395
396
397
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


Pierre Dittgen's avatar
Pierre Dittgen committed
398
399
400
401
402
403
404
405
406
407
408
409
410
def ui_config_with_schema_metadata(ui_config, schema_catalog_map):
    """Replace catalog url within ui_config by schema references
    containing schema metadata properties"""

    extended_ui_config = ui_config.copy()
    for section in extended_ui_config['sections']:
        section_code = section['code']
        if section_code not in schema_catalog_map:
            continue
        schema_catalog = schema_catalog_map[section_code]
        schema_list = []
        for ref in schema_catalog.references:
            # Loads default table schema for each schema reference
411
            table_schema = schema_from_url(ref.get_schema_url(check_exists=False))
Pierre Dittgen's avatar
Pierre Dittgen committed
412
413
414
415
416
417
418
            schema_list.append({
                'name': '{}.{}'.format(section_code, ref.name),
                # Extracts title, description, ...
                **extract_schema_metadata(table_schema)
            })
        section['catalog'] = schema_list
    return extended_ui_config
419

420
421
422
423
424
425
# Routes


@app.route('/')
def home():
    """ Home page """
Christophe Benz's avatar
Christophe Benz committed
426
    flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu.')
Pierre Dittgen's avatar
Pierre Dittgen committed
427
428
    home_config = ui_config_with_schema_metadata(ui_config, schema_catalog_map)
    return render_template('home.html', title='Accueil', config=home_config)
429
430


Pierre Dittgen's avatar
Pierre Dittgen committed
431
432
@app.route('/pdf')
def pdf_report():
433
    """PDF report generation"""
434
    err_prefix = 'Erreur de génération du rapport PDF'
435
436
437

    url_param = request.args.get('url')
    if not url_param:
438
        flash_error(err_prefix + ': URL non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
439
        return redirect(url_for('home'))
440

Pierre Dittgen's avatar
Pierre Dittgen committed
441
    schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
442
443
    if schema_instance is None:
        flash_error(err_prefix + ': Information de schema non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
444
445
        return redirect(url_for('home'))

446
447
448
449
450
451
452
453
454
455
456
    # Compute pdf url report
    base_url = url_for('custom_validator', _external=True)
    parameter_dict = {
        'input': 'url',
        'print': 'true',
        'url': url_param,
        **schema_instance.request_parameters()
    }
    validation_url = base_url + '?' + urlencode(parameter_dict)

    # Create temp file to save validation report
Pierre Dittgen's avatar
Pierre Dittgen committed
457
    with tempfile.NamedTemporaryFile(prefix='validata_{}_report_'.format(datetime.now().timestamp()), suffix='.pdf') as tmpfile:
Christophe Benz's avatar
Christophe Benz committed
458
        tmp_pdf_report = Path(tmpfile.name)
459

460
    # Use chromium headless to generate PDF from validation report page
461
462
463
464
    cmd = ['chromium', '--headless', '--disable-gpu',
           '--print-to-pdf={}'.format(str(tmp_pdf_report)), validation_url]
    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if result.returncode != 0:
465
466
        flash_error(err_prefix)
        log.error("Command %r returned an error: %r", cmd, result.stdout.decode('utf-8'))
467
468
        if tmp_pdf_report.exists():
            tmp_pdf_report.unlink()
Pierre Dittgen's avatar
Pierre Dittgen committed
469
        return redirect(url_for('home'))
470

471
    # Send PDF report
Pierre Dittgen's avatar
Pierre Dittgen committed
472
    pdf_filename = 'Rapport de validation {}.pdf'.format(datetime.now().strftime('%d-%m-%Y %Hh%M'))
Christophe Benz's avatar
Christophe Benz committed
473
    response = make_response(tmp_pdf_report.read_bytes())
Pierre Dittgen's avatar
Pierre Dittgen committed
474
    response.headers.set('Content-disposition', 'attachment', filename=pdf_filename)
475
476
477
478
479
480
481
482
    response.headers.set('Content-type', 'application/pdf')
    response.headers.set('Content-length', tmp_pdf_report.stat().st_size)

    tmp_pdf_report.unlink()

    return response


Pierre Dittgen's avatar
Pierre Dittgen committed
483
484
485
486
487
def extract_schema_metadata(table_schema: tableschema.Schema):
    """Gets author, contibutor, version...metadata from schema header"""
    return {k: v for k, v in table_schema.descriptor.items() if k != 'fields'}


Pierre Dittgen's avatar
Pierre Dittgen committed
488
def compute_schema_info(table_schema: tableschema.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
489
    """Factor code for validator form page"""
490

Pierre Dittgen's avatar
Pierre Dittgen committed
491
492
493
494
495
496
497
498
    # Schema URL + schema metadata info
    schema_info = {
        'url': schema_url,
        **extract_schema_metadata(table_schema)
    }
    meta_title = schema_info.get('title')

    title = "Schéma « {} »".format(meta_title if meta_title else '...')
499
500
501
502
503
504
505
506
507
    return schema_info, title


def compute_validation_form_url(schema_instance: SchemaInstance):
    """Computes validation form url with schema URL parameter"""
    url = url_for('custom_validator')
    param_list = ['{}={}'.format(k, quote_plus(v))
                  for k, v in schema_instance.request_parameters().items()]
    return "{}?{}".format(url, '&'.join(param_list))
Pierre Dittgen's avatar
Pierre Dittgen committed
508
509


510
@app.route('/table_schema', methods=['GET', 'POST'])
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
511
def custom_validator():
512
    """Validator form"""
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
513

514
    # Check that validata-api URL is set
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
515
516
517
518
519
520
    if config.API_VALIDATE_ENDPOINT is None:
        flash_error("URL de connexion à l'API non indiquée :-(")
        return redirect(url_for('home'))

    if request.method == 'GET':

521
522
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
523
        input_param = request.args.get('input')
524
525

        # url of resource to be validated
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
526
527
        url_param = request.args.get("url")

Pierre Dittgen's avatar
Pierre Dittgen committed
528
        schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
529
530
        if schema_instance is None:
            flash_error("Aucun schéma passé en paramètre")
Pierre Dittgen's avatar
Pierre Dittgen committed
531
532
            return redirect(url_for('home'))

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
533
534
        # First form display
        if input_param is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
535
536
537

            schema_versions = schema_instance.versions
            schema_info, title = compute_schema_info(schema_instance.spec, schema_instance.url)
Pierre Dittgen's avatar
Pierre Dittgen committed
538
            return render_template('validation_form.html', title=title,
Pierre Dittgen's avatar
Pierre Dittgen committed
539
540
                                   schema_info=schema_info, schema_versions=schema_versions,
                                   schema_current_version=schema_instance.ref,
541
                                   schema_params=schema_instance.request_parameters(),
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
542
543
544
545
546
547
                                   breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])

        # Process URL
        else:
            if url_param is None or url_param == '':
                flash_error("Vous n'avez pas indiqué d'url à valider")
548
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
549
            try:
550
                return validate(schema_instance, URLValidataResource(url_param))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
551
552
553
            except tabulator.exceptions.FormatError as e:
                flash_error('Erreur : Format de ressource non supporté')
                log.info(e)
554
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
555
556
557
            except tabulator.exceptions.HTTPError as e:
                flash_error('Erreur : impossible d\'accéder au fichier source en ligne')
                log.info(e)
558
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
559
560

    else:  # POST
561

Pierre Dittgen's avatar
Pierre Dittgen committed
562
        schema_instance = SchemaInstance.from_parameters(request.form, schema_catalog_map)
563
        if schema_instance is None:
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
564
            flash_error('Aucun schéma défini')
Pierre Dittgen's avatar
Pierre Dittgen committed
565
566
567
568
569
            return redirect(url_for('home'))

        input_param = request.form.get('input')
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
570
            return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
571
572
573
574
575
576

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
577
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
578

579
            return validate(schema_instance, UploadedFileValidataResource(f.filename, bytes_data(f)))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
580
581

        return 'Bizarre, vous avez dit bizarre ?'