views.py 22.8 KB
Newer Older
1
2
3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
5
import io
6
import json
Christophe Benz's avatar
Christophe Benz committed
7
import logging
8
import subprocess
Christophe Benz's avatar
Christophe Benz committed
9
import tempfile
10
from datetime import datetime
11
from operator import itemgetter
12
from pathlib import Path
Christophe Benz's avatar
Christophe Benz committed
13
from urllib.parse import urlencode, urljoin
14

15
import jsonschema
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
16
import requests
17
import tableschema
18
from backports.datetime_fromisoformat import MonkeyPatch
19
from commonmark import commonmark
20
from flask import abort, make_response, redirect, render_template, request, url_for
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
21

Pierre Dittgen's avatar
Pierre Dittgen committed
22
import tabulator
23
from opendataschema import GitSchemaReference, by_commit_date
Pierre Dittgen's avatar
Pierre Dittgen committed
24
from validata_core import messages, repair
25

26
from . import app, config, schema_catalog_registry, tableschema_from_url
27
from .ui_util import flash_error, flash_warning
28
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource, strip_accents
29

30
31
MonkeyPatch.patch_fromisoformat()

Christophe Benz's avatar
Christophe Benz committed
32
33
log = logging.getLogger(__name__)

34

35
36
37
38
39
def get_schema_catalog(section_name):
    """Return a schema catalog associated to a section_name"""
    return schema_catalog_registry.build_schema_catalog(section_name)


40
41
42
class SchemaInstance:
    """Handy class to handle schema information"""

43
    def __init__(self, parameter_dict):
44
45
46
47
48
49
50
51
52
53
        """Initializes schema instance from requests dict and tableschema catalog (for name ref)"""
        self.section_name = None
        self.section_title = None
        self.name = None
        self.url = None
        self.ref = None
        self.reference = None
        self.doc_url = None
        self.branches = None
        self.tags = None
54
55

        # From schema_url
Christophe Benz's avatar
Christophe Benz committed
56
        if parameter_dict.get("schema_url"):
57
58
            self.url = parameter_dict["schema_url"]
            self.section_title = "Autre schéma"
59
60

        # from schema_name (and schema_ref)
Christophe Benz's avatar
Christophe Benz committed
61
        elif parameter_dict.get('schema_name'):
62
63
            self.schema_and_section_name = parameter_dict['schema_name']
            self.ref = parameter_dict.get('schema_ref')
64

Pierre Dittgen's avatar
Pierre Dittgen committed
65
            # Check schema name
66
            chunks = self.schema_and_section_name.split('.')
Pierre Dittgen's avatar
Pierre Dittgen committed
67
            if len(chunks) != 2:
68
                abort(400, "Paramètre 'schema_name' invalide")
Pierre Dittgen's avatar
Pierre Dittgen committed
69

70
71
            self.section_name, self.name = chunks
            self.section_title = self.find_section_title(self.section_name)
Pierre Dittgen's avatar
Pierre Dittgen committed
72
73

            # Look for schema catalog first
74
75
76
77
78
            try:
                table_schema_catalog = get_schema_catalog(self.section_name)
            except Exception as ex:
                log.exception(ex)
                abort(400, "Erreur de traitement du catalogue")
Pierre Dittgen's avatar
Pierre Dittgen committed
79
            if table_schema_catalog is None:
80
                abort(400, "Catalogue indisponible")
81
82
83
84
85
86

            schema_reference = table_schema_catalog.reference_by_name.get(self.name)
            if schema_reference is None:
                abort(400, "Schéma '{}' non trouvé dans le catalogue de la section '{}'".format(self.name, self.section_name))

            if isinstance(schema_reference, GitSchemaReference):
87
                self.tags = sorted(schema_reference.iter_tags(), key=by_commit_date, reverse=True)
88
                if self.ref is None:
89
                    schema_ref = self.tags[0] if self.tags else schema_reference.get_default_branch()
90
91
92
93
94
95
96
                    abort(redirect(compute_validation_form_url({
                        'schema_name': self.schema_and_section_name,
                        'schema_ref': schema_ref.name
                    })))
                tag_names = [tag.name for tag in self.tags]
                self.branches = [branch for branch in schema_reference.iter_branches()
                                 if branch.name not in tag_names]
97
98
                self.doc_url = schema_reference.get_doc_url(ref=self.ref) or \
                    schema_reference.get_project_url(ref=self.ref)
99

100
            self.url = schema_reference.get_schema_url(ref=self.ref)
101
102

        else:
103
            abort(400, "L'un des paramètres est nécessaire : 'schema_name', 'schema_url'")
104

105
106
107
108
109
110
111
112
113
114
        try:
            self.schema = tableschema_from_url(self.url)
        except json.JSONDecodeError as e:
            log.exception(e)
            flash_error("Format de schéma non reconnu")
            abort(redirect(url_for('home')))
        except Exception as e:
            log.exception(e)
            flash_error("Erreur lors de l'obtention du schéma")
            abort(redirect(url_for('home')))
115
116
117
118

    def request_parameters(self):
        if self.name:
            return {
119
                'schema_name': self.schema_and_section_name,
120
121
122
123
124
125
                'schema_ref': '' if self.ref is None else self.ref
            }
        return {
            'schema_url': self.url
        }

126
    def find_section_title(self, section_name):
Christophe Benz's avatar
Christophe Benz committed
127
128
129
130
131
132
        if config.HOMEPAGE_CONFIG:
            for section in config.HOMEPAGE_CONFIG['sections']:
                if section["name"] == section_name:
                    return section.get("title")
        return None

133

Pierre Dittgen's avatar
Pierre Dittgen committed
134
def extract_source_data(source: ValidataResource, schema_descriptor, preview_rows_nb=5):
135
    """ Computes table preview """
136
137
138

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
139
        return '' if val is None else str(val)
140

141
142
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
143
    nb_rows = 0
144

145
    tabulator_source, tabulator_options = source.build_tabulator_stream_args()
Pierre Dittgen's avatar
Pierre Dittgen committed
146
147
    fixed_source, _ = repair(tabulator_source, schema_descriptor, **tabulator_options)
    with tabulator.Stream(fixed_source, {**tabulator_options, 'scheme': 'stream', 'format': 'inline'}) as stream:
148
149
        for row in stream:
            if header is None:
150
                header = ['' if v is None else v for v in row]
151
            else:
152
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
153
                nb_rows += 1
154
    preview_rows_nb = min(preview_rows_nb, nb_rows)
155
156
    return {'header': header,
            'rows_nb': nb_rows,
157
158
159
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
160
161


Pierre Dittgen's avatar
Pierre Dittgen committed
162
163
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
164

Pierre Dittgen's avatar
Pierre Dittgen committed
165
166
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
167

Pierre Dittgen's avatar
Pierre Dittgen committed
168
169
170
171
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
172

Pierre Dittgen's avatar
Pierre Dittgen committed
173
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
174

Pierre Dittgen's avatar
Pierre Dittgen committed
175
176
177
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
178

Pierre Dittgen's avatar
Pierre Dittgen committed
179
180
181
182
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
183

Pierre Dittgen's avatar
Pierre Dittgen committed
184
185
186
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
187

Pierre Dittgen's avatar
Pierre Dittgen committed
188
189
190
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
191

Pierre Dittgen's avatar
Pierre Dittgen committed
192
        return {**err, **update_keys}
193

Pierre Dittgen's avatar
Pierre Dittgen committed
194
    return list(map(improve_err, errors))
195
196


Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
197
def create_validata_ui_report(validata_core_report, schema_dict):
198
199
200
201
202
203
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
204
        - error messages are improved
205
    """
Pierre Dittgen's avatar
Pierre Dittgen committed
206
    report = copy.deepcopy(validata_core_report)
207
208
209
210
211
212
213
214
215

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
216
217
218
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
219
220
221
222
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
223
    # Computes column info
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
224
225
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', ''))
                   for f in schema_dict.get('fields', [])}
Pierre Dittgen's avatar
Pierre Dittgen committed
226
227
228
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
229

230
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
231
232
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
233

234
235
236
237
238
239
240
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
241
        if err['tag'] == 'structure':
242
243
244
245
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
246
247
248
249
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
250
251
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
252
253
254
255
256
257
258
259
260
261
262
263
264
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
265
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
266

267
268
269
270
271
272
    # Sort by error names in statistics
    stats = report['table']['error-stats']
    code_title_map = messages.ERROR_MESSAGE_DEFAULT_TITLE
    for key in ('structure-errors', 'value-errors'):
        # convert dict into tuples with french title instead of error code
        # and sorts by title
Christophe Benz's avatar
Christophe Benz committed
273
274
        stats[key]['count-by-code'] = sorted(((code_title_map.get(k, k), v)
                                              for k, v in stats[key]['count-by-code'].items()), key=itemgetter(0))
275

Pierre Dittgen's avatar
Pierre Dittgen committed
276
277
278
    return report


Pierre Dittgen's avatar
Pierre Dittgen committed
279
280
281
282
283
284
285
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
    structure = badge['structure']
    body = badge.get('body')

    # Bad structure, stop here
    if structure == 'KO':
286
        return ('structure invalide', 'red')
Pierre Dittgen's avatar
Pierre Dittgen committed
287
288
289
290
291
292

    # No body error
    if body == 'OK':
        return ('structure invalide', 'orange') if structure == 'WARN' else ('valide', 'green')

    # else compute quality ratio percent
Christophe Benz's avatar
Christophe Benz committed
293
    p = (1 - badge['error-ratio']) * 100.0
Pierre Dittgen's avatar
Pierre Dittgen committed
294
295
296
297
298
299
300
301
    msg = 'cellules valides : {:.1f}%'.format(p)
    return (msg, 'red') if body == 'KO' else (msg, 'orange')


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
Christophe Benz's avatar
Christophe Benz committed
302
303
304
305
306
    badge_url = "{}?{}".format(
        urljoin(config.SHIELDS_IO_BASE_URL, '/static/v1.svg'),
        urlencode({"label": "Validata", "message": msg, "color":  color}),
    )
    return (badge_url, msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
307
308


309
def validate(schema_instance: SchemaInstance, source: ValidataResource):
310
311
    """ Validate source and display report """

312
    # Useful to receive response as JSON
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
313
314
    headers = {"Accept": "application/json"}

Pierre Dittgen's avatar
Pierre Dittgen committed
315
    try:
316
        if source.type == 'url':
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
317
            params = {
318
                "schema": schema_instance.url,
319
                "url": source.url,
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
320
            }
321
            response = requests.get(config.API_VALIDATE_ENDPOINT, params=params, headers=headers)
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
322
        else:
323
            files = {'file': (source.filename, source.build_reader())}
324
            data = {"schema": schema_instance.url}
325
            response = requests.post(config.API_VALIDATE_ENDPOINT, data=data, files=files, headers=headers)
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
326
327
    except requests.ConnectionError as err:
        logging.exception(err)
Christophe Benz's avatar
Christophe Benz committed
328
        flash_error("Erreur technique lors de la validation")
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
329
        return redirect(url_for('home'))
330

331
332
333
    if not response.ok:
        flash_error("Erreur technique lors de la validation")
        return redirect(compute_validation_form_url(schema_instance.request_parameters()))
334

335
    json_response = response.json()
336
    validata_core_report = json_response['report']
Pierre Dittgen's avatar
Pierre Dittgen committed
337
    badge_info = json_response.get('badge')
338

Pierre Dittgen's avatar
Pierre Dittgen committed
339
    # Computes badge from report and badge configuration
Pierre Dittgen's avatar
Pierre Dittgen committed
340
341
342
343
    badge_url, badge_msg = None, None
    display_badge = badge_info and config.SHIELDS_IO_BASE_URL
    if display_badge:
        badge_url, badge_msg = get_badge_url_and_message(badge_info)
Pierre Dittgen's avatar
Pierre Dittgen committed
344

345
346
347
348
349
    source_errors = [
        err
        for err in validata_core_report['tables'][0]['errors']
        if err['code'] in {'source-error', 'unknown-csv-dialect'}
    ]
350
351
352
353
354
    if source_errors:
        err = source_errors[0]
        msg = "l'encodage du fichier est invalide. Veuillez le corriger" if 'charmap' in err[
            'message'] else err['message']
        flash_error('Erreur de source : {}'.format(msg))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
355
        return redirect(url_for('custom_validator'))
356

Pierre Dittgen's avatar
Pierre Dittgen committed
357
    source_data = extract_source_data(source, schema_instance.schema.descriptor)
358

Pierre Dittgen's avatar
Pierre Dittgen committed
359
    # handle report date
360
    report_datetime = datetime.fromisoformat(validata_core_report['date']).astimezone()
361

Pierre Dittgen's avatar
Pierre Dittgen committed
362
    # Enhance validata_core_report
Pierre Dittgen's avatar
Pierre Dittgen committed
363
    validata_report = create_validata_ui_report(validata_core_report, schema_instance.schema.descriptor)
Pierre Dittgen's avatar
Pierre Dittgen committed
364

Pierre Dittgen's avatar
Pierre Dittgen committed
365
    # Display report to the user
366
    validator_form_url = compute_validation_form_url(schema_instance.request_parameters())
Christophe Benz's avatar
Christophe Benz committed
367
    schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
Christophe Benz's avatar
Christophe Benz committed
368
369
370
371
    pdf_report_url = "{}?{}".format(url_for('pdf_report'),
                                    urlencode({
                                        **schema_instance.request_parameters(),
                                        "url": source.url,
372
                                    })) if source.type == 'url' else None
373

Christophe Benz's avatar
Christophe Benz committed
374
    return render_template('validation_report.html',
375
376
377
378
379
380
381
382
383
                           badge_msg=badge_msg,
                           badge_url=badge_url,
                           breadcrumbs=[
                               {'title': 'Accueil', 'url': url_for('home')},
                               {'title': schema_instance.section_title},
                               {'title': schema_info['title'], 'url': validator_form_url},
                               {'title': 'Rapport de validation'},
                           ],
                           display_badge=display_badge,
384
                           doc_url=schema_instance.doc_url,
385
                           pdf_report_url=pdf_report_url,
386
                           print_mode=request.args.get('print', 'false') == 'true',
387
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
388
389
390
                           report=validata_report,
                           schema_current_version=schema_instance.ref,
                           schema_info=schema_info,
Christophe Benz's avatar
Christophe Benz committed
391
                           section_title=schema_instance.section_title,
392
393
394
395
                           source_data=source_data,
                           source=source,
                           validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
                           )
396
397


398
399
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
400
    iob = io.BytesIO()
401
402
403
404
405
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


406
407
408
409
410
411
# Routes


@app.route('/')
def home():
    """ Home page """
412
413
414
415
416
417

    def iter_sections():
        """Yield sections of the home page, filled with schema metadata."""
        if not config.HOMEPAGE_CONFIG:
            return
        for section in config.HOMEPAGE_CONFIG['sections']:
418
            home_section = {k: v for k, v in section.items() if k != 'catalog'}
419
            if "catalog" in section:
420
421
422
423
424
425
426
427
428
429

                try:
                    schema_catalog = get_schema_catalog(section['name'])
                except Exception as exc:
                    log.exception(exc)
                    err_msg = "une erreur s'est produite"
                    if isinstance(exc, requests.ConnectionError):
                        err_msg = "problème de connexion"
                    elif isinstance(exc, json.decoder.JSONDecodeError):
                        err_msg = "format JSON incorrect"
430
431
                    elif isinstance(exc, jsonschema.exceptions.ValidationError):
                        err_msg = "le catalogue ne respecte pas le schéma de référence"
432
433
                    home_section['err'] = err_msg
                else:
434
435
436
                    home_section_catalog = []
                    for schema_reference in schema_catalog.references:
                        # Loads default table schema for each schema reference
437
438
439
440
441
442
443
444
445
446
447
                        schema_info = {
                            'name': schema_reference.name
                        }
                        try:
                            table_schema = tableschema_from_url(schema_reference.get_schema_url())
                        except:
                            schema_info['err'] = True
                            schema_info['title'] = 'Schéma "{}" non disponible'.format(schema_reference.name)
                        else:
                            schema_info['title'] = table_schema.descriptor.get("title") or schema_reference.name
                        home_section_catalog.append(schema_info)
448
449
                    home_section['catalog'] = sorted(
                        home_section_catalog, key=lambda sc: strip_accents(sc['title'].lower()))
450

451
452
453
454
455
            if "links" in section:
                home_section["links"] = section["links"]
            yield home_section

    return render_template('home.html', sections=list(iter_sections()))
456
457


Pierre Dittgen's avatar
Pierre Dittgen committed
458
459
@app.route('/pdf')
def pdf_report():
460
    """PDF report generation"""
461
    err_prefix = 'Erreur de génération du rapport PDF'
462
463
464

    url_param = request.args.get('url')
    if not url_param:
465
        flash_error(err_prefix + ' : URL non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
466
        return redirect(url_for('home'))
467

468
    schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
469

470
471
472
473
474
475
476
477
    # Compute pdf url report
    base_url = url_for('custom_validator', _external=True)
    parameter_dict = {
        'input': 'url',
        'print': 'true',
        'url': url_param,
        **schema_instance.request_parameters()
    }
Christophe Benz's avatar
Christophe Benz committed
478
    validation_url = "{}?{}".format(base_url, urlencode(parameter_dict))
479
480

    # Create temp file to save validation report
Pierre Dittgen's avatar
Pierre Dittgen committed
481
    with tempfile.NamedTemporaryFile(prefix='validata_{}_report_'.format(datetime.now().timestamp()), suffix='.pdf') as tmpfile:
Christophe Benz's avatar
Christophe Benz committed
482
        tmp_pdf_report = Path(tmpfile.name)
483

484
    # Use chromium headless to generate PDF from validation report page
485
    cmd = ['chromium', '--headless', '--no-sandbox',
486
487
488
           '--print-to-pdf={}'.format(str(tmp_pdf_report)), validation_url]
    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if result.returncode != 0:
489
490
        flash_error(err_prefix)
        log.error("Command %r returned an error: %r", cmd, result.stdout.decode('utf-8'))
491
492
        if tmp_pdf_report.exists():
            tmp_pdf_report.unlink()
Pierre Dittgen's avatar
Pierre Dittgen committed
493
        return redirect(url_for('home'))
494

495
    # Send PDF report
Pierre Dittgen's avatar
Pierre Dittgen committed
496
    pdf_filename = 'Rapport de validation {}.pdf'.format(datetime.now().strftime('%d-%m-%Y %Hh%M'))
Christophe Benz's avatar
Christophe Benz committed
497
    response = make_response(tmp_pdf_report.read_bytes())
Pierre Dittgen's avatar
Pierre Dittgen committed
498
    response.headers.set('Content-disposition', 'attachment', filename=pdf_filename)
499
500
501
502
503
504
505
506
    response.headers.set('Content-type', 'application/pdf')
    response.headers.set('Content-length', tmp_pdf_report.stat().st_size)

    tmp_pdf_report.unlink()

    return response


Pierre Dittgen's avatar
Pierre Dittgen committed
507
508
509
510
511
def extract_schema_metadata(table_schema: tableschema.Schema):
    """Gets author, contibutor, version...metadata from schema header"""
    return {k: v for k, v in table_schema.descriptor.items() if k != 'fields'}


Pierre Dittgen's avatar
Pierre Dittgen committed
512
def compute_schema_info(table_schema: tableschema.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
513
    """Factor code for validator form page"""
514

Pierre Dittgen's avatar
Pierre Dittgen committed
515
516
    # Schema URL + schema metadata info
    schema_info = {
Christophe Benz's avatar
Christophe Benz committed
517
518
519
        'path': schema_url,
        # a "path" metadata property can be found in Table Schema, and we'd like it to override the `schema_url`
        # given by the user (in case schema was given by URL)
Pierre Dittgen's avatar
Pierre Dittgen committed
520
521
        **extract_schema_metadata(table_schema)
    }
Christophe Benz's avatar
Christophe Benz committed
522
    return schema_info
523
524


525
def compute_validation_form_url(request_parameters: dict):
526
527
    """Computes validation form url with schema URL parameter"""
    url = url_for('custom_validator')
Christophe Benz's avatar
Christophe Benz committed
528
    return "{}?{}".format(url, urlencode(request_parameters))
Pierre Dittgen's avatar
Pierre Dittgen committed
529
530


531
@app.route('/table-schema', methods=['GET', 'POST'])
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
532
def custom_validator():
533
    """Validator form"""
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
534
535
536

    if request.method == 'GET':

537
538
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
539
        input_param = request.args.get('input')
540
541

        # url of resource to be validated
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
542
543
        url_param = request.args.get("url")

544
        schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
545

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
546
547
        # First form display
        if input_param is None:
Christophe Benz's avatar
Christophe Benz committed
548
549
            schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
            return render_template('validation_form.html',
550
                                   branches=schema_instance.branches,
Christophe Benz's avatar
Christophe Benz committed
551
552
553
554
                                   breadcrumbs=[
                                       {'url': url_for('home'), 'title': 'Accueil'},
                                       {'title': schema_instance.section_title},
                                       {'title': schema_info['title']},
Christophe Benz's avatar
Christophe Benz committed
555
556
557
558
559
560
561
562
                                   ],
                                   doc_url=schema_instance.doc_url,
                                   schema_current_version=schema_instance.ref,
                                   schema_info=schema_info,
                                   schema_params=schema_instance.request_parameters(),
                                   section_title=schema_instance.section_title,
                                   tags=schema_instance.tags,
                                   )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
563
564
565

        # Process URL
        else:
566
            if not url_param:
Christophe Benz's avatar
Christophe Benz committed
567
                flash_error("Vous n'avez pas indiqué d'URL à valider")
568
569
570
571
572
                return redirect(compute_validation_form_url(schema_instance.request_parameters()))
            return validate(schema_instance, URLValidataResource(url_param))

    elif request.method == 'POST':

573
        schema_instance = SchemaInstance(request.form)
Pierre Dittgen's avatar
Pierre Dittgen committed
574
575
576
577

        input_param = request.form.get('input')
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
578
            return redirect(compute_validation_form_url(schema_instance.request_parameters()))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
579
580
581
582
583
584

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
585
                return redirect(compute_validation_form_url(schema_instance.request_parameters()))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
586

587
            return validate(schema_instance, UploadedFileValidataResource(f.filename, bytes_data(f)))
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
588

589
590
591
592
        return 'Combinaison de paramètres non supportée', 400

    else:
        return "Method not allowed", 405