views.py 22.8 KB
Newer Older
1 2 3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
5
import io
Pierre Dittgen's avatar
Pierre Dittgen committed
6
import itertools
7
import json
8
import logging
9
import subprocess
10
import tempfile
11
from datetime import datetime
12
from operator import itemgetter
13
from pathlib import Path
14
from urllib.parse import quote_plus, urlencode
15

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
16
import requests
17
import tableschema
18
import tabulator
19
from backports.datetime_fromisoformat import MonkeyPatch
20
from commonmark import commonmark
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
21
from flask import make_response, redirect, render_template, request, url_for
22

Pierre Dittgen's avatar
Pierre Dittgen committed
23
from validata_core import messages
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
24

25
from . import app, config, schema_catalog_map, schema_from_url
26
from .ui_util import flash_error, flash_warning
27
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
28

29 30
MonkeyPatch.patch_fromisoformat()

31 32
log = logging.getLogger(__name__)

33

34 35 36
class SchemaInstance():
    """Handly class to handle schema information"""

Christophe Benz's avatar
Christophe Benz committed
37
    def __init__(self, url=None, section_name=None, name=None, ref=None, schema=None, versions=None, doc_url=None):
38 39 40
        """This function is not intended to be called directly
        but via from_parameters() static method!"""
        self.url = url
Christophe Benz's avatar
Christophe Benz committed
41 42 43 44
        self.section_name = section_name
        self.section_title = self.get_section_title(section_name) \
            if section_name \
            else "Autre schéma"
45 46
        self.name = name
        self.ref = ref
Pierre Dittgen's avatar
Pierre Dittgen committed
47
        self.schema = schema
48
        self.versions = versions
49
        self.doc_url = doc_url
50 51

    @staticmethod
52
    def from_parameters(parameter_dict, schema_catalog_map):
53 54
        """Initializes schema instance from requests dict and tableschema catalog (for name ref)
        """
Christophe Benz's avatar
Christophe Benz committed
55
        schema_url, section_name, schema_name, schema_ref, versions, doc_url = None, None, None, None, None, None
56 57

        # From schema_url
Christophe Benz's avatar
Christophe Benz committed
58
        if parameter_dict.get("schema_url"):
59 60 61
            schema_url = parameter_dict["schema_url"]

        # from schema_name (and schema_ref)
Christophe Benz's avatar
Christophe Benz committed
62
        elif parameter_dict.get('schema_name'):
63 64 65
            schema_name = parameter_dict['schema_name']
            schema_ref = parameter_dict.get('schema_ref')

66 67 68 69 70
            # Check schema name
            chunks = schema_name.split('.')
            if len(chunks) != 2:
                return None

Christophe Benz's avatar
Christophe Benz committed
71
            section_name, ref_name = chunks
72 73

            # Look for schema catalog first
Christophe Benz's avatar
Christophe Benz committed
74
            table_schema_catalog = schema_catalog_map.get(section_name)
75 76 77
            if table_schema_catalog is None:
                return None

78
            # Unknown schema name?
79
            table_schema_reference = table_schema_catalog.reference_by_name.get(ref_name)
80 81 82
            if table_schema_reference is None:
                return None

83 84 85
            # Git refs
            if table_schema_reference:
                versions = table_schema_reference.get_refs()
86
                doc_url = table_schema_reference.doc_url
87 88
            options = {'ref': schema_ref} if schema_ref else {}
            schema_url = table_schema_reference.get_schema_url(**options)
89 90 91 92 93

        # else???
        else:
            return None

Christophe Benz's avatar
Christophe Benz committed
94
        return SchemaInstance(url=schema_url, section_name=section_name, name=schema_name, ref=schema_ref,
Pierre Dittgen's avatar
Pierre Dittgen committed
95
                              schema=schema_from_url(schema_url), versions=versions, doc_url=doc_url)
96 97 98 99 100 101 102 103 104 105 106

    def request_parameters(self):
        if self.name:
            return {
                'schema_name': self.name,
                'schema_ref': '' if self.ref is None else self.ref
            }
        return {
            'schema_url': self.url
        }

Christophe Benz's avatar
Christophe Benz committed
107 108 109 110 111 112 113
    def get_section_title(self, section_name):
        if config.HOMEPAGE_CONFIG:
            for section in config.HOMEPAGE_CONFIG['sections']:
                if section["name"] == section_name:
                    return section.get("title")
        return None

114

115
def extract_source_data(source: ValidataResource, preview_rows_nb=5):
116
    """ Computes table preview """
117 118 119

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
120
        return '' if val is None else str(val)
121

122 123
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
124
    nb_rows = 0
125

126
    tabulator_source, tabulator_options = source.build_tabulator_stream_args()
Pierre Dittgen's avatar
Pierre Dittgen committed
127
    with tabulator.Stream(tabulator_source, **tabulator_options) as stream:
128 129
        for row in stream:
            if header is None:
130
                header = ['' if v is None else v for v in row]
131
            else:
132
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
133
                nb_rows += 1
134
    preview_rows_nb = min(preview_rows_nb, nb_rows)
135 136
    return {'header': header,
            'rows_nb': nb_rows,
137 138 139
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
140 141


Pierre Dittgen's avatar
Pierre Dittgen committed
142 143
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
144

Pierre Dittgen's avatar
Pierre Dittgen committed
145 146
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
147

Pierre Dittgen's avatar
Pierre Dittgen committed
148 149 150 151
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
152

Pierre Dittgen's avatar
Pierre Dittgen committed
153
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
154

Pierre Dittgen's avatar
Pierre Dittgen committed
155 156 157
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
158

Pierre Dittgen's avatar
Pierre Dittgen committed
159 160 161 162
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
163

Pierre Dittgen's avatar
Pierre Dittgen committed
164 165 166
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
167

Pierre Dittgen's avatar
Pierre Dittgen committed
168 169 170
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
171

Pierre Dittgen's avatar
Pierre Dittgen committed
172
        return {**err, **update_keys}
173

Pierre Dittgen's avatar
Pierre Dittgen committed
174
    return list(map(improve_err, errors))
175 176


Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
177
def create_validata_ui_report(validata_core_report, schema_dict):
178 179 180 181 182 183
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
184
        - error messages are improved
185
    """
Pierre Dittgen's avatar
Pierre Dittgen committed
186
    report = copy.deepcopy(validata_core_report)
187 188 189 190 191 192 193 194 195

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
196 197 198
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
199 200 201 202
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
203
    # Computes column info
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
204 205
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', ''))
                   for f in schema_dict.get('fields', [])}
Pierre Dittgen's avatar
Pierre Dittgen committed
206 207 208
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
209

210
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
211 212
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
213

214 215 216 217 218 219 220
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
221
        if err['tag'] == 'structure':
222 223 224 225
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
226
    # Checks if there are structure errors different to invalid-column-delimiter
Pierre Dittgen's avatar
Pierre Dittgen committed
227 228 229
    structure_errors = report['table']['errors']['structure']
    report['table']['do_display_body_errors'] = len(structure_errors) == 0 or \
        all(err['code'] == 'invalid-column-delimiter' for err in structure_errors)
230

Pierre Dittgen's avatar
Pierre Dittgen committed
231 232 233 234 235 236 237 238
    # Checks if a column comparison is needed
    header_errors = ('missing-headers', 'extra-headers', 'wrong-headers-order')
    structure_errors = [{**err, 'in_column_comp': err['code'] in header_errors} for err in structure_errors]
    report['table']['errors']['structure'] = structure_errors
    column_comparison_needed = any(err['in_column_comp'] == True for err in structure_errors)
    column_comparison_table = []
    if column_comparison_needed:
        column_comparison_table = []
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
239
        field_names = [f['name'] for f in schema_dict.get('fields', [])]
240
        has_case_errors = False
Pierre Dittgen's avatar
Pierre Dittgen committed
241 242
        for t in itertools.zip_longest(headers, field_names, fillvalue=''):
            status = 'ok' if t[0] == t[1] else 'ko'
243 244
            if not has_case_errors and status == 'ko' and t[0].lower() == t[1].lower():
                has_case_errors = True
Pierre Dittgen's avatar
Pierre Dittgen committed
245
            column_comparison_table.append((*t, status))
246 247 248 249 250
        info = {}
        info['table'] = column_comparison_table
        info['has_missing'] = len(headers) < len(field_names)
        info['has_case_errors'] = has_case_errors
        report['table']['column_comparison_info'] = info
Pierre Dittgen's avatar
Pierre Dittgen committed
251 252
    report['table']['column_comparison_needed'] = column_comparison_needed

Pierre Dittgen's avatar
Pierre Dittgen committed
253 254 255 256
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
257 258
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
259 260 261 262 263 264 265 266 267 268 269 270 271
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
272
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
273

274 275 276 277 278 279
    # Sort by error names in statistics
    stats = report['table']['error-stats']
    code_title_map = messages.ERROR_MESSAGE_DEFAULT_TITLE
    for key in ('structure-errors', 'value-errors'):
        # convert dict into tuples with french title instead of error code
        # and sorts by title
Christophe Benz's avatar
Christophe Benz committed
280 281
        stats[key]['count-by-code'] = sorted(((code_title_map.get(k, k), v)
                                              for k, v in stats[key]['count-by-code'].items()), key=itemgetter(0))
282

Pierre Dittgen's avatar
Pierre Dittgen committed
283 284 285
    return report


Pierre Dittgen's avatar
Pierre Dittgen committed
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
    structure = badge['structure']
    body = badge.get('body')

    # Bad structure, stop here
    if structure == 'KO':
        return (
            'structure invalide', 'red')

    # No body error
    if body == 'OK':
        return ('structure invalide', 'orange') if structure == 'WARN' else ('valide', 'green')

    # else compute quality ratio percent
Christophe Benz's avatar
Christophe Benz committed
301
    p = (1 - badge['error-ratio']) * 100.0
Pierre Dittgen's avatar
Pierre Dittgen committed
302 303 304 305 306 307 308 309
    msg = 'cellules valides : {:.1f}%'.format(p)
    return (msg, 'red') if body == 'KO' else (msg, 'orange')


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
310 311
    return ('{}static/v1.svg?label=Validata&message={}&color={}'.format(
        config.SHIELDS_IO_BASE_URL, quote_plus(msg), color), msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
312 313


314
def validate(schema_instance: SchemaInstance, source: ValidataResource):
315 316
    """ Validate source and display report """

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
317
    api_url = config.API_VALIDATE_ENDPOINT
318 319

    # Useful to receive response as JSON
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
320 321
    headers = {"Accept": "application/json"}

322
    try:
323
        if source.type == 'url':
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
324
            params = {
325
                "schema": schema_instance.url,
326
                "url": source.url,
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
327 328 329 330
            }
            req = requests.get(api_url, params=params, headers=headers)

        else:
331
            files = {'file': (source.filename, source.build_reader())}
332
            data = {"schema": schema_instance.url}
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
333 334 335
            req = requests.post(api_url, data=data, files=files, headers=headers)
    except requests.ConnectionError as err:
        logging.exception(err)
Christophe Benz's avatar
Christophe Benz committed
336
        flash_error("Erreur technique lors de la validation")
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
337
        return redirect(url_for('home'))
338

339 340 341 342 343 344
    if not req.ok:
        flash_error("Une erreur s'est produite côté serveur :-(")
        return redirect(compute_validation_form_url(schema_instance))

    json_response = req.json()
    validata_core_report = json_response['report']
Pierre Dittgen's avatar
Pierre Dittgen committed
345
    badge_info = json_response.get('badge')
346

Pierre Dittgen's avatar
Pierre Dittgen committed
347
    # Computes badge from report and badge configuration
Pierre Dittgen's avatar
Pierre Dittgen committed
348 349 350 351
    badge_url, badge_msg = None, None
    display_badge = badge_info and config.SHIELDS_IO_BASE_URL
    if display_badge:
        badge_url, badge_msg = get_badge_url_and_message(badge_info)
Pierre Dittgen's avatar
Pierre Dittgen committed
352

353 354 355 356 357
    source_errors = [
        err
        for err in validata_core_report['tables'][0]['errors']
        if err['code'] in {'source-error', 'unknown-csv-dialect'}
    ]
358 359 360 361 362
    if source_errors:
        err = source_errors[0]
        msg = "l'encodage du fichier est invalide. Veuillez le corriger" if 'charmap' in err[
            'message'] else err['message']
        flash_error('Erreur de source : {}'.format(msg))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
363
        return redirect(url_for('custom_validator'))
364

365
    source_data = extract_source_data(source)
366

Pierre Dittgen's avatar
Pierre Dittgen committed
367
    # handle report date
368
    report_datetime = datetime.fromisoformat(validata_core_report['date']).astimezone()
369

Pierre Dittgen's avatar
Pierre Dittgen committed
370
    # Enhance validata_core_report
Pierre Dittgen's avatar
Pierre Dittgen committed
371
    validata_report = create_validata_ui_report(validata_core_report, schema_instance.schema.descriptor)
Pierre Dittgen's avatar
Pierre Dittgen committed
372

Pierre Dittgen's avatar
Pierre Dittgen committed
373
    # Display report to the user
374
    validator_form_url = compute_validation_form_url(schema_instance)
Christophe Benz's avatar
Christophe Benz committed
375
    schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
376
    pdf_report_url = url_for('pdf_report')+'?'+urlencode(schema_instance.request_parameters())
Christophe Benz's avatar
Christophe Benz committed
377
    return render_template('validation_report.html',
378 379
                           schema_info=schema_info, report=validata_report,
                           pdf_report_url=pdf_report_url,
380
                           validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
381
                           source=source, source_data=source_data,
382
                           schema_current_version=schema_instance.ref,
383
                           doc_url=schema_instance.doc_url,
384
                           print_mode=request.args.get('print', 'false') == 'true',
Pierre Dittgen's avatar
Pierre Dittgen committed
385
                           display_badge=display_badge, badge_url=badge_url, badge_msg=badge_msg,
386
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
Christophe Benz's avatar
Christophe Benz committed
387 388 389 390 391 392 393
                           section_title=schema_instance.section_title,
                           breadcrumbs=[
                               {'url': url_for('home'), 'title': 'Accueil'},
                               {'title': schema_instance.section_title},
                               {'url': validator_form_url, 'title': schema_info['title']},
                               {'title': 'Rapport de validation'},
                           ])
394 395


396 397
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
398
    iob = io.BytesIO()
399 400 401 402 403
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


404
def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
405 406 407 408 409
    """Replace catalog url within ui_config by schema references
    containing schema metadata properties"""

    extended_ui_config = ui_config.copy()
    for section in extended_ui_config['sections']:
Christophe Benz's avatar
Christophe Benz committed
410 411
        section_name = section['name']
        if section_name not in schema_catalog_map:
412
            continue
Christophe Benz's avatar
Christophe Benz committed
413
        schema_catalog = schema_catalog_map[section_name]
414 415 416
        schema_list = []
        for ref in schema_catalog.references:
            # Loads default table schema for each schema reference
417
            table_schema = schema_from_url(ref.get_schema_url(check_exists=False))
418
            schema_list.append({
Christophe Benz's avatar
Christophe Benz committed
419
                'name': '{}.{}'.format(section_name, ref.name),
420 421 422 423 424
                # Extracts title, description, ...
                **extract_schema_metadata(table_schema)
            })
        section['catalog'] = schema_list
    return extended_ui_config
425

426 427 428 429 430 431
# Routes


@app.route('/')
def home():
    """ Home page """
432
    home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map)
Christophe Benz's avatar
Christophe Benz committed
433
    return render_template('home.html', config=home_config)
434 435


Pierre Dittgen's avatar
Pierre Dittgen committed
436 437
@app.route('/pdf')
def pdf_report():
438
    """PDF report generation"""
439
    err_prefix = 'Erreur de génération du rapport PDF'
440 441 442

    url_param = request.args.get('url')
    if not url_param:
443
        flash_error(err_prefix + ': URL non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
444
        return redirect(url_for('home'))
445

446
    schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
447 448
    if schema_instance is None:
        flash_error(err_prefix + ': Information de schema non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
449 450
        return redirect(url_for('home'))

451 452 453 454 455 456 457 458 459 460 461
    # Compute pdf url report
    base_url = url_for('custom_validator', _external=True)
    parameter_dict = {
        'input': 'url',
        'print': 'true',
        'url': url_param,
        **schema_instance.request_parameters()
    }
    validation_url = base_url + '?' + urlencode(parameter_dict)

    # Create temp file to save validation report
Pierre Dittgen's avatar
Pierre Dittgen committed
462
    with tempfile.NamedTemporaryFile(prefix='validata_{}_report_'.format(datetime.now().timestamp()), suffix='.pdf') as tmpfile:
463
        tmp_pdf_report = Path(tmpfile.name)
464

465
    # Use chromium headless to generate PDF from validation report page
466 467 468 469
    cmd = ['chromium', '--headless', '--disable-gpu',
           '--print-to-pdf={}'.format(str(tmp_pdf_report)), validation_url]
    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if result.returncode != 0:
470 471
        flash_error(err_prefix)
        log.error("Command %r returned an error: %r", cmd, result.stdout.decode('utf-8'))
472 473
        if tmp_pdf_report.exists():
            tmp_pdf_report.unlink()
Pierre Dittgen's avatar
Pierre Dittgen committed
474
        return redirect(url_for('home'))
475

476
    # Send PDF report
Pierre Dittgen's avatar
Pierre Dittgen committed
477
    pdf_filename = 'Rapport de validation {}.pdf'.format(datetime.now().strftime('%d-%m-%Y %Hh%M'))
478
    response = make_response(tmp_pdf_report.read_bytes())
Pierre Dittgen's avatar
Pierre Dittgen committed
479
    response.headers.set('Content-disposition', 'attachment', filename=pdf_filename)
480 481 482 483 484 485 486 487
    response.headers.set('Content-type', 'application/pdf')
    response.headers.set('Content-length', tmp_pdf_report.stat().st_size)

    tmp_pdf_report.unlink()

    return response


488 489 490 491 492
def extract_schema_metadata(table_schema: tableschema.Schema):
    """Gets author, contibutor, version...metadata from schema header"""
    return {k: v for k, v in table_schema.descriptor.items() if k != 'fields'}


493
def compute_schema_info(table_schema: tableschema.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
494
    """Factor code for validator form page"""
495

496 497
    # Schema URL + schema metadata info
    schema_info = {
Christophe Benz's avatar
Christophe Benz committed
498 499 500
        'path': schema_url,
        # a "path" metadata property can be found in Table Schema, and we'd like it to override the `schema_url`
        # given by the user (in case schema was given by URL)
501 502
        **extract_schema_metadata(table_schema)
    }
Christophe Benz's avatar
Christophe Benz committed
503
    return schema_info
504 505 506 507 508 509 510 511


def compute_validation_form_url(schema_instance: SchemaInstance):
    """Computes validation form url with schema URL parameter"""
    url = url_for('custom_validator')
    param_list = ['{}={}'.format(k, quote_plus(v))
                  for k, v in schema_instance.request_parameters().items()]
    return "{}?{}".format(url, '&'.join(param_list))
Pierre Dittgen's avatar
Pierre Dittgen committed
512 513


514
@app.route('/table-schema', methods=['GET', 'POST'])
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
515
def custom_validator():
516
    """Validator form"""
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
517 518 519

    if request.method == 'GET':

520 521
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
522
        input_param = request.args.get('input')
523 524

        # url of resource to be validated
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
525 526
        url_param = request.args.get("url")

527 528 529 530 531 532 533 534 535 536
        schema_instance = None
        try:
            schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
        except json.JSONDecodeError as e:
            flash_error("Format de schéma non reconnu")
            return redirect(url_for('home'))
        except Exception as e:
            log.exception(e)
            flash_error("Erreur lors de l'obtention du schéma")
            return redirect(url_for('home'))
537 538
        if schema_instance is None:
            flash_error("Aucun schéma passé en paramètre")
Pierre Dittgen's avatar
Pierre Dittgen committed
539 540
            return redirect(url_for('home'))

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
541 542
        # First form display
        if input_param is None:
543 544

            schema_versions = schema_instance.versions
Christophe Benz's avatar
Christophe Benz committed
545 546 547 548
            schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
            return render_template('validation_form.html',
                                   schema_info=schema_info,
                                   schema_versions=schema_versions,
549
                                   schema_current_version=schema_instance.ref,
550
                                   doc_url=schema_instance.doc_url,
551
                                   schema_params=schema_instance.request_parameters(),
Christophe Benz's avatar
Christophe Benz committed
552 553 554 555 556 557
                                   section_title=schema_instance.section_title,
                                   breadcrumbs=[
                                       {'url': url_for('home'), 'title': 'Accueil'},
                                       {'title': schema_instance.section_title},
                                       {'title': schema_info['title']},
                                   ])
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
558 559 560 561

        # Process URL
        else:
            if url_param is None or url_param == '':
Christophe Benz's avatar
Christophe Benz committed
562
                flash_error("Vous n'avez pas indiqué d'URL à valider")
563
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
564
            try:
565
                return validate(schema_instance, URLValidataResource(url_param))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
566 567 568
            except tabulator.exceptions.FormatError as e:
                flash_error('Erreur : Format de ressource non supporté')
                log.info(e)
569
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
570 571 572
            except tabulator.exceptions.HTTPError as e:
                flash_error('Erreur : impossible d\'accéder au fichier source en ligne')
                log.info(e)
573
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
574 575

    else:  # POST
576

577
        schema_instance = SchemaInstance.from_parameters(request.form, schema_catalog_map)
578
        if schema_instance is None:
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
579
            flash_error('Aucun schéma défini')
Pierre Dittgen's avatar
Pierre Dittgen committed
580 581 582 583 584
            return redirect(url_for('home'))

        input_param = request.form.get('input')
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
585
            return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
586 587 588 589 590 591

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
592
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
593

594
            return validate(schema_instance, UploadedFileValidataResource(f.filename, bytes_data(f)))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
595 596

        return 'Bizarre, vous avez dit bizarre ?'