views.py 22.7 KB
Newer Older
1 2 3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
5
import io
Pierre Dittgen's avatar
Pierre Dittgen committed
6
import itertools
7
import json
8
import logging
9
import subprocess
10
import tempfile
11
from datetime import datetime
12
from operator import itemgetter
13
from pathlib import Path
14
from urllib.parse import quote_plus, urlencode
15

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
16
import requests
17
import tableschema
18
import tabulator
19
from backports.datetime_fromisoformat import MonkeyPatch
20
from commonmark import commonmark
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
21
from flask import make_response, redirect, render_template, request, url_for
22

23
from validata_core import compute_badge, messages
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
24

25
from . import app, config, schema_catalog_map, schema_from_url
26
from .ui_util import flash_error, flash_warning
27
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
28

29 30
MonkeyPatch.patch_fromisoformat()

31 32
log = logging.getLogger(__name__)

33

34 35 36
class SchemaInstance():
    """Handly class to handle schema information"""

37
    def __init__(self, url=None, name=None, ref=None, spec=None, versions=None):
38 39 40 41 42 43
        """This function is not intended to be called directly
        but via from_parameters() static method!"""
        self.url = url
        self.name = name
        self.ref = ref
        self.spec = spec
44
        self.versions = versions
45 46

    @staticmethod
47
    def from_parameters(parameter_dict, schema_catalog_map):
48 49
        """Initializes schema instance from requests dict and tableschema catalog (for name ref)
        """
50
        schema_url, schema_name, schema_ref, versions = None, None, None, None
51 52 53 54 55 56 57 58 59 60

        # From schema_url
        if 'schema_url' in parameter_dict:
            schema_url = parameter_dict["schema_url"]

        # from schema_name (and schema_ref)
        elif 'schema_name' in parameter_dict:
            schema_name = parameter_dict['schema_name']
            schema_ref = parameter_dict.get('schema_ref')

61 62 63 64 65 66 67 68 69 70 71 72
            # Check schema name
            chunks = schema_name.split('.')
            if len(chunks) != 2:
                return None

            section_code, ref_name = chunks

            # Look for schema catalog first
            table_schema_catalog = schema_catalog_map.get(section_code)
            if table_schema_catalog is None:
                return None

73
            # Unknown schema name?
74
            table_schema_reference = table_schema_catalog.reference_by_name.get(ref_name)
75 76 77
            if table_schema_reference is None:
                return None

78 79 80 81 82
            # Git refs
            if table_schema_reference:
                versions = table_schema_reference.get_refs()
            options = {'ref': schema_ref} if schema_ref else {}
            schema_url = table_schema_reference.get_schema_url(**options)
83 84 85 86 87

        # else???
        else:
            return None

88
        return SchemaInstance(schema_url, schema_name, schema_ref, schema_from_url(schema_url), versions)
89 90 91 92 93 94 95 96 97 98 99 100

    def request_parameters(self):
        if self.name:
            return {
                'schema_name': self.name,
                'schema_ref': '' if self.ref is None else self.ref
            }
        return {
            'schema_url': self.url
        }


101
def extract_source_data(source: ValidataResource, preview_rows_nb=5):
102
    """ Computes table preview """
103 104 105

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
106
        return '' if val is None else str(val)
107

108 109
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
110
    nb_rows = 0
111

112 113 114 115
    # if source.format == "csv":
    #     options['delimiter'] = csv_helpers.detect_dialect(source.source, format=source.format, scheme=source.scheme,
    #                                                       custom_loaders=custom_loaders).delimiter
    tabulator_source, tabulator_options = source.build_tabulator_stream_args()
Pierre Dittgen's avatar
Pierre Dittgen committed
116
    with tabulator.Stream(tabulator_source, **tabulator_options) as stream:
117 118
        for row in stream:
            if header is None:
119
                header = ['' if v is None else v for v in row]
120
            else:
121
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
122
                nb_rows += 1
123
    preview_rows_nb = min(preview_rows_nb, nb_rows)
124 125
    return {'header': header,
            'rows_nb': nb_rows,
126 127 128
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
129 130


Pierre Dittgen's avatar
Pierre Dittgen committed
131 132
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
133

Pierre Dittgen's avatar
Pierre Dittgen committed
134 135
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
136

Pierre Dittgen's avatar
Pierre Dittgen committed
137 138 139 140
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
141

Pierre Dittgen's avatar
Pierre Dittgen committed
142
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
143

Pierre Dittgen's avatar
Pierre Dittgen committed
144 145 146
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
147

Pierre Dittgen's avatar
Pierre Dittgen committed
148 149 150 151
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
152

Pierre Dittgen's avatar
Pierre Dittgen committed
153 154 155
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
156

Pierre Dittgen's avatar
Pierre Dittgen committed
157 158 159
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
160

Pierre Dittgen's avatar
Pierre Dittgen committed
161
        return {**err, **update_keys}
162

Pierre Dittgen's avatar
Pierre Dittgen committed
163
    return list(map(improve_err, errors))
164 165


Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
166
def create_validata_ui_report(validata_core_report, schema_dict):
167 168 169 170 171 172
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
173
        - error messages are improved
174
    """
Pierre Dittgen's avatar
Pierre Dittgen committed
175
    report = copy.deepcopy(validata_core_report)
176 177 178 179 180 181 182 183 184

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
185 186 187
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
188 189 190 191
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
192
    # Computes column info
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
193 194
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', ''))
                   for f in schema_dict.get('fields', [])}
Pierre Dittgen's avatar
Pierre Dittgen committed
195 196 197
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
198

199
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
200 201
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
202

203 204 205 206 207 208 209
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
210
        if err['tag'] == 'structure':
211 212 213 214
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
215
    # Checks if there are structure errors different to invalid-column-delimiter
Pierre Dittgen's avatar
Pierre Dittgen committed
216 217 218
    structure_errors = report['table']['errors']['structure']
    report['table']['do_display_body_errors'] = len(structure_errors) == 0 or \
        all(err['code'] == 'invalid-column-delimiter' for err in structure_errors)
219

Pierre Dittgen's avatar
Pierre Dittgen committed
220 221 222 223 224 225 226 227
    # Checks if a column comparison is needed
    header_errors = ('missing-headers', 'extra-headers', 'wrong-headers-order')
    structure_errors = [{**err, 'in_column_comp': err['code'] in header_errors} for err in structure_errors]
    report['table']['errors']['structure'] = structure_errors
    column_comparison_needed = any(err['in_column_comp'] == True for err in structure_errors)
    column_comparison_table = []
    if column_comparison_needed:
        column_comparison_table = []
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
228
        field_names = [f['name'] for f in schema_dict.get('fields', [])]
229
        has_case_errors = False
Pierre Dittgen's avatar
Pierre Dittgen committed
230 231
        for t in itertools.zip_longest(headers, field_names, fillvalue=''):
            status = 'ok' if t[0] == t[1] else 'ko'
232 233
            if not has_case_errors and status == 'ko' and t[0].lower() == t[1].lower():
                has_case_errors = True
Pierre Dittgen's avatar
Pierre Dittgen committed
234
            column_comparison_table.append((*t, status))
235 236 237 238 239
        info = {}
        info['table'] = column_comparison_table
        info['has_missing'] = len(headers) < len(field_names)
        info['has_case_errors'] = has_case_errors
        report['table']['column_comparison_info'] = info
Pierre Dittgen's avatar
Pierre Dittgen committed
240 241
    report['table']['column_comparison_needed'] = column_comparison_needed

Pierre Dittgen's avatar
Pierre Dittgen committed
242 243 244 245
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
246 247
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
248 249 250 251 252 253 254 255 256 257 258 259 260
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
261
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
262

263 264 265 266 267 268 269 270 271
    # Sort by error names in statistics
    stats = report['table']['error-stats']
    code_title_map = messages.ERROR_MESSAGE_DEFAULT_TITLE
    for key in ('structure-errors', 'value-errors'):
        # convert dict into tuples with french title instead of error code
        # and sorts by title
        stats[key]['count-by-code'] = sorted([(code_title_map.get(k, k), v) for k, v in stats[key]['count-by-code'].items()],
                                             key=itemgetter(0))

Pierre Dittgen's avatar
Pierre Dittgen committed
272 273 274
    return report


Pierre Dittgen's avatar
Pierre Dittgen committed
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
    structure = badge['structure']
    body = badge.get('body')

    # Bad structure, stop here
    if structure == 'KO':
        return (
            'structure invalide', 'red')

    # No body error
    if body == 'OK':
        return ('structure invalide', 'orange') if structure == 'WARN' else ('valide', 'green')

    # else compute quality ratio percent
Christophe Benz's avatar
Christophe Benz committed
290
    p = (1 - badge['error-ratio']) * 100.0
Pierre Dittgen's avatar
Pierre Dittgen committed
291 292 293 294 295 296 297 298
    msg = 'cellules valides : {:.1f}%'.format(p)
    return (msg, 'red') if body == 'KO' else (msg, 'orange')


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
299 300
    return ('{}static/v1.svg?label=Validata&message={}&color={}'.format(
        config.SHIELDS_IO_BASE_URL, quote_plus(msg), color), msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
301 302


303
def validate(schema_instance: SchemaInstance, source: ValidataResource):
304 305
    """ Validate source and display report """

306
    # Validation is done through http call to validata-api
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
307 308 309 310
    if config.API_VALIDATE_ENDPOINT is None:
        flash_error("No Validate endpoint defined :-(")
        return redirect(url_for("custom_validator"))
    api_url = config.API_VALIDATE_ENDPOINT
311 312

    # Useful to receive response as JSON
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
313 314
    headers = {"Accept": "application/json"}

315
    try:
316
        if source.type == 'url':
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
317
            params = {
318
                "schema": schema_instance.url,
319
                "url": source.url,
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
320 321 322 323
            }
            req = requests.get(api_url, params=params, headers=headers)

        else:
324
            files = {'file': (source.filename, source.build_reader())}
325
            data = {"schema": schema_instance.url}
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
326 327
            req = requests.post(api_url, data=data, files=files, headers=headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
328 329 330
        # 400
        if req.status_code == 400:
            json_response = req.json()
Pierre Dittgen's avatar
Pierre Dittgen committed
331 332 333 334 335 336 337 338
            if json_response.get('_meta') and json_response.get('_meta').get('args'):
                response_args = json_response.get('_meta').get('args')
                if response_args.get('code') == 'cant_detect_csv_dialect':
                    flash_error("Format de fichier CSV non reconnu")
                    return redirect(compute_validation_form_url(schema_instance))
                if response_args.get('code') == 'missing_headers':
                    flash_error("Impossible d'extraire les noms de colonne du fichier")
                    return redirect(compute_validation_form_url(schema_instance))
339

Pierre Dittgen's avatar
Pierre Dittgen committed
340 341
            flash_error("Une erreur est survenue durant la validation: {}"
                        .format(json_response.get('message')))
342
            return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
Pierre Dittgen committed
343

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
344
        if not req.ok:
Pierre Dittgen's avatar
Pierre Dittgen committed
345
            flash_error("Un erreur s'est produite côté serveur :-(")
346
            return redirect(compute_validation_form_url(schema_instance))
347

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
348 349 350
        json_response = req.json()
        validata_core_report = json_response['report']
        schema_dict = json_response['schema']
351

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
352 353 354 355
    except requests.ConnectionError as err:
        logging.exception(err)
        flash_error(str(err))
        return redirect(url_for('home'))
356

Pierre Dittgen's avatar
Pierre Dittgen committed
357
    # Computes badge from report and badge configuration
358
    badge = compute_badge(validata_core_report, config.BADGE_CONFIG)
Pierre Dittgen's avatar
Pierre Dittgen committed
359 360
    badge_url, badge_msg = get_badge_url_and_message(badge)

361 362 363 364 365 366
    source_errors = [err for err in validata_core_report['tables'][0]['errors'] if err['code'] == 'source-error']
    if source_errors:
        err = source_errors[0]
        msg = "l'encodage du fichier est invalide. Veuillez le corriger" if 'charmap' in err[
            'message'] else err['message']
        flash_error('Erreur de source : {}'.format(msg))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
367
        return redirect(url_for('custom_validator'))
368

369
    source_data = extract_source_data(source)
370

Pierre Dittgen's avatar
Pierre Dittgen committed
371
    # handle report date
372
    report_datetime = datetime.fromisoformat(validata_core_report['date']).astimezone()
373

Pierre Dittgen's avatar
Pierre Dittgen committed
374
    # Enhance validata_core_report
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
375
    validata_report = create_validata_ui_report(validata_core_report, schema_dict)
Pierre Dittgen's avatar
Pierre Dittgen committed
376

Pierre Dittgen's avatar
Pierre Dittgen committed
377
    # Display report to the user
378
    validator_form_url = compute_validation_form_url(schema_instance)
379
    schema_info, validator_title = compute_schema_info(schema_instance.spec, schema_instance.url)
380
    pdf_report_url = url_for('pdf_report')+'?'+urlencode(schema_instance.request_parameters())
381
    return render_template('validation_report.html', title='Rapport de validation',
382 383
                           schema_info=schema_info, report=validata_report,
                           pdf_report_url=pdf_report_url,
384
                           validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
385
                           source=source, source_data=source_data,
386
                           schema_current_version=schema_instance.ref,
387
                           print_mode=request.args.get('print', 'false') == 'true',
Pierre Dittgen's avatar
Pierre Dittgen committed
388
                           badge_url=badge_url, badge_msg=badge_msg,
389
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
Pierre Dittgen's avatar
Pierre Dittgen committed
390
                           breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
Pierre Dittgen's avatar
Pierre Dittgen committed
391
                                        {'url': validator_form_url, 'title': validator_title},
Pierre Dittgen's avatar
Pierre Dittgen committed
392
                                        ])
393 394


395 396
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
397
    iob = io.BytesIO()
398 399 400 401 402
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


403
def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
404 405 406 407 408 409 410 411 412 413 414 415
    """Replace catalog url within ui_config by schema references
    containing schema metadata properties"""

    extended_ui_config = ui_config.copy()
    for section in extended_ui_config['sections']:
        section_code = section['code']
        if section_code not in schema_catalog_map:
            continue
        schema_catalog = schema_catalog_map[section_code]
        schema_list = []
        for ref in schema_catalog.references:
            # Loads default table schema for each schema reference
416
            table_schema = schema_from_url(ref.get_schema_url(check_exists=False))
417 418 419 420 421 422 423
            schema_list.append({
                'name': '{}.{}'.format(section_code, ref.name),
                # Extracts title, description, ...
                **extract_schema_metadata(table_schema)
            })
        section['catalog'] = schema_list
    return extended_ui_config
424

425 426 427 428 429 430
# Routes


@app.route('/')
def home():
    """ Home page """
Christophe Benz's avatar
Christophe Benz committed
431
    flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu.')
432
    home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map)
433
    return render_template('home.html', title='Accueil', config=home_config)
434 435


Pierre Dittgen's avatar
Pierre Dittgen committed
436 437
@app.route('/pdf')
def pdf_report():
438
    """PDF report generation"""
439
    err_prefix = 'Erreur de génération du rapport PDF'
440 441 442

    url_param = request.args.get('url')
    if not url_param:
443
        flash_error(err_prefix + ': URL non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
444
        return redirect(url_for('home'))
445

446
    schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
447 448
    if schema_instance is None:
        flash_error(err_prefix + ': Information de schema non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
449 450
        return redirect(url_for('home'))

451 452 453 454 455 456 457 458 459 460 461
    # Compute pdf url report
    base_url = url_for('custom_validator', _external=True)
    parameter_dict = {
        'input': 'url',
        'print': 'true',
        'url': url_param,
        **schema_instance.request_parameters()
    }
    validation_url = base_url + '?' + urlencode(parameter_dict)

    # Create temp file to save validation report
Pierre Dittgen's avatar
Pierre Dittgen committed
462
    with tempfile.NamedTemporaryFile(prefix='validata_{}_report_'.format(datetime.now().timestamp()), suffix='.pdf') as tmpfile:
463
        tmp_pdf_report = Path(tmpfile.name)
464

465
    # Use chromium headless to generate PDF from validation report page
466 467 468 469
    cmd = ['chromium', '--headless', '--disable-gpu',
           '--print-to-pdf={}'.format(str(tmp_pdf_report)), validation_url]
    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if result.returncode != 0:
470 471
        flash_error(err_prefix)
        log.error("Command %r returned an error: %r", cmd, result.stdout.decode('utf-8'))
472 473
        if tmp_pdf_report.exists():
            tmp_pdf_report.unlink()
Pierre Dittgen's avatar
Pierre Dittgen committed
474
        return redirect(url_for('home'))
475

476
    # Send PDF report
Pierre Dittgen's avatar
Pierre Dittgen committed
477
    pdf_filename = 'Rapport de validation {}.pdf'.format(datetime.now().strftime('%d-%m-%Y %Hh%M'))
478
    response = make_response(tmp_pdf_report.read_bytes())
Pierre Dittgen's avatar
Pierre Dittgen committed
479
    response.headers.set('Content-disposition', 'attachment', filename=pdf_filename)
480 481 482 483 484 485 486 487
    response.headers.set('Content-type', 'application/pdf')
    response.headers.set('Content-length', tmp_pdf_report.stat().st_size)

    tmp_pdf_report.unlink()

    return response


488 489 490 491 492
def extract_schema_metadata(table_schema: tableschema.Schema):
    """Gets author, contibutor, version...metadata from schema header"""
    return {k: v for k, v in table_schema.descriptor.items() if k != 'fields'}


493
def compute_schema_info(table_schema: tableschema.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
494
    """Factor code for validator form page"""
495

496 497 498 499 500 501 502 503
    # Schema URL + schema metadata info
    schema_info = {
        'url': schema_url,
        **extract_schema_metadata(table_schema)
    }
    meta_title = schema_info.get('title')

    title = "Schéma « {} »".format(meta_title if meta_title else '...')
504 505 506 507 508 509 510 511 512
    return schema_info, title


def compute_validation_form_url(schema_instance: SchemaInstance):
    """Computes validation form url with schema URL parameter"""
    url = url_for('custom_validator')
    param_list = ['{}={}'.format(k, quote_plus(v))
                  for k, v in schema_instance.request_parameters().items()]
    return "{}?{}".format(url, '&'.join(param_list))
Pierre Dittgen's avatar
Pierre Dittgen committed
513 514


515
@app.route('/table-schema', methods=['GET', 'POST'])
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
516
def custom_validator():
517
    """Validator form"""
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
518

519
    # Check that validata-api URL is set
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
520 521 522 523 524 525
    if config.API_VALIDATE_ENDPOINT is None:
        flash_error("URL de connexion à l'API non indiquée :-(")
        return redirect(url_for('home'))

    if request.method == 'GET':

526 527
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
528
        input_param = request.args.get('input')
529 530

        # url of resource to be validated
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
531 532
        url_param = request.args.get("url")

533
        schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
534 535
        if schema_instance is None:
            flash_error("Aucun schéma passé en paramètre")
Pierre Dittgen's avatar
Pierre Dittgen committed
536 537
            return redirect(url_for('home'))

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
538 539
        # First form display
        if input_param is None:
540 541 542

            schema_versions = schema_instance.versions
            schema_info, title = compute_schema_info(schema_instance.spec, schema_instance.url)
Pierre Dittgen's avatar
Pierre Dittgen committed
543
            return render_template('validation_form.html', title=title,
544 545
                                   schema_info=schema_info, schema_versions=schema_versions,
                                   schema_current_version=schema_instance.ref,
546
                                   schema_params=schema_instance.request_parameters(),
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
547 548 549 550 551 552
                                   breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])

        # Process URL
        else:
            if url_param is None or url_param == '':
                flash_error("Vous n'avez pas indiqué d'url à valider")
553
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
554
            try:
555
                return validate(schema_instance, URLValidataResource(url_param))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
556 557 558
            except tabulator.exceptions.FormatError as e:
                flash_error('Erreur : Format de ressource non supporté')
                log.info(e)
559
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
560 561 562
            except tabulator.exceptions.HTTPError as e:
                flash_error('Erreur : impossible d\'accéder au fichier source en ligne')
                log.info(e)
563
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
564 565

    else:  # POST
566

567
        schema_instance = SchemaInstance.from_parameters(request.form, schema_catalog_map)
568
        if schema_instance is None:
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
569
            flash_error('Aucun schéma défini')
Pierre Dittgen's avatar
Pierre Dittgen committed
570 571 572 573 574
            return redirect(url_for('home'))

        input_param = request.form.get('input')
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
575
            return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
576 577 578 579 580 581

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
582
                return redirect(compute_validation_form_url(schema_instance))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
583

584
            return validate(schema_instance, UploadedFileValidataResource(f.filename, bytes_data(f)))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
585 586

        return 'Bizarre, vous avez dit bizarre ?'