views.py 22.6 KB
Newer Older
1 2 3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
5
import io
Pierre Dittgen's avatar
Pierre Dittgen committed
6
import itertools
7
import json
Christophe Benz's avatar
Christophe Benz committed
8
import logging
9
import subprocess
Christophe Benz's avatar
Christophe Benz committed
10
import tempfile
11
from datetime import datetime
12
from operator import itemgetter
13
from pathlib import Path
14
from urllib.parse import quote_plus, urlencode
15

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
16
import requests
17
import tableschema
18
import tabulator
19
from backports.datetime_fromisoformat import MonkeyPatch
20
from commonmark import commonmark
21
from flask import abort, make_response, redirect, render_template, request, url_for
Pierre Dittgen's avatar
Pierre Dittgen committed
22
from validata_core import messages
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
23

Pierre Dittgen's avatar
Pierre Dittgen committed
24
from opendataschema import GitSchemaReference, by_semver
25

26
from . import app, config, schema_catalog_registry, tableschema_from_url
27
from .ui_util import flash_error, flash_warning
28
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
29

30 31
MonkeyPatch.patch_fromisoformat()

Christophe Benz's avatar
Christophe Benz committed
32 33
log = logging.getLogger(__name__)

34

35 36 37 38 39
def get_schema_catalog(section_name):
    """Return a schema catalog associated to a section_name"""
    return schema_catalog_registry.build_schema_catalog(section_name)


40 41 42
class SchemaInstance:
    """Handy class to handle schema information"""

43
    def __init__(self, parameter_dict):
44 45 46 47 48 49 50 51 52 53
        """Initializes schema instance from requests dict and tableschema catalog (for name ref)"""
        self.section_name = None
        self.section_title = None
        self.name = None
        self.url = None
        self.ref = None
        self.reference = None
        self.doc_url = None
        self.branches = None
        self.tags = None
54 55

        # From schema_url
Christophe Benz's avatar
Christophe Benz committed
56
        if parameter_dict.get("schema_url"):
57 58
            self.url = parameter_dict["schema_url"]
            self.section_title = "Autre schéma"
59 60

        # from schema_name (and schema_ref)
Christophe Benz's avatar
Christophe Benz committed
61
        elif parameter_dict.get('schema_name'):
62 63
            self.schema_and_section_name = parameter_dict['schema_name']
            self.ref = parameter_dict.get('schema_ref')
64

Pierre Dittgen's avatar
Pierre Dittgen committed
65
            # Check schema name
66
            chunks = self.schema_and_section_name.split('.')
Pierre Dittgen's avatar
Pierre Dittgen committed
67
            if len(chunks) != 2:
68
                abort(400, "Paramètre 'schema_name' invalide")
Pierre Dittgen's avatar
Pierre Dittgen committed
69

70 71
            self.section_name, self.name = chunks
            self.section_title = self.find_section_title(self.section_name)
Pierre Dittgen's avatar
Pierre Dittgen committed
72 73

            # Look for schema catalog first
74
            table_schema_catalog = get_schema_catalog(self.section_name)
Pierre Dittgen's avatar
Pierre Dittgen committed
75
            if table_schema_catalog is None:
76 77 78 79 80 81 82 83
                abort(400, "Section '{}' non trouvée dans la configuration".format(self.section_name))

            schema_reference = table_schema_catalog.reference_by_name.get(self.name)
            if schema_reference is None:
                abort(400, "Schéma '{}' non trouvé dans le catalogue de la section '{}'".format(self.name, self.section_name))

            if isinstance(schema_reference, GitSchemaReference):
                if self.ref is None:
84
                    schema_ref = schema_reference.get_latest_tag() or schema_reference.get_default_branch()
85 86 87 88
                    abort(redirect(compute_validation_form_url({
                        'schema_name': self.schema_and_section_name,
                        'schema_ref': schema_ref.name
                    })))
Pierre Dittgen's avatar
Pierre Dittgen committed
89
                self.tags = sorted(schema_reference.iter_tags(), key=by_semver, reverse=True)
90 91 92
                tag_names = [tag.name for tag in self.tags]
                self.branches = [branch for branch in schema_reference.iter_branches()
                                 if branch.name not in tag_names]
93 94
                self.doc_url = schema_reference.get_doc_url(ref=self.ref) or \
                    schema_reference.get_project_url(ref=self.ref)
95

96
            self.url = schema_reference.get_schema_url(ref=self.ref)
97 98

        else:
99
            abort(400, "L'un des paramètres est nécessaire : 'schema_name', 'schema_url'")
100

101 102 103 104 105 106 107 108 109 110
        try:
            self.schema = tableschema_from_url(self.url)
        except json.JSONDecodeError as e:
            log.exception(e)
            flash_error("Format de schéma non reconnu")
            abort(redirect(url_for('home')))
        except Exception as e:
            log.exception(e)
            flash_error("Erreur lors de l'obtention du schéma")
            abort(redirect(url_for('home')))
111 112 113 114

    def request_parameters(self):
        if self.name:
            return {
115
                'schema_name': self.schema_and_section_name,
116 117 118 119 120 121
                'schema_ref': '' if self.ref is None else self.ref
            }
        return {
            'schema_url': self.url
        }

122
    def find_section_title(self, section_name):
Christophe Benz's avatar
Christophe Benz committed
123 124 125 126 127 128
        if config.HOMEPAGE_CONFIG:
            for section in config.HOMEPAGE_CONFIG['sections']:
                if section["name"] == section_name:
                    return section.get("title")
        return None

129

130
def extract_source_data(source: ValidataResource, preview_rows_nb=5):
131
    """ Computes table preview """
132 133 134

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
135
        return '' if val is None else str(val)
136

137 138
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
139
    nb_rows = 0
140

141
    tabulator_source, tabulator_options = source.build_tabulator_stream_args()
Pierre Dittgen's avatar
Pierre Dittgen committed
142
    with tabulator.Stream(tabulator_source, **tabulator_options) as stream:
143 144
        for row in stream:
            if header is None:
145
                header = ['' if v is None else v for v in row]
146
            else:
147
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
148
                nb_rows += 1
149
    preview_rows_nb = min(preview_rows_nb, nb_rows)
150 151
    return {'header': header,
            'rows_nb': nb_rows,
152 153 154
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
155 156


Pierre Dittgen's avatar
Pierre Dittgen committed
157 158
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
159

Pierre Dittgen's avatar
Pierre Dittgen committed
160 161
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
162

Pierre Dittgen's avatar
Pierre Dittgen committed
163 164 165 166
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
167

Pierre Dittgen's avatar
Pierre Dittgen committed
168
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
169

Pierre Dittgen's avatar
Pierre Dittgen committed
170 171 172
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
173

Pierre Dittgen's avatar
Pierre Dittgen committed
174 175 176 177
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
178

Pierre Dittgen's avatar
Pierre Dittgen committed
179 180 181
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
182

Pierre Dittgen's avatar
Pierre Dittgen committed
183 184 185
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
186

Pierre Dittgen's avatar
Pierre Dittgen committed
187
        return {**err, **update_keys}
188

Pierre Dittgen's avatar
Pierre Dittgen committed
189
    return list(map(improve_err, errors))
190 191


Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
192
def create_validata_ui_report(validata_core_report, schema_dict):
193 194 195 196 197 198
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
199
        - error messages are improved
200
    """
Pierre Dittgen's avatar
Pierre Dittgen committed
201
    report = copy.deepcopy(validata_core_report)
202 203 204 205 206 207 208 209 210

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
211 212 213
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
214 215 216 217
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
218
    # Computes column info
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
219 220
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', ''))
                   for f in schema_dict.get('fields', [])}
Pierre Dittgen's avatar
Pierre Dittgen committed
221 222 223
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
224

225
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
226 227
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
228

229 230 231 232 233 234 235
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
236
        if err['tag'] == 'structure':
237 238 239 240
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
241
    # Checks if there are structure errors different to invalid-column-delimiter
Pierre Dittgen's avatar
Pierre Dittgen committed
242 243 244
    structure_errors = report['table']['errors']['structure']
    report['table']['do_display_body_errors'] = len(structure_errors) == 0 or \
        all(err['code'] == 'invalid-column-delimiter' for err in structure_errors)
245

Pierre Dittgen's avatar
Pierre Dittgen committed
246 247 248 249 250 251 252 253
    # Checks if a column comparison is needed
    header_errors = ('missing-headers', 'extra-headers', 'wrong-headers-order')
    structure_errors = [{**err, 'in_column_comp': err['code'] in header_errors} for err in structure_errors]
    report['table']['errors']['structure'] = structure_errors
    column_comparison_needed = any(err['in_column_comp'] == True for err in structure_errors)
    column_comparison_table = []
    if column_comparison_needed:
        column_comparison_table = []
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
254
        field_names = [f['name'] for f in schema_dict.get('fields', [])]
255
        has_case_errors = False
Pierre Dittgen's avatar
Pierre Dittgen committed
256 257
        for t in itertools.zip_longest(headers, field_names, fillvalue=''):
            status = 'ok' if t[0] == t[1] else 'ko'
258 259
            if not has_case_errors and status == 'ko' and t[0].lower() == t[1].lower():
                has_case_errors = True
Pierre Dittgen's avatar
Pierre Dittgen committed
260
            column_comparison_table.append((*t, status))
261 262 263 264 265
        info = {}
        info['table'] = column_comparison_table
        info['has_missing'] = len(headers) < len(field_names)
        info['has_case_errors'] = has_case_errors
        report['table']['column_comparison_info'] = info
Pierre Dittgen's avatar
Pierre Dittgen committed
266 267
    report['table']['column_comparison_needed'] = column_comparison_needed

Pierre Dittgen's avatar
Pierre Dittgen committed
268 269 270 271
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
272 273
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
274 275 276 277 278 279 280 281 282 283 284 285 286
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
287
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
288

289 290 291 292 293 294
    # Sort by error names in statistics
    stats = report['table']['error-stats']
    code_title_map = messages.ERROR_MESSAGE_DEFAULT_TITLE
    for key in ('structure-errors', 'value-errors'):
        # convert dict into tuples with french title instead of error code
        # and sorts by title
Christophe Benz's avatar
Christophe Benz committed
295 296
        stats[key]['count-by-code'] = sorted(((code_title_map.get(k, k), v)
                                              for k, v in stats[key]['count-by-code'].items()), key=itemgetter(0))
297

Pierre Dittgen's avatar
Pierre Dittgen committed
298 299 300
    return report


Pierre Dittgen's avatar
Pierre Dittgen committed
301 302 303 304 305 306 307
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
    structure = badge['structure']
    body = badge.get('body')

    # Bad structure, stop here
    if structure == 'KO':
308
        return ('structure invalide', 'red')
Pierre Dittgen's avatar
Pierre Dittgen committed
309 310 311 312 313 314

    # No body error
    if body == 'OK':
        return ('structure invalide', 'orange') if structure == 'WARN' else ('valide', 'green')

    # else compute quality ratio percent
Christophe Benz's avatar
Christophe Benz committed
315
    p = (1 - badge['error-ratio']) * 100.0
Pierre Dittgen's avatar
Pierre Dittgen committed
316 317 318 319 320 321 322 323
    msg = 'cellules valides : {:.1f}%'.format(p)
    return (msg, 'red') if body == 'KO' else (msg, 'orange')


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
324 325
    return ('{}static/v1.svg?label=Validata&message={}&color={}'.format(
        config.SHIELDS_IO_BASE_URL, quote_plus(msg), color), msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
326 327


328
def validate(schema_instance: SchemaInstance, source: ValidataResource):
329 330
    """ Validate source and display report """

331
    # Useful to receive response as JSON
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
332 333
    headers = {"Accept": "application/json"}

Pierre Dittgen's avatar
Pierre Dittgen committed
334
    try:
335
        if source.type == 'url':
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
336
            params = {
337
                "schema": schema_instance.url,
338
                "url": source.url,
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
339
            }
340
            response = requests.get(config.API_VALIDATE_ENDPOINT, params=params, headers=headers)
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
341
        else:
342
            files = {'file': (source.filename, source.build_reader())}
343
            data = {"schema": schema_instance.url}
344
            response = requests.post(config.API_VALIDATE_ENDPOINT, data=data, files=files, headers=headers)
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
345 346
    except requests.ConnectionError as err:
        logging.exception(err)
Christophe Benz's avatar
Christophe Benz committed
347
        flash_error("Erreur technique lors de la validation")
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
348
        return redirect(url_for('home'))
349

350 351 352
    if not response.ok:
        flash_error("Erreur technique lors de la validation")
        return redirect(compute_validation_form_url(schema_instance.request_parameters()))
353

354
    json_response = response.json()
355
    validata_core_report = json_response['report']
Pierre Dittgen's avatar
Pierre Dittgen committed
356
    badge_info = json_response.get('badge')
357

Pierre Dittgen's avatar
Pierre Dittgen committed
358
    # Computes badge from report and badge configuration
Pierre Dittgen's avatar
Pierre Dittgen committed
359 360 361 362
    badge_url, badge_msg = None, None
    display_badge = badge_info and config.SHIELDS_IO_BASE_URL
    if display_badge:
        badge_url, badge_msg = get_badge_url_and_message(badge_info)
Pierre Dittgen's avatar
Pierre Dittgen committed
363

364 365 366 367 368
    source_errors = [
        err
        for err in validata_core_report['tables'][0]['errors']
        if err['code'] in {'source-error', 'unknown-csv-dialect'}
    ]
369 370 371 372 373
    if source_errors:
        err = source_errors[0]
        msg = "l'encodage du fichier est invalide. Veuillez le corriger" if 'charmap' in err[
            'message'] else err['message']
        flash_error('Erreur de source : {}'.format(msg))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
374
        return redirect(url_for('custom_validator'))
375

376
    source_data = extract_source_data(source)
377

Pierre Dittgen's avatar
Pierre Dittgen committed
378
    # handle report date
379
    report_datetime = datetime.fromisoformat(validata_core_report['date']).astimezone()
380

Pierre Dittgen's avatar
Pierre Dittgen committed
381
    # Enhance validata_core_report
Pierre Dittgen's avatar
Pierre Dittgen committed
382
    validata_report = create_validata_ui_report(validata_core_report, schema_instance.schema.descriptor)
Pierre Dittgen's avatar
Pierre Dittgen committed
383

Pierre Dittgen's avatar
Pierre Dittgen committed
384
    # Display report to the user
385
    validator_form_url = compute_validation_form_url(schema_instance.request_parameters())
Christophe Benz's avatar
Christophe Benz committed
386
    schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
387 388
    pdf_report_url = url_for('pdf_report') + '?' + urlencode(schema_instance.request_parameters())

Christophe Benz's avatar
Christophe Benz committed
389
    return render_template('validation_report.html',
390 391 392 393 394 395 396 397 398
                           badge_msg=badge_msg,
                           badge_url=badge_url,
                           breadcrumbs=[
                               {'title': 'Accueil', 'url': url_for('home')},
                               {'title': schema_instance.section_title},
                               {'title': schema_info['title'], 'url': validator_form_url},
                               {'title': 'Rapport de validation'},
                           ],
                           display_badge=display_badge,
399
                           doc_url=schema_instance.doc_url,
400
                           pdf_report_url=pdf_report_url,
401
                           print_mode=request.args.get('print', 'false') == 'true',
402
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
403 404 405
                           report=validata_report,
                           schema_current_version=schema_instance.ref,
                           schema_info=schema_info,
Christophe Benz's avatar
Christophe Benz committed
406
                           section_title=schema_instance.section_title,
407 408 409 410
                           source_data=source_data,
                           source=source,
                           validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
                           )
411 412


413 414
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
415
    iob = io.BytesIO()
416 417 418 419 420
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


421
def homepage_config_with_schema_metadata(ui_config):
Pierre Dittgen's avatar
Pierre Dittgen committed
422 423 424 425 426
    """Replace catalog url within ui_config by schema references
    containing schema metadata properties"""

    extended_ui_config = ui_config.copy()
    for section in extended_ui_config['sections']:
Christophe Benz's avatar
Christophe Benz committed
427
        section_name = section['name']
428 429
        schema_catalog = get_schema_catalog(section_name)
        if schema_catalog is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
430 431 432 433
            continue
        schema_list = []
        for ref in schema_catalog.references:
            # Loads default table schema for each schema reference
434
            table_schema = tableschema_from_url(ref.get_schema_url())
Pierre Dittgen's avatar
Pierre Dittgen committed
435
            schema_list.append({
Christophe Benz's avatar
Christophe Benz committed
436
                'name': '{}.{}'.format(section_name, ref.name),
Pierre Dittgen's avatar
Pierre Dittgen committed
437 438 439 440 441
                # Extracts title, description, ...
                **extract_schema_metadata(table_schema)
            })
        section['catalog'] = schema_list
    return extended_ui_config
442

443 444 445 446 447 448
# Routes


@app.route('/')
def home():
    """ Home page """
449
    home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG)
Christophe Benz's avatar
Christophe Benz committed
450
    return render_template('home.html', config=home_config)
451 452


Pierre Dittgen's avatar
Pierre Dittgen committed
453 454
@app.route('/pdf')
def pdf_report():
455
    """PDF report generation"""
456
    err_prefix = 'Erreur de génération du rapport PDF'
457 458 459

    url_param = request.args.get('url')
    if not url_param:
460
        flash_error(err_prefix + ' : URL non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
461
        return redirect(url_for('home'))
462

463
    schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
464

465 466 467 468 469 470 471 472 473 474 475
    # Compute pdf url report
    base_url = url_for('custom_validator', _external=True)
    parameter_dict = {
        'input': 'url',
        'print': 'true',
        'url': url_param,
        **schema_instance.request_parameters()
    }
    validation_url = base_url + '?' + urlencode(parameter_dict)

    # Create temp file to save validation report
Pierre Dittgen's avatar
Pierre Dittgen committed
476
    with tempfile.NamedTemporaryFile(prefix='validata_{}_report_'.format(datetime.now().timestamp()), suffix='.pdf') as tmpfile:
Christophe Benz's avatar
Christophe Benz committed
477
        tmp_pdf_report = Path(tmpfile.name)
478

479
    # Use chromium headless to generate PDF from validation report page
480 481 482 483
    cmd = ['chromium', '--headless', '--disable-gpu',
           '--print-to-pdf={}'.format(str(tmp_pdf_report)), validation_url]
    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if result.returncode != 0:
484 485
        flash_error(err_prefix)
        log.error("Command %r returned an error: %r", cmd, result.stdout.decode('utf-8'))
486 487
        if tmp_pdf_report.exists():
            tmp_pdf_report.unlink()
Pierre Dittgen's avatar
Pierre Dittgen committed
488
        return redirect(url_for('home'))
489

490
    # Send PDF report
Pierre Dittgen's avatar
Pierre Dittgen committed
491
    pdf_filename = 'Rapport de validation {}.pdf'.format(datetime.now().strftime('%d-%m-%Y %Hh%M'))
Christophe Benz's avatar
Christophe Benz committed
492
    response = make_response(tmp_pdf_report.read_bytes())
Pierre Dittgen's avatar
Pierre Dittgen committed
493
    response.headers.set('Content-disposition', 'attachment', filename=pdf_filename)
494 495 496 497 498 499 500 501
    response.headers.set('Content-type', 'application/pdf')
    response.headers.set('Content-length', tmp_pdf_report.stat().st_size)

    tmp_pdf_report.unlink()

    return response


Pierre Dittgen's avatar
Pierre Dittgen committed
502 503 504 505 506
def extract_schema_metadata(table_schema: tableschema.Schema):
    """Gets author, contibutor, version...metadata from schema header"""
    return {k: v for k, v in table_schema.descriptor.items() if k != 'fields'}


Pierre Dittgen's avatar
Pierre Dittgen committed
507
def compute_schema_info(table_schema: tableschema.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
508
    """Factor code for validator form page"""
509

Pierre Dittgen's avatar
Pierre Dittgen committed
510 511
    # Schema URL + schema metadata info
    schema_info = {
Christophe Benz's avatar
Christophe Benz committed
512 513 514
        'path': schema_url,
        # a "path" metadata property can be found in Table Schema, and we'd like it to override the `schema_url`
        # given by the user (in case schema was given by URL)
Pierre Dittgen's avatar
Pierre Dittgen committed
515 516
        **extract_schema_metadata(table_schema)
    }
Christophe Benz's avatar
Christophe Benz committed
517
    return schema_info
518 519


520
def compute_validation_form_url(request_parameters: dict):
521 522
    """Computes validation form url with schema URL parameter"""
    url = url_for('custom_validator')
523
    param_list = ['{}={}'.format(k, quote_plus(v)) for k, v in request_parameters.items()]
524
    return "{}?{}".format(url, '&'.join(param_list))
Pierre Dittgen's avatar
Pierre Dittgen committed
525 526


527
@app.route('/table-schema', methods=['GET', 'POST'])
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
528
def custom_validator():
529
    """Validator form"""
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
530 531 532

    if request.method == 'GET':

533 534
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
535
        input_param = request.args.get('input')
536 537

        # url of resource to be validated
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
538 539
        url_param = request.args.get("url")

540
        schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
541

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
542 543
        # First form display
        if input_param is None:
Christophe Benz's avatar
Christophe Benz committed
544 545
            schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
            return render_template('validation_form.html',
546
                                   branches=schema_instance.branches,
Christophe Benz's avatar
Christophe Benz committed
547 548 549 550
                                   breadcrumbs=[
                                       {'url': url_for('home'), 'title': 'Accueil'},
                                       {'title': schema_instance.section_title},
                                       {'title': schema_info['title']},
Christophe Benz's avatar
Christophe Benz committed
551 552 553 554 555 556 557 558
                                   ],
                                   doc_url=schema_instance.doc_url,
                                   schema_current_version=schema_instance.ref,
                                   schema_info=schema_info,
                                   schema_params=schema_instance.request_parameters(),
                                   section_title=schema_instance.section_title,
                                   tags=schema_instance.tags,
                                   )
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
559 560 561

        # Process URL
        else:
562
            if not url_param:
Christophe Benz's avatar
Christophe Benz committed
563
                flash_error("Vous n'avez pas indiqué d'URL à valider")
564 565 566 567 568
                return redirect(compute_validation_form_url(schema_instance.request_parameters()))
            return validate(schema_instance, URLValidataResource(url_param))

    elif request.method == 'POST':

569
        schema_instance = SchemaInstance(request.form)
Pierre Dittgen's avatar
Pierre Dittgen committed
570 571 572 573

        input_param = request.form.get('input')
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
574
            return redirect(compute_validation_form_url(schema_instance.request_parameters()))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
575 576 577 578 579 580

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
581
                return redirect(compute_validation_form_url(schema_instance.request_parameters()))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
582

583
            return validate(schema_instance, UploadedFileValidataResource(f.filename, bytes_data(f)))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
584

585 586 587 588
        return 'Combinaison de paramètres non supportée', 400

    else:
        return "Method not allowed", 405