views.py 22.6 KB
Newer Older
1 2 3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
5
import io
Pierre Dittgen's avatar
Pierre Dittgen committed
6
import itertools
7
import json
Christophe Benz's avatar
Christophe Benz committed
8
import logging
9
import subprocess
Christophe Benz's avatar
Christophe Benz committed
10
import tempfile
11
from datetime import datetime
12
from operator import itemgetter
13
from pathlib import Path
14
from urllib.parse import quote_plus, urlencode
15

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
16
import requests
17
import tableschema
18
import tabulator
19
from backports.datetime_fromisoformat import MonkeyPatch
20
from commonmark import commonmark
21
from flask import abort, make_response, redirect, render_template, request, url_for
Pierre Dittgen's avatar
Pierre Dittgen committed
22
from validata_core import messages
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
23

24 25 26
from opendataschema import GitSchemaReference

from . import app, config, schema_catalog_map, tableschema_from_url
27
from .ui_util import flash_error, flash_warning
28
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
29

30 31
MonkeyPatch.patch_fromisoformat()

Christophe Benz's avatar
Christophe Benz committed
32 33
log = logging.getLogger(__name__)

34

35 36 37 38 39 40 41 42 43 44 45 46 47 48
class SchemaInstance:
    """Handy class to handle schema information"""

    def __init__(self, parameter_dict, schema_catalog_map):
        """Initializes schema instance from requests dict and tableschema catalog (for name ref)"""
        self.section_name = None
        self.section_title = None
        self.name = None
        self.url = None
        self.ref = None
        self.reference = None
        self.doc_url = None
        self.branches = None
        self.tags = None
49 50

        # From schema_url
Christophe Benz's avatar
Christophe Benz committed
51
        if parameter_dict.get("schema_url"):
52 53
            self.url = parameter_dict["schema_url"]
            self.section_title = "Autre schéma"
54 55

        # from schema_name (and schema_ref)
Christophe Benz's avatar
Christophe Benz committed
56
        elif parameter_dict.get('schema_name'):
57 58
            self.schema_and_section_name = parameter_dict['schema_name']
            self.ref = parameter_dict.get('schema_ref')
59

Pierre Dittgen's avatar
Pierre Dittgen committed
60
            # Check schema name
61
            chunks = self.schema_and_section_name.split('.')
Pierre Dittgen's avatar
Pierre Dittgen committed
62
            if len(chunks) != 2:
63
                abort(400, "Paramètre 'schema_name' invalide")
Pierre Dittgen's avatar
Pierre Dittgen committed
64

65 66
            self.section_name, self.name = chunks
            self.section_title = self.find_section_title(self.section_name)
Pierre Dittgen's avatar
Pierre Dittgen committed
67 68

            # Look for schema catalog first
69
            table_schema_catalog = schema_catalog_map.get(self.section_name)
Pierre Dittgen's avatar
Pierre Dittgen committed
70
            if table_schema_catalog is None:
71 72 73 74 75 76 77 78
                abort(400, "Section '{}' non trouvée dans la configuration".format(self.section_name))

            schema_reference = table_schema_catalog.reference_by_name.get(self.name)
            if schema_reference is None:
                abort(400, "Schéma '{}' non trouvé dans le catalogue de la section '{}'".format(self.name, self.section_name))

            if isinstance(schema_reference, GitSchemaReference):
                if self.ref is None:
79
                    schema_ref = schema_reference.get_latest_tag() or schema_reference.get_default_branch()
80 81 82 83 84 85 86 87
                    abort(redirect(compute_validation_form_url({
                        'schema_name': self.schema_and_section_name,
                        'schema_ref': schema_ref.name
                    })))
                self.tags = list(schema_reference.iter_tags())
                tag_names = [tag.name for tag in self.tags]
                self.branches = [branch for branch in schema_reference.iter_branches()
                                 if branch.name not in tag_names]
88 89
                self.doc_url = schema_reference.get_doc_url(ref=self.ref) or \
                    schema_reference.get_project_url(ref=self.ref)
90

91
            self.url = schema_reference.get_schema_url(ref=self.ref)
92 93

        else:
94
            abort(400, "L'un des paramètres est nécessaire : 'schema_name', 'schema_url'")
95

96 97 98 99 100 101 102 103 104 105
        try:
            self.schema = tableschema_from_url(self.url)
        except json.JSONDecodeError as e:
            log.exception(e)
            flash_error("Format de schéma non reconnu")
            abort(redirect(url_for('home')))
        except Exception as e:
            log.exception(e)
            flash_error("Erreur lors de l'obtention du schéma")
            abort(redirect(url_for('home')))
106 107 108 109

    def request_parameters(self):
        if self.name:
            return {
110
                'schema_name': self.schema_and_section_name,
111 112 113 114 115 116
                'schema_ref': '' if self.ref is None else self.ref
            }
        return {
            'schema_url': self.url
        }

117
    def find_section_title(self, section_name):
Christophe Benz's avatar
Christophe Benz committed
118 119 120 121 122 123
        if config.HOMEPAGE_CONFIG:
            for section in config.HOMEPAGE_CONFIG['sections']:
                if section["name"] == section_name:
                    return section.get("title")
        return None

124

125
def extract_source_data(source: ValidataResource, preview_rows_nb=5):
126
    """ Computes table preview """
127 128 129

    def stringify(val):
        """ Transform value into string """
Pierre Dittgen's avatar
Pierre Dittgen committed
130
        return '' if val is None else str(val)
131

132 133
    header = None
    rows = []
Pierre Dittgen's avatar
Pierre Dittgen committed
134
    nb_rows = 0
135

136
    tabulator_source, tabulator_options = source.build_tabulator_stream_args()
Pierre Dittgen's avatar
Pierre Dittgen committed
137
    with tabulator.Stream(tabulator_source, **tabulator_options) as stream:
138 139
        for row in stream:
            if header is None:
140
                header = ['' if v is None else v for v in row]
141
            else:
142
                rows.append(list(map(stringify, row)))
Pierre Dittgen's avatar
Pierre Dittgen committed
143
                nb_rows += 1
144
    preview_rows_nb = min(preview_rows_nb, nb_rows)
145 146
    return {'header': header,
            'rows_nb': nb_rows,
147 148 149
            'data_rows': rows,
            'preview_rows_nb': preview_rows_nb,
            'preview_rows': rows[:preview_rows_nb]}
150 151


Pierre Dittgen's avatar
Pierre Dittgen committed
152 153
def improve_errors(errors):
    """Add context to errors, converts markdown content to HTML"""
154

Pierre Dittgen's avatar
Pierre Dittgen committed
155 156
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
157

Pierre Dittgen's avatar
Pierre Dittgen committed
158 159 160 161
        # Context
        update_keys = {
            'context': 'body' if 'row-number' in err and not err['row-number'] is None else 'table',
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
162

Pierre Dittgen's avatar
Pierre Dittgen committed
163
        # markdown to HTML (with default values for 'title' and 'content')
Pierre Dittgen's avatar
Pierre Dittgen committed
164

Pierre Dittgen's avatar
Pierre Dittgen committed
165 166 167
        # Set default title if no title
        if not 'title' in err:
            update_keys['title'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
168

Pierre Dittgen's avatar
Pierre Dittgen committed
169 170 171 172
        # Convert message to markdown only if no content
        # => for pre-checks errors
        if 'message' in err and not 'content' in err:
            update_keys['message'] = commonmark(err['message'])
Pierre Dittgen's avatar
Pierre Dittgen committed
173

Pierre Dittgen's avatar
Pierre Dittgen committed
174 175 176
        # Else, default message
        elif not 'message' in err or err['message'] is None:
            update_keys['message'] = '[{}]'.format(err['code'])
Pierre Dittgen's avatar
Pierre Dittgen committed
177

Pierre Dittgen's avatar
Pierre Dittgen committed
178 179 180
        # Message content
        md_content = '*content soon available*' if not 'content' in err else err['content']
        update_keys['content'] = commonmark(md_content)
181

Pierre Dittgen's avatar
Pierre Dittgen committed
182
        return {**err, **update_keys}
183

Pierre Dittgen's avatar
Pierre Dittgen committed
184
    return list(map(improve_err, errors))
185 186


Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
187
def create_validata_ui_report(validata_core_report, schema_dict):
188 189 190 191 192 193
    """ Creates an error report easier to handle and display in templates:
        - only one table
        - errors are contextualized
        - error-counts is ok
        - errors are grouped by lines
        - errors are separated into "structure" and "body"
194
        - error messages are improved
195
    """
Pierre Dittgen's avatar
Pierre Dittgen committed
196
    report = copy.deepcopy(validata_core_report)
197 198 199 200 201 202 203 204 205

    # One table is enough
    del report['table-count']
    report['table'] = report['tables'][0]
    del report['tables']
    del report['table']['error-count']
    del report['table']['time']
    del report['table']['valid']
    del report['valid']
206 207 208
    # use _ instead of - to ease information picking in jinja2 template
    report['table']['row_count'] = report['table']['row-count']

Pierre Dittgen's avatar
Pierre Dittgen committed
209 210 211 212
    # Handy col_count info
    headers = report['table'].get('headers', [])
    report['table']['col_count'] = len(headers)

Pierre Dittgen's avatar
Pierre Dittgen committed
213
    # Computes column info
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
214 215
    fields_dict = {f['name']: (f.get('title', 'titre non défini'), f.get('description', ''))
                   for f in schema_dict.get('fields', [])}
Pierre Dittgen's avatar
Pierre Dittgen committed
216 217 218
    report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'colonne inconnue' for h in headers]
    report['table']['headers_description'] = [fields_dict[h][1]
                                              if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
219

220
    # Provide better (french) messages
Pierre Dittgen's avatar
Pierre Dittgen committed
221 222
    errors = improve_errors(report['table']['errors'])
    del report['table']['errors']
223

224 225 226 227 228 229 230
    # Count errors
    report['error_count'] = len(errors)
    del report['error-count']

    # Then group them in 2 groups : structure and body
    report['table']['errors'] = {'structure': [], 'body': []}
    for err in errors:
231
        if err['tag'] == 'structure':
232 233 234 235
            report['table']['errors']['structure'].append(err)
        else:
            report['table']['errors']['body'].append(err)

Pierre Dittgen's avatar
Pierre Dittgen committed
236
    # Checks if there are structure errors different to invalid-column-delimiter
Pierre Dittgen's avatar
Pierre Dittgen committed
237 238 239
    structure_errors = report['table']['errors']['structure']
    report['table']['do_display_body_errors'] = len(structure_errors) == 0 or \
        all(err['code'] == 'invalid-column-delimiter' for err in structure_errors)
240

Pierre Dittgen's avatar
Pierre Dittgen committed
241 242 243 244 245 246 247 248
    # Checks if a column comparison is needed
    header_errors = ('missing-headers', 'extra-headers', 'wrong-headers-order')
    structure_errors = [{**err, 'in_column_comp': err['code'] in header_errors} for err in structure_errors]
    report['table']['errors']['structure'] = structure_errors
    column_comparison_needed = any(err['in_column_comp'] == True for err in structure_errors)
    column_comparison_table = []
    if column_comparison_needed:
        column_comparison_table = []
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
249
        field_names = [f['name'] for f in schema_dict.get('fields', [])]
250
        has_case_errors = False
Pierre Dittgen's avatar
Pierre Dittgen committed
251 252
        for t in itertools.zip_longest(headers, field_names, fillvalue=''):
            status = 'ok' if t[0] == t[1] else 'ko'
253 254
            if not has_case_errors and status == 'ko' and t[0].lower() == t[1].lower():
                has_case_errors = True
Pierre Dittgen's avatar
Pierre Dittgen committed
255
            column_comparison_table.append((*t, status))
256 257 258 259 260
        info = {}
        info['table'] = column_comparison_table
        info['has_missing'] = len(headers) < len(field_names)
        info['has_case_errors'] = has_case_errors
        report['table']['column_comparison_info'] = info
Pierre Dittgen's avatar
Pierre Dittgen committed
261 262
    report['table']['column_comparison_needed'] = column_comparison_needed

Pierre Dittgen's avatar
Pierre Dittgen committed
263 264 265 266
    # Group body errors by row id
    rows = []
    current_row_id = 0
    for err in report['table']['errors']['body']:
267 268
        if not 'row-number' in err:
            print('ERR', err)
Pierre Dittgen's avatar
Pierre Dittgen committed
269 270 271 272 273 274 275 276 277 278 279 280 281
        row_id = err['row-number']
        del err['row-number']
        del err['context']
        if row_id != current_row_id:
            current_row_id = row_id
            rows.append({'row_id': current_row_id, 'errors': {}})

        column_id = err.get('column-number')
        if column_id is not None:
            del err['column-number']
            rows[-1]['errors'][column_id] = err
        else:
            rows[-1]['errors']['row'] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
282
    report['table']['errors']['body_by_rows'] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
283

284 285 286 287 288 289
    # Sort by error names in statistics
    stats = report['table']['error-stats']
    code_title_map = messages.ERROR_MESSAGE_DEFAULT_TITLE
    for key in ('structure-errors', 'value-errors'):
        # convert dict into tuples with french title instead of error code
        # and sorts by title
Christophe Benz's avatar
Christophe Benz committed
290 291
        stats[key]['count-by-code'] = sorted(((code_title_map.get(k, k), v)
                                              for k, v in stats[key]['count-by-code'].items()), key=itemgetter(0))
292

Pierre Dittgen's avatar
Pierre Dittgen committed
293 294 295
    return report


Pierre Dittgen's avatar
Pierre Dittgen committed
296 297 298 299 300 301 302
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
    structure = badge['structure']
    body = badge.get('body')

    # Bad structure, stop here
    if structure == 'KO':
303
        return ('structure invalide', 'red')
Pierre Dittgen's avatar
Pierre Dittgen committed
304 305 306 307 308 309

    # No body error
    if body == 'OK':
        return ('structure invalide', 'orange') if structure == 'WARN' else ('valide', 'green')

    # else compute quality ratio percent
Christophe Benz's avatar
Christophe Benz committed
310
    p = (1 - badge['error-ratio']) * 100.0
Pierre Dittgen's avatar
Pierre Dittgen committed
311 312 313 314 315 316 317 318
    msg = 'cellules valides : {:.1f}%'.format(p)
    return (msg, 'red') if body == 'KO' else (msg, 'orange')


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
319 320
    return ('{}static/v1.svg?label=Validata&message={}&color={}'.format(
        config.SHIELDS_IO_BASE_URL, quote_plus(msg), color), msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
321 322


323
def validate(schema_instance: SchemaInstance, source: ValidataResource):
324 325
    """ Validate source and display report """

326
    # Useful to receive response as JSON
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
327 328
    headers = {"Accept": "application/json"}

Pierre Dittgen's avatar
Pierre Dittgen committed
329
    try:
330
        if source.type == 'url':
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
331
            params = {
332
                "schema": schema_instance.url,
333
                "url": source.url,
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
334
            }
335
            response = requests.get(config.API_VALIDATE_ENDPOINT, params=params, headers=headers)
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
336
        else:
337
            files = {'file': (source.filename, source.build_reader())}
338
            data = {"schema": schema_instance.url}
339
            response = requests.post(config.API_VALIDATE_ENDPOINT, data=data, files=files, headers=headers)
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
340 341
    except requests.ConnectionError as err:
        logging.exception(err)
Christophe Benz's avatar
Christophe Benz committed
342
        flash_error("Erreur technique lors de la validation")
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
343
        return redirect(url_for('home'))
344

345 346 347
    if not response.ok:
        flash_error("Erreur technique lors de la validation")
        return redirect(compute_validation_form_url(schema_instance.request_parameters()))
348

349
    json_response = response.json()
350
    validata_core_report = json_response['report']
Pierre Dittgen's avatar
Pierre Dittgen committed
351
    badge_info = json_response.get('badge')
352

Pierre Dittgen's avatar
Pierre Dittgen committed
353
    # Computes badge from report and badge configuration
Pierre Dittgen's avatar
Pierre Dittgen committed
354 355 356 357
    badge_url, badge_msg = None, None
    display_badge = badge_info and config.SHIELDS_IO_BASE_URL
    if display_badge:
        badge_url, badge_msg = get_badge_url_and_message(badge_info)
Pierre Dittgen's avatar
Pierre Dittgen committed
358

359 360 361 362 363
    source_errors = [
        err
        for err in validata_core_report['tables'][0]['errors']
        if err['code'] in {'source-error', 'unknown-csv-dialect'}
    ]
364 365 366 367 368
    if source_errors:
        err = source_errors[0]
        msg = "l'encodage du fichier est invalide. Veuillez le corriger" if 'charmap' in err[
            'message'] else err['message']
        flash_error('Erreur de source : {}'.format(msg))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
369
        return redirect(url_for('custom_validator'))
370

371
    source_data = extract_source_data(source)
372

Pierre Dittgen's avatar
Pierre Dittgen committed
373
    # handle report date
374
    report_datetime = datetime.fromisoformat(validata_core_report['date']).astimezone()
375

Pierre Dittgen's avatar
Pierre Dittgen committed
376
    # Enhance validata_core_report
Pierre Dittgen's avatar
Pierre Dittgen committed
377
    validata_report = create_validata_ui_report(validata_core_report, schema_instance.schema.descriptor)
Pierre Dittgen's avatar
Pierre Dittgen committed
378

Pierre Dittgen's avatar
Pierre Dittgen committed
379
    # Display report to the user
380
    validator_form_url = compute_validation_form_url(schema_instance.request_parameters())
Christophe Benz's avatar
Christophe Benz committed
381
    schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
382 383
    pdf_report_url = url_for('pdf_report') + '?' + urlencode(schema_instance.request_parameters())

Christophe Benz's avatar
Christophe Benz committed
384
    return render_template('validation_report.html',
385 386 387 388 389 390 391 392 393
                           badge_msg=badge_msg,
                           badge_url=badge_url,
                           breadcrumbs=[
                               {'title': 'Accueil', 'url': url_for('home')},
                               {'title': schema_instance.section_title},
                               {'title': schema_info['title'], 'url': validator_form_url},
                               {'title': 'Rapport de validation'},
                           ],
                           display_badge=display_badge,
394
                           doc_url=schema_instance.doc_url,
395
                           pdf_report_url=pdf_report_url,
396
                           print_mode=request.args.get('print', 'false') == 'true',
397
                           report_str=json.dumps(validata_report, sort_keys=True, indent=2),
398 399 400
                           report=validata_report,
                           schema_current_version=schema_instance.ref,
                           schema_info=schema_info,
Christophe Benz's avatar
Christophe Benz committed
401
                           section_title=schema_instance.section_title,
402 403 404 405
                           source_data=source_data,
                           source=source,
                           validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
                           )
406 407


408 409
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
410
    iob = io.BytesIO()
411 412 413 414 415
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


416
def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
Pierre Dittgen's avatar
Pierre Dittgen committed
417 418 419 420 421
    """Replace catalog url within ui_config by schema references
    containing schema metadata properties"""

    extended_ui_config = ui_config.copy()
    for section in extended_ui_config['sections']:
Christophe Benz's avatar
Christophe Benz committed
422 423
        section_name = section['name']
        if section_name not in schema_catalog_map:
Pierre Dittgen's avatar
Pierre Dittgen committed
424
            continue
Christophe Benz's avatar
Christophe Benz committed
425
        schema_catalog = schema_catalog_map[section_name]
Pierre Dittgen's avatar
Pierre Dittgen committed
426 427 428
        schema_list = []
        for ref in schema_catalog.references:
            # Loads default table schema for each schema reference
429
            table_schema = tableschema_from_url(ref.get_schema_url())
Pierre Dittgen's avatar
Pierre Dittgen committed
430
            schema_list.append({
Christophe Benz's avatar
Christophe Benz committed
431
                'name': '{}.{}'.format(section_name, ref.name),
Pierre Dittgen's avatar
Pierre Dittgen committed
432 433 434 435 436
                # Extracts title, description, ...
                **extract_schema_metadata(table_schema)
            })
        section['catalog'] = schema_list
    return extended_ui_config
437

438 439 440 441 442 443
# Routes


@app.route('/')
def home():
    """ Home page """
444
    home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map)
Christophe Benz's avatar
Christophe Benz committed
445
    return render_template('home.html', config=home_config)
446 447


Pierre Dittgen's avatar
Pierre Dittgen committed
448 449
@app.route('/pdf')
def pdf_report():
450
    """PDF report generation"""
451
    err_prefix = 'Erreur de génération du rapport PDF'
452 453 454

    url_param = request.args.get('url')
    if not url_param:
455
        flash_error(err_prefix + ' : URL non fournie')
Pierre Dittgen's avatar
Pierre Dittgen committed
456
        return redirect(url_for('home'))
457

458
    schema_instance = SchemaInstance(request.args, schema_catalog_map)
Pierre Dittgen's avatar
Pierre Dittgen committed
459

460 461 462 463 464 465 466 467 468 469 470
    # Compute pdf url report
    base_url = url_for('custom_validator', _external=True)
    parameter_dict = {
        'input': 'url',
        'print': 'true',
        'url': url_param,
        **schema_instance.request_parameters()
    }
    validation_url = base_url + '?' + urlencode(parameter_dict)

    # Create temp file to save validation report
Pierre Dittgen's avatar
Pierre Dittgen committed
471
    with tempfile.NamedTemporaryFile(prefix='validata_{}_report_'.format(datetime.now().timestamp()), suffix='.pdf') as tmpfile:
Christophe Benz's avatar
Christophe Benz committed
472
        tmp_pdf_report = Path(tmpfile.name)
473

474
    # Use chromium headless to generate PDF from validation report page
475 476 477 478
    cmd = ['chromium', '--headless', '--disable-gpu',
           '--print-to-pdf={}'.format(str(tmp_pdf_report)), validation_url]
    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if result.returncode != 0:
479 480
        flash_error(err_prefix)
        log.error("Command %r returned an error: %r", cmd, result.stdout.decode('utf-8'))
481 482
        if tmp_pdf_report.exists():
            tmp_pdf_report.unlink()
Pierre Dittgen's avatar
Pierre Dittgen committed
483
        return redirect(url_for('home'))
484

485
    # Send PDF report
Pierre Dittgen's avatar
Pierre Dittgen committed
486
    pdf_filename = 'Rapport de validation {}.pdf'.format(datetime.now().strftime('%d-%m-%Y %Hh%M'))
Christophe Benz's avatar
Christophe Benz committed
487
    response = make_response(tmp_pdf_report.read_bytes())
Pierre Dittgen's avatar
Pierre Dittgen committed
488
    response.headers.set('Content-disposition', 'attachment', filename=pdf_filename)
489 490 491 492 493 494 495 496
    response.headers.set('Content-type', 'application/pdf')
    response.headers.set('Content-length', tmp_pdf_report.stat().st_size)

    tmp_pdf_report.unlink()

    return response


Pierre Dittgen's avatar
Pierre Dittgen committed
497 498 499 500 501
def extract_schema_metadata(table_schema: tableschema.Schema):
    """Gets author, contibutor, version...metadata from schema header"""
    return {k: v for k, v in table_schema.descriptor.items() if k != 'fields'}


Pierre Dittgen's avatar
Pierre Dittgen committed
502
def compute_schema_info(table_schema: tableschema.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
503
    """Factor code for validator form page"""
504

Pierre Dittgen's avatar
Pierre Dittgen committed
505 506
    # Schema URL + schema metadata info
    schema_info = {
Christophe Benz's avatar
Christophe Benz committed
507 508 509
        'path': schema_url,
        # a "path" metadata property can be found in Table Schema, and we'd like it to override the `schema_url`
        # given by the user (in case schema was given by URL)
Pierre Dittgen's avatar
Pierre Dittgen committed
510 511
        **extract_schema_metadata(table_schema)
    }
Christophe Benz's avatar
Christophe Benz committed
512
    return schema_info
513 514


515
def compute_validation_form_url(request_parameters: dict):
516 517
    """Computes validation form url with schema URL parameter"""
    url = url_for('custom_validator')
518
    param_list = ['{}={}'.format(k, quote_plus(v)) for k, v in request_parameters.items()]
519
    return "{}?{}".format(url, '&'.join(param_list))
Pierre Dittgen's avatar
Pierre Dittgen committed
520 521


522
@app.route('/table-schema', methods=['GET', 'POST'])
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
523
def custom_validator():
524
    """Validator form"""
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
525 526 527

    if request.method == 'GET':

528 529
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
530
        input_param = request.args.get('input')
531 532

        # url of resource to be validated
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
533 534
        url_param = request.args.get("url")

535
        schema_instance = SchemaInstance(request.args, schema_catalog_map)
Pierre Dittgen's avatar
Pierre Dittgen committed
536

Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
537 538
        # First form display
        if input_param is None:
Christophe Benz's avatar
Christophe Benz committed
539 540
            schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
            return render_template('validation_form.html',
541
                                   branches=schema_instance.branches,
Christophe Benz's avatar
Christophe Benz committed
542 543 544 545
                                   breadcrumbs=[
                                       {'url': url_for('home'), 'title': 'Accueil'},
                                       {'title': schema_instance.section_title},
                                       {'title': schema_info['title']},
Christophe Benz's avatar
Christophe Benz committed
546 547 548 549 550 551 552 553
                                   ],
                                   doc_url=schema_instance.doc_url,
                                   schema_current_version=schema_instance.ref,
                                   schema_info=schema_info,
                                   schema_params=schema_instance.request_parameters(),
                                   section_title=schema_instance.section_title,
                                   tags=schema_instance.tags,
                                   )
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
554 555 556

        # Process URL
        else:
557
            if not url_param:
Christophe Benz's avatar
Christophe Benz committed
558
                flash_error("Vous n'avez pas indiqué d'URL à valider")
559 560 561 562 563 564
                return redirect(compute_validation_form_url(schema_instance.request_parameters()))
            return validate(schema_instance, URLValidataResource(url_param))

    elif request.method == 'POST':

        schema_instance = SchemaInstance(request.form, schema_catalog_map)
Pierre Dittgen's avatar
Pierre Dittgen committed
565 566 567 568

        input_param = request.form.get('input')
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
569
            return redirect(compute_validation_form_url(schema_instance.request_parameters()))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
570 571 572 573 574 575

        # File validation
        if input_param == 'file':
            f = request.files.get('file')
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
576
                return redirect(compute_validation_form_url(schema_instance.request_parameters()))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
577

578
            return validate(schema_instance, UploadedFileValidataResource(f.filename, bytes_data(f)))
Pierre Dittgen's avatar
wip  
Pierre Dittgen committed
579

580 581 582 583
        return 'Combinaison de paramètres non supportée', 400

    else:
        return "Method not allowed", 405