views.py 23.7 KB
Newer Older
1
2
3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
5
import io
6
import json
Christophe Benz's avatar
Christophe Benz committed
7
import logging
8
from collections import Counter
9
from datetime import datetime
Christophe Benz's avatar
Christophe Benz committed
10
from urllib.parse import urlencode, urljoin
11

Pierre Dittgen's avatar
Pierre Dittgen committed
12
import frictionless
13
import jsonschema
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
14
import requests
Christophe Benz's avatar
Christophe Benz committed
15
import validata_core
16
from commonmark import commonmark
17
from flask import abort, make_response, redirect, render_template, request, url_for
18
from opendataschema import GitSchemaReference, by_commit_date
Pierre Dittgen's avatar
Pierre Dittgen committed
19
20
from validata_core.helpers import (
    FileContentValidataResource,
Christophe Benz's avatar
Christophe Benz committed
21
22
    URLValidataResource,
    ValidataResource,
23
24
    is_body_error,
    is_structure_error,
Pierre Dittgen's avatar
Pierre Dittgen committed
25
)
26

Christophe Benz's avatar
Christophe Benz committed
27
from . import app, config, fetch_schema, pdf_service, schema_catalog_registry
Pierre Dittgen's avatar
Pierre Dittgen committed
28
from .model import Section
29
from .ui_util import flash_error, flash_warning
30
31
from .validata_util import strip_accents

Christophe Benz's avatar
Christophe Benz committed
32
33
log = logging.getLogger(__name__)

34

35
36
37
38
39
def get_schema_catalog(section_name):
    """Return a schema catalog associated to a section_name"""
    return schema_catalog_registry.build_schema_catalog(section_name)


40
41
42
class SchemaInstance:
    """Handy class to handle schema information"""

43
    def __init__(self, parameter_dict):
Christophe Benz's avatar
Christophe Benz committed
44
45
        """Initializes schema instance from requests dict and tableschema Catalog
        (for name ref)."""
46
47
48
49
50
51
52
53
54
        self.section_name = None
        self.section_title = None
        self.name = None
        self.url = None
        self.ref = None
        self.reference = None
        self.doc_url = None
        self.branches = None
        self.tags = None
55
56

        # From schema_url
Christophe Benz's avatar
Christophe Benz committed
57
        if parameter_dict.get("schema_url"):
58
59
            self.url = parameter_dict["schema_url"]
            self.section_title = "Autre schéma"
60
61

        # from schema_name (and schema_ref)
Pierre Dittgen's avatar
Pierre Dittgen committed
62
63
64
        elif parameter_dict.get("schema_name"):
            self.schema_and_section_name = parameter_dict["schema_name"]
            self.ref = parameter_dict.get("schema_ref")
65

Pierre Dittgen's avatar
Pierre Dittgen committed
66
            # Check schema name
Pierre Dittgen's avatar
Pierre Dittgen committed
67
            chunks = self.schema_and_section_name.split(".")
Pierre Dittgen's avatar
Pierre Dittgen committed
68
            if len(chunks) != 2:
69
                abort(400, "Paramètre 'schema_name' invalide")
Pierre Dittgen's avatar
Pierre Dittgen committed
70

71
72
            self.section_name, self.name = chunks
            self.section_title = self.find_section_title(self.section_name)
Pierre Dittgen's avatar
Pierre Dittgen committed
73
74

            # Look for schema catalog first
75
76
77
78
79
            try:
                table_schema_catalog = get_schema_catalog(self.section_name)
            except Exception as ex:
                log.exception(ex)
                abort(400, "Erreur de traitement du catalogue")
Pierre Dittgen's avatar
Pierre Dittgen committed
80
            if table_schema_catalog is None:
81
                abort(400, "Catalogue indisponible")
82
83
84

            schema_reference = table_schema_catalog.reference_by_name.get(self.name)
            if schema_reference is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
85
86
                abort(
                    400,
Christophe Benz's avatar
Christophe Benz committed
87
88
                    f"Schéma {self.name!r} non trouvé dans le catalogue de la "
                    f"section {self.section_name!r}",
Pierre Dittgen's avatar
Pierre Dittgen committed
89
                )
90
91

            if isinstance(schema_reference, GitSchemaReference):
Pierre Dittgen's avatar
Pierre Dittgen committed
92
93
94
                self.tags = sorted(
                    schema_reference.iter_tags(), key=by_commit_date, reverse=True
                )
95
                if self.ref is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
                    schema_ref = (
                        self.tags[0]
                        if self.tags
                        else schema_reference.get_default_branch()
                    )
                    abort(
                        redirect(
                            compute_validation_form_url(
                                {
                                    "schema_name": self.schema_and_section_name,
                                    "schema_ref": schema_ref.name,
                                }
                            )
                        )
                    )
111
                tag_names = [tag.name for tag in self.tags]
Pierre Dittgen's avatar
Pierre Dittgen committed
112
113
114
115
116
117
118
119
                self.branches = [
                    branch
                    for branch in schema_reference.iter_branches()
                    if branch.name not in tag_names
                ]
                self.doc_url = schema_reference.get_doc_url(
                    ref=self.ref
                ) or schema_reference.get_project_url(ref=self.ref)
120

121
            self.url = schema_reference.get_schema_url(ref=self.ref)
122
123

        else:
Pierre Dittgen's avatar
Pierre Dittgen committed
124
            flash_error("Erreur dans la récupération des informations de schéma")
Pierre Dittgen's avatar
Pierre Dittgen committed
125
            abort(redirect(url_for("home")))
126

127
        try:
Pierre Dittgen's avatar
Pierre Dittgen committed
128
            self.schema = fetch_schema(self.url)
129
130
        except json.JSONDecodeError as e:
            log.exception(e)
Pierre Dittgen's avatar
Pierre Dittgen committed
131
            flash_error("Le format du schéma n'est pas reconnu")
Pierre Dittgen's avatar
Pierre Dittgen committed
132
            abort(redirect(url_for("home")))
133
134
        except Exception as e:
            log.exception(e)
Pierre Dittgen's avatar
Pierre Dittgen committed
135
            flash_error("Impossible de récupérer le schéma")
Pierre Dittgen's avatar
Pierre Dittgen committed
136
            abort(redirect(url_for("home")))
137
138
139
140

    def request_parameters(self):
        if self.name:
            return {
Pierre Dittgen's avatar
Pierre Dittgen committed
141
142
                "schema_name": self.schema_and_section_name,
                "schema_ref": "" if self.ref is None else self.ref,
143
            }
Pierre Dittgen's avatar
Pierre Dittgen committed
144
        return {"schema_url": self.url}
145

146
    def find_section_title(self, section_name):
147
        if config.CONFIG:
Pierre Dittgen's avatar
Pierre Dittgen committed
148
149
150
            for section in config.CONFIG.homepage.sections:
                if section.name == section_name:
                    return section.title
Christophe Benz's avatar
Christophe Benz committed
151
152
        return None

153

154
155
156
def build_template_source_data(header, rows, preview_rows_nb=5):
    """Build source data information to preview in validation report page."""
    source_header_info = [(colname, False) for colname in header]
Pierre Dittgen's avatar
Pierre Dittgen committed
157

158
    rows_count = len(rows)
Pierre Dittgen's avatar
Pierre Dittgen committed
159
    preview_rows_count = min(preview_rows_nb, rows_count)
Pierre Dittgen's avatar
Pierre Dittgen committed
160
    return {
Pierre Dittgen's avatar
Pierre Dittgen committed
161
        "source_header_info": source_header_info,
162
        "header": header,
Pierre Dittgen's avatar
Pierre Dittgen committed
163
        "rows_nb": rows_count,
164
        "data_rows": rows,
Pierre Dittgen's avatar
Pierre Dittgen committed
165
        "preview_rows_count": preview_rows_count,
166
        "preview_rows": rows[:preview_rows_count],
Pierre Dittgen's avatar
Pierre Dittgen committed
167
    }
168
169


170
def build_ui_errors(errors):
Pierre Dittgen's avatar
Pierre Dittgen committed
171
    """Add context to errors, converts markdown content to HTML"""
172

Pierre Dittgen's avatar
Pierre Dittgen committed
173
174
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
175

Pierre Dittgen's avatar
Pierre Dittgen committed
176
177
        # Context
        update_keys = {
Pierre Dittgen's avatar
Pierre Dittgen committed
178
            "context": "body"
179
            if "row-number" in err and err["row-number"] is not None
Pierre Dittgen's avatar
Pierre Dittgen committed
180
            else "table",
Pierre Dittgen's avatar
Pierre Dittgen committed
181
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
182

183
        # Set title
184
        if "title" not in err:
185
            update_keys["title"] = err["name"]
186

187
        # Set content
188
        content = "*content soon available*"
189
        if "message" in err:
190
            content = err["message"]
191
192
        elif "description" in err:
            content = err["description"]
193
        update_keys["content"] = commonmark(content)
194

Pierre Dittgen's avatar
Pierre Dittgen committed
195
        return {**err, **update_keys}
196

Pierre Dittgen's avatar
Pierre Dittgen committed
197
    return list(map(improve_err, errors))
198
199


Pierre Dittgen's avatar
Pierre Dittgen committed
200
def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict):
201
202
203
204
205
206
207
    """Creates an error report easier to handle and display in templates:
    - only one table
    - errors are contextualized
    - error-counts is ok
    - errors are grouped by lines
    - errors are separated into "structure" and "body"
    - error messages are improved
208
    """
209
    v_report = copy.deepcopy(validata_core_report)
210

211
212
213
    # Create a new UI report from information picked in validata report
    ui_report = {}
    ui_report["table"] = {}
214

215
    # source headers
Pierre Dittgen's avatar
Pierre Dittgen committed
216
    headers = v_report.table["header"]
217
    ui_report["table"]["header"] = headers
Pierre Dittgen's avatar
Pierre Dittgen committed
218

219
    # source dimension
Pierre Dittgen's avatar
Pierre Dittgen committed
220
    ui_report["table"]["col_count"] = len(headers)
Pierre Dittgen's avatar
Pierre Dittgen committed
221
    ui_report["table"]["row_count"] = rows_count
222
223

    # Computes column info from schema
Pierre Dittgen's avatar
Pierre Dittgen committed
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
    fields_dict = {
        f["name"]: (f.get("title", f["name"]), f.get("description", ""))
        for f in schema_dict.get("fields", [])
    }
    ui_report["table"]["headers_title"] = [
        fields_dict[h][0] if h in fields_dict else "Colonne inconnue" for h in headers
    ]
    ui_report["table"]["headers_description"] = [
        fields_dict[h][1]
        if h in fields_dict
        else "Cette colonne n'est pas définie dans le schema"
        for h in headers
    ]
    missing_headers = [
        err["message-data"]["column-name"]
        for err in v_report.table["errors"]
        if err["code"] == "missing-header"
    ]
    ui_report["table"]["cols_alert"] = [
        "table-danger" if h not in fields_dict or h in missing_headers else ""
        for h in headers
    ]
246

247
    # prepare error structure for UI needs
Pierre Dittgen's avatar
Pierre Dittgen committed
248
    errors = build_ui_errors(v_report.table["errors"])
249

Pierre Dittgen's avatar
Pierre Dittgen committed
250
    # Count errors and warnings
Pierre Dittgen's avatar
Pierre Dittgen committed
251
    ui_report["error_count"] = len(errors)
Pierre Dittgen's avatar
Pierre Dittgen committed
252
253
    ui_report["warn_count"] = len(v_report.table["structure_warnings"])
    ui_report["warnings"] = v_report.table["structure_warnings"]
254
255

    # Then group them in 2 groups : structure and body
Pierre Dittgen's avatar
Pierre Dittgen committed
256
    ui_report["table"]["errors"] = {"structure": [], "body": []}
257
    for err in errors:
258
        if is_structure_error(err):
Pierre Dittgen's avatar
Pierre Dittgen committed
259
            ui_report["table"]["errors"]["structure"].append(err)
260
        elif is_body_error(err):
Pierre Dittgen's avatar
Pierre Dittgen committed
261
            ui_report["table"]["errors"]["body"].append(err)
262

Pierre Dittgen's avatar
Pierre Dittgen committed
263
264
265
    # Group body errors by row id
    rows = []
    current_row_id = 0
Pierre Dittgen's avatar
Pierre Dittgen committed
266
    for err in ui_report["table"]["errors"]["body"]:
267
        if "rowPosition" not in err:
268
            continue
Pierre Dittgen's avatar
Pierre Dittgen committed
269
        row_id = err["rowPosition"]
Pierre Dittgen's avatar
Pierre Dittgen committed
270
271
        if row_id != current_row_id:
            current_row_id = row_id
Pierre Dittgen's avatar
Pierre Dittgen committed
272
            rows.append({"row_id": current_row_id, "errors": {}})
Pierre Dittgen's avatar
Pierre Dittgen committed
273

Pierre Dittgen's avatar
Pierre Dittgen committed
274
        column_id = err.get("fieldPosition")
Pierre Dittgen's avatar
Pierre Dittgen committed
275
        if column_id is not None:
Pierre Dittgen's avatar
Pierre Dittgen committed
276
            rows[-1]["errors"][column_id] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
277
        else:
Pierre Dittgen's avatar
Pierre Dittgen committed
278
279
            rows[-1]["errors"]["row"] = err
    ui_report["table"]["errors"]["body_by_rows"] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
280

281
    # Sort by error names in statistics
282
    ui_report["table"]["count-by-code"] = {}
283
284
    stats = {}
    total_errors_count = 0
Pierre Dittgen's avatar
Pierre Dittgen committed
285
    for key in ("structure", "body"):
286
287
        # convert dict into tuples with french title instead of error code
        # and sorts by title
288
        key_errors = ui_report["table"]["errors"][key]
289
        key_errors_count = len(key_errors)
290
        ct = Counter(ke["name"] for ke in key_errors)
291
292
293

        error_stats = {
            "count": key_errors_count,
Pierre Dittgen's avatar
Pierre Dittgen committed
294
            "count-by-code": sorted((k, v) for k, v in ct.items()),
295
296
297
298
        }
        total_errors_count += key_errors_count

        # Add error rows count
Pierre Dittgen's avatar
Pierre Dittgen committed
299
300
301
302
        if key == "body":
            error_rows = {
                err["rowPosition"] for err in key_errors if "rowPosition" in err
            }
303
304
305
306
307
            error_stats["rows-count"] = len(error_rows)

        stats[f"{key}-errors"] = error_stats

    stats["count"] = total_errors_count
308
    ui_report["table"]["error-stats"] = stats
309

310
    return ui_report
Pierre Dittgen's avatar
Pierre Dittgen committed
311
312


Pierre Dittgen's avatar
Pierre Dittgen committed
313
314
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
Pierre Dittgen's avatar
Pierre Dittgen committed
315
316
    structure = badge["structure"]
    body = badge.get("body")
Pierre Dittgen's avatar
Pierre Dittgen committed
317
318

    # Bad structure, stop here
Pierre Dittgen's avatar
Pierre Dittgen committed
319
320
    if structure == "KO":
        return ("structure invalide", "red")
Pierre Dittgen's avatar
Pierre Dittgen committed
321
322

    # No body error
Pierre Dittgen's avatar
Pierre Dittgen committed
323
324
    if body == "OK":
        return (
Pierre Dittgen's avatar
Pierre Dittgen committed
325
            ("partiellement valide", "yellowgreen")
Pierre Dittgen's avatar
Pierre Dittgen committed
326
327
328
            if structure == "WARN"
            else ("valide", "green")
        )
Pierre Dittgen's avatar
Pierre Dittgen committed
329
330

    # else compute quality ratio percent
Pierre Dittgen's avatar
Pierre Dittgen committed
331
332
333
    p = (1 - badge["error-ratio"]) * 100.0
    msg = "cellules valides : {:.1f}%".format(p)
    return (msg, "red") if body == "KO" else (msg, "orange")
Pierre Dittgen's avatar
Pierre Dittgen committed
334
335
336
337
338
339


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
Christophe Benz's avatar
Christophe Benz committed
340
    badge_url = "{}?{}".format(
Pierre Dittgen's avatar
Pierre Dittgen committed
341
342
        urljoin(config.SHIELDS_IO_BASE_URL, "/static/v1.svg"),
        urlencode({"label": "Validata", "message": msg, "color": color}),
Christophe Benz's avatar
Christophe Benz committed
343
344
    )
    return (badge_url, msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
345
346


347
def validate(schema_instance: SchemaInstance, validata_resource: ValidataResource):
348
349
    """ Validate source and display report """

Pierre Dittgen's avatar
Pierre Dittgen committed
350
    # Parse source data once
351
    header, rows = validata_resource.extract_tabular_data()
Pierre Dittgen's avatar
Pierre Dittgen committed
352
    rows_count = len(rows)
Pierre Dittgen's avatar
Pierre Dittgen committed
353
354

    # Call validata_core with parsed data
Pierre Dittgen's avatar
Pierre Dittgen committed
355
    validata_core_report = validata_core.validate(
356
        [header] + rows, schema_instance.schema
Pierre Dittgen's avatar
Pierre Dittgen committed
357
    )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
358

359
    # disable badge
Pierre Dittgen's avatar
Pierre Dittgen committed
360
    badge_config = config.BADGE_CONFIG
361

Pierre Dittgen's avatar
Pierre Dittgen committed
362
    # Computes badge from report and badge configuration
Pierre Dittgen's avatar
Pierre Dittgen committed
363
    badge_url, badge_msg = None, None
Pierre Dittgen's avatar
Pierre Dittgen committed
364
    display_badge = badge_config and config.SHIELDS_IO_BASE_URL
Pierre Dittgen's avatar
Pierre Dittgen committed
365
    if display_badge:
Pierre Dittgen's avatar
Pierre Dittgen committed
366
367
368
        badge_stats = validata_core.compute_badge_metrics(
            validata_core_report, badge_config
        )
369
370
371
372
373
374
375
376
377
378
379
        if badge_stats:
            badge_url, badge_msg = get_badge_url_and_message(badge_stats)

    # Non table errors
    if validata_core_report["errors"]:
        non_table_error = validata_core_report["errors"][0]
        msg = non_table_error["note"]
        flash_error(f"Une erreur est survenue empêchant la validation : {msg}")
        return redirect(
            compute_validation_form_url(schema_instance.request_parameters())
        )
Pierre Dittgen's avatar
Pierre Dittgen committed
380

381
    # Source error
382
383
    source_errors = [
        err
Pierre Dittgen's avatar
Pierre Dittgen committed
384
385
        for err in validata_core_report["tables"][0]["errors"]
        if err["code"] in {"source-error", "unknown-csv-dialect"}
386
    ]
387
388
    if source_errors:
        err = source_errors[0]
Pierre Dittgen's avatar
Pierre Dittgen committed
389
390
391
392
393
394
395
        msg = (
            "l'encodage du fichier est invalide. Veuillez le corriger"
            if "charmap" in err["message"]
            else err["message"]
        )
        flash_error("Erreur de source : {}".format(msg))
        return redirect(url_for("custom_validator"))
396

Pierre Dittgen's avatar
Pierre Dittgen committed
397
    # handle report date
Pierre Dittgen's avatar
Pierre Dittgen committed
398
    report_datetime = datetime.fromisoformat(validata_core_report["date"]).astimezone()
399

400
    # create ui_report
Pierre Dittgen's avatar
Pierre Dittgen committed
401
402
403
    ui_report = create_validata_ui_report(
        rows_count, validata_core_report, schema_instance.schema
    )
Pierre Dittgen's avatar
Pierre Dittgen committed
404

Pierre Dittgen's avatar
Pierre Dittgen committed
405
    # Display report to the user
Pierre Dittgen's avatar
Pierre Dittgen committed
406
407
408
    validator_form_url = compute_validation_form_url(
        schema_instance.request_parameters()
    )
Christophe Benz's avatar
Christophe Benz committed
409
    schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
410
411

    # Build PDF report URL
412
413
414
    # PDF report is available if:
    # - a pdf_service has been configured
    # - tabular resource to validate is defined as an URL
415
    pdf_report_url = None
416
    if pdf_service and isinstance(validata_resource, URLValidataResource):
417
418
        base_url = url_for("pdf_report")
        query_string = urlencode(
419
420
421
422
            {
                **schema_instance.request_parameters(),
                "url": validata_resource.url,
            }
Pierre Dittgen's avatar
Pierre Dittgen committed
423
        )
424
        pdf_report_url = f"{base_url}?{query_string}"
Pierre Dittgen's avatar
Pierre Dittgen committed
425
426
427

    return render_template(
        "validation_report.html",
428
        config=config.CONFIG,
Pierre Dittgen's avatar
Pierre Dittgen committed
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
        badge_msg=badge_msg,
        badge_url=badge_url,
        breadcrumbs=[
            {"title": "Accueil", "url": url_for("home")},
            {"title": schema_instance.section_title},
            {"title": schema_info["title"], "url": validator_form_url},
            {"title": "Rapport de validation"},
        ],
        display_badge=display_badge,
        doc_url=schema_instance.doc_url,
        pdf_report_url=pdf_report_url,
        print_mode=request.args.get("print", "false") == "true",
        report=ui_report,
        schema_current_version=schema_instance.ref,
        schema_info=schema_info,
        section_title=schema_instance.section_title,
445
446
        source_data=build_template_source_data(header, rows),
        source=validata_resource,
Pierre Dittgen's avatar
Pierre Dittgen committed
447
448
        validation_date=report_datetime.strftime("le %d/%m/%Y à %Hh%M"),
    )
449
450


451
452
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
453
    iob = io.BytesIO()
454
455
456
457
458
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


Pierre Dittgen's avatar
Pierre Dittgen committed
459
def retrieve_schema_catalog(section: Section):
Pierre Dittgen's avatar
Pierre Dittgen committed
460
    """Retrieve schema catalog and return formatted error if it fails."""
461
462
463
464

    def format_error_message(err_message, exc):
        """Prepare a bootstrap error message with details if wanted."""

Pierre Dittgen's avatar
Pierre Dittgen committed
465
        exception_text = "\n".join([str(arg) for arg in exc.args])
466

Pierre Dittgen's avatar
Pierre Dittgen committed
467
        return f"""{err_msg}
468
        <div class="float-right">
Christophe Benz's avatar
Christophe Benz committed
469
470
            <button type="button" class="btn btn-info btn-xs" data-toggle="collapse"
                data-target="#exception_info">détails</button>
471
472
473
474
475
476
477
        </div>
        <div id="exception_info" class="collapse">
                <pre>{exception_text}</pre>
        </div>
"""

    try:
Pierre Dittgen's avatar
Pierre Dittgen committed
478
        schema_catalog = get_schema_catalog(section.name)
479
        return (schema_catalog, None)
Pierre Dittgen's avatar
Pierre Dittgen committed
480

481
482
483
484
485
486
487
488
489
    except Exception as exc:
        log.exception(exc)
        err_msg = "une erreur s'est produite"
        if isinstance(exc, requests.ConnectionError):
            err_msg = "problème de connexion"
        elif isinstance(exc, json.decoder.JSONDecodeError):
            err_msg = "format JSON incorrect"
        elif isinstance(exc, jsonschema.exceptions.ValidationError):
            err_msg = "le catalogue ne respecte pas le schéma de référence"
Pierre Dittgen's avatar
Pierre Dittgen committed
490

491
        error_catalog = {
Pierre Dittgen's avatar
Pierre Dittgen committed
492
            **{k: v for k, v in section.dict().items() if k != "catalog"},
Pierre Dittgen's avatar
Pierre Dittgen committed
493
            "err": format_error_message(err_msg, exc),
494
495
496
497
        }
        return None, error_catalog


498
499
500
# Routes


Pierre Dittgen's avatar
Pierre Dittgen committed
501
@app.route("/")
502
503
def home():
    """ Home page """
504
505
506

    def iter_sections():
        """Yield sections of the home page, filled with schema metadata."""
Pierre Dittgen's avatar
Pierre Dittgen committed
507
508

        # Iterate on all sections
Pierre Dittgen's avatar
Pierre Dittgen committed
509
        for section in config.CONFIG.homepage.sections:
Pierre Dittgen's avatar
Pierre Dittgen committed
510

Pierre Dittgen's avatar
Pierre Dittgen committed
511
            # section with only links to external validators
Pierre Dittgen's avatar
Pierre Dittgen committed
512
            if section.links:
Pierre Dittgen's avatar
Pierre Dittgen committed
513
                yield section
514
                continue
515

Pierre Dittgen's avatar
Pierre Dittgen committed
516
            # section with catalog
Pierre Dittgen's avatar
Pierre Dittgen committed
517
            if section.catalog is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
518
519
520
                # skip section
                continue

521
522
523
524
525
            # retrieving schema catatalog
            schema_catalog, catalog_error = retrieve_schema_catalog(section)
            if schema_catalog is None:
                yield catalog_error
                continue
Pierre Dittgen's avatar
Pierre Dittgen committed
526
527
528
529
530

            # Working on catalog
            schema_info_list = []
            for schema_reference in schema_catalog.references:
                # Loads default table schema for each schema reference
Pierre Dittgen's avatar
Pierre Dittgen committed
531
                schema_info = {"name": schema_reference.name}
532
                try:
Pierre Dittgen's avatar
Pierre Dittgen committed
533
                    table_schema = fetch_schema(schema_reference.get_schema_url())
Pierre Dittgen's avatar
Pierre Dittgen committed
534
                except json.JSONDecodeError:
Pierre Dittgen's avatar
Pierre Dittgen committed
535
                    schema_info["err"] = True
Christophe Benz's avatar
Christophe Benz committed
536
537
538
539
                    schema_info["title"] = (
                        f"le format du schéma « {schema_info['name']} » "
                        "n'est pas reconnu"
                    )
540
                except Exception:
Pierre Dittgen's avatar
Pierre Dittgen committed
541
                    schema_info["err"] = True
Christophe Benz's avatar
Christophe Benz committed
542
543
544
                    schema_info["title"] = (
                        f"le schéma « {schema_info['name']} » " "n'est pas disponible"
                    )
545
                else:
Pierre Dittgen's avatar
Pierre Dittgen committed
546
547
548
                    schema_info["title"] = (
                        table_schema.get("title") or schema_info["name"]
                    )
Pierre Dittgen's avatar
Pierre Dittgen committed
549
550
                schema_info_list.append(schema_info)
            schema_info_list = sorted(
Pierre Dittgen's avatar
Pierre Dittgen committed
551
552
                schema_info_list, key=lambda sc: strip_accents(sc["title"].lower())
            )
Pierre Dittgen's avatar
Pierre Dittgen committed
553

Pierre Dittgen's avatar
Pierre Dittgen committed
554
            yield {
Pierre Dittgen's avatar
Pierre Dittgen committed
555
                **{k: v for k, v in section.dict().items() if k != "catalog"},
Pierre Dittgen's avatar
Pierre Dittgen committed
556
557
                "catalog": schema_info_list,
            }
558

Pierre Dittgen's avatar
Pierre Dittgen committed
559
560
561
    return render_template(
        "home.html", config=config.CONFIG, sections=list(iter_sections())
    )
562
563


Pierre Dittgen's avatar
Pierre Dittgen committed
564
@app.route("/pdf")
Pierre Dittgen's avatar
Pierre Dittgen committed
565
def pdf_report():
566
    """PDF report generation"""
Pierre Dittgen's avatar
Pierre Dittgen committed
567
    err_prefix = "Erreur de génération du rapport PDF"
568

Pierre Dittgen's avatar
Pierre Dittgen committed
569
    url_param = request.args.get("url")
570
    if not url_param:
Pierre Dittgen's avatar
Pierre Dittgen committed
571
572
        flash_error(err_prefix + " : URL non fournie")
        return redirect(url_for("home"))
573

574
575
576
577
578
    if pdf_service is None:
        flash_error(err_prefix + " : service de génération non configuré")
        return redirect(url_for("home"))

    # Compute validation report URL
579
    schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
580

Pierre Dittgen's avatar
Pierre Dittgen committed
581
    base_url = url_for("custom_validator", _external=True)
582
    parameter_dict = {
Pierre Dittgen's avatar
Pierre Dittgen committed
583
584
585
586
        "input": "url",
        "print": "true",
        "url": url_param,
        **schema_instance.request_parameters(),
587
    }
Christophe Benz's avatar
Christophe Benz committed
588
    validation_url = "{}?{}".format(base_url, urlencode(parameter_dict))
Pierre Dittgen's avatar
Pierre Dittgen committed
589
    log.info("Validation URL = %s", validation_url)
590

591
592
593
    # Ask for PDF report generation
    try:
        pdf_bytes_content = pdf_service.render(validation_url)
Christophe Benz's avatar
Christophe Benz committed
594
    except Exception:
595
596
597
        log.exception(err_prefix)
        flash_error(err_prefix + " : contactez votre administrateur")
        return redirect(url_for("home"))
598

599
600
601
602
    # Compute pdf filename
    pdf_filename = "Rapport de validation {}.pdf".format(
        datetime.now().strftime("%d-%m-%Y %Hh%M")
    )
603

604
605
606
607
608
609
    # Prepare and send response
    response = make_response(pdf_bytes_content)
    response.headers.set("Content-Disposition", "attachment", filename=pdf_filename)
    response.headers.set("Content-Length", len(pdf_bytes_content))
    response.headers.set("Content-Type", "application/pdf")
    return response
610
611


Pierre Dittgen's avatar
Pierre Dittgen committed
612
def extract_schema_metadata(table_schema: frictionless.Schema):
Pierre Dittgen's avatar
Pierre Dittgen committed
613
    """Gets author, contibutor, version...metadata from schema header"""
Pierre Dittgen's avatar
Pierre Dittgen committed
614
    return {k: v for k, v in table_schema.items() if k != "fields"}
Pierre Dittgen's avatar
Pierre Dittgen committed
615
616


Pierre Dittgen's avatar
Pierre Dittgen committed
617
def compute_schema_info(table_schema: frictionless.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
618
    """Factor code for validator form page"""
619

Pierre Dittgen's avatar
Pierre Dittgen committed
620
621
    # Schema URL + schema metadata info
    schema_info = {
Pierre Dittgen's avatar
Pierre Dittgen committed
622
        "path": schema_url,
Christophe Benz's avatar
Christophe Benz committed
623
624
        # a "path" metadata property can be found in Table Schema,
        # and we'd like it to override the `schema_url`
Christophe Benz's avatar
Christophe Benz committed
625
        # given by the user (in case schema was given by URL)
Pierre Dittgen's avatar
Pierre Dittgen committed
626
        **extract_schema_metadata(table_schema),
Pierre Dittgen's avatar
Pierre Dittgen committed
627
    }
Christophe Benz's avatar
Christophe Benz committed
628
    return schema_info
629
630


631
def compute_validation_form_url(request_parameters: dict):
632
    """Computes validation form url with schema URL parameter"""
Pierre Dittgen's avatar
Pierre Dittgen committed
633
    url = url_for("custom_validator")
Christophe Benz's avatar
Christophe Benz committed
634
    return "{}?{}".format(url, urlencode(request_parameters))
Pierre Dittgen's avatar
Pierre Dittgen committed
635
636


Pierre Dittgen's avatar
Pierre Dittgen committed
637
@app.route("/table-schema", methods=["GET", "POST"])
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
638
def custom_validator():
639
    """Validator form"""
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
640

Pierre Dittgen's avatar
Pierre Dittgen committed
641
    if request.method == "GET":
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
642

643
644
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
Pierre Dittgen committed
645
        input_param = request.args.get("input")
646
647

        # url of resource to be validated
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
648
649
        url_param = request.args.get("url")

650
        schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
651

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
652
653
        # First form display
        if input_param is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
654
655
656
657
658
            schema_info = compute_schema_info(
                schema_instance.schema, schema_instance.url
            )
            return render_template(
                "validation_form.html",
659
                config=config.CONFIG,
Pierre Dittgen's avatar
Pierre Dittgen committed
660
661
662
663
664
665
666
667
668
669
670
671
672
                branches=schema_instance.branches,
                breadcrumbs=[
                    {"url": url_for("home"), "title": "Accueil"},
                    {"title": schema_instance.section_title},
                    {"title": schema_info["title"]},
                ],
                doc_url=schema_instance.doc_url,
                schema_current_version=schema_instance.ref,
                schema_info=schema_info,
                schema_params=schema_instance.request_parameters(),
                section_title=schema_instance.section_title,
                tags=schema_instance.tags,
            )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
673
674
675

        # Process URL
        else:
676
            if not url_param:
Christophe Benz's avatar
Christophe Benz committed
677
                flash_error("Vous n'avez pas indiqué d'URL à valider")
Pierre Dittgen's avatar
Pierre Dittgen committed
678
679
680
                return redirect(
                    compute_validation_form_url(schema_instance.request_parameters())
                )
681
682
            return validate(schema_instance, URLValidataResource(url_param))

Pierre Dittgen's avatar
Pierre Dittgen committed
683
    elif request.method == "POST":
684

685
        schema_instance = SchemaInstance(request.form)
Pierre Dittgen's avatar
Pierre Dittgen committed
686

Pierre Dittgen's avatar
Pierre Dittgen committed
687
        input_param = request.form.get("input")
Pierre Dittgen's avatar
Pierre Dittgen committed
688
689
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
Pierre Dittgen's avatar
Pierre Dittgen committed
690
691
692
            return redirect(
                compute_validation_form_url(schema_instance.request_parameters())
            )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
693
694

        # File validation
Pierre Dittgen's avatar
Pierre Dittgen committed
695
696
        if input_param == "file":
            f = request.files.get("file")
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
697
698
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
Pierre Dittgen's avatar
Pierre Dittgen committed
699
700
701
702
                return redirect(
                    compute_validation_form_url(schema_instance.request_parameters())
                )
            return validate(
703
                schema_instance, FileContentValidataResource(f.filename, bytes_data(f))
Pierre Dittgen's avatar
Pierre Dittgen committed
704
            )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
705

Pierre Dittgen's avatar
Pierre Dittgen committed
706
        return "Combinaison de paramètres non supportée", 400
707
708
709

    else:
        return "Method not allowed", 405