views.py 24.1 KB
Newer Older
1
2
3
"""
    Routes
"""
4
import copy
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
5
import io
6
import json
Christophe Benz's avatar
Christophe Benz committed
7
import logging
8
from collections import Counter
9
from datetime import datetime
10
from pathlib import Path
Christophe Benz's avatar
Christophe Benz committed
11
from urllib.parse import urlencode, urljoin
12

Pierre Dittgen's avatar
Pierre Dittgen committed
13
import frictionless
14
import jsonschema
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
15
import requests
Christophe Benz's avatar
Christophe Benz committed
16
import validata_core
17
from commonmark import commonmark
18
from flask import abort, make_response, redirect, render_template, request, url_for
19
from opendataschema import GitSchemaReference, by_commit_date
Pierre Dittgen's avatar
Pierre Dittgen committed
20
21
from validata_core.helpers import (
    FileContentValidataResource,
Christophe Benz's avatar
Christophe Benz committed
22
23
    URLValidataResource,
    ValidataResource,
24
25
    is_body_error,
    is_structure_error,
Pierre Dittgen's avatar
Pierre Dittgen committed
26
)
27

Christophe Benz's avatar
Christophe Benz committed
28
from . import app, config, fetch_schema, pdf_service, schema_catalog_registry
Pierre Dittgen's avatar
Pierre Dittgen committed
29
from .model import Section
30
from .ui_util import flash_error, flash_warning
31
32
from .validata_util import strip_accents

Christophe Benz's avatar
Christophe Benz committed
33
34
log = logging.getLogger(__name__)

35

36
37
38
39
40
def get_schema_catalog(section_name):
    """Return a schema catalog associated to a section_name"""
    return schema_catalog_registry.build_schema_catalog(section_name)


41
42
43
class SchemaInstance:
    """Handy class to handle schema information"""

44
    def __init__(self, parameter_dict):
Christophe Benz's avatar
Christophe Benz committed
45
46
        """Initializes schema instance from requests dict and tableschema Catalog
        (for name ref)."""
47
48
49
50
51
52
53
54
55
        self.section_name = None
        self.section_title = None
        self.name = None
        self.url = None
        self.ref = None
        self.reference = None
        self.doc_url = None
        self.branches = None
        self.tags = None
56
57

        # From schema_url
Christophe Benz's avatar
Christophe Benz committed
58
        if parameter_dict.get("schema_url"):
59
60
            self.url = parameter_dict["schema_url"]
            self.section_title = "Autre schéma"
61
62

        # from schema_name (and schema_ref)
Pierre Dittgen's avatar
Pierre Dittgen committed
63
64
65
        elif parameter_dict.get("schema_name"):
            self.schema_and_section_name = parameter_dict["schema_name"]
            self.ref = parameter_dict.get("schema_ref")
66

Pierre Dittgen's avatar
Pierre Dittgen committed
67
            # Check schema name
Pierre Dittgen's avatar
Pierre Dittgen committed
68
            chunks = self.schema_and_section_name.split(".")
Pierre Dittgen's avatar
Pierre Dittgen committed
69
            if len(chunks) != 2:
70
                abort(400, "Paramètre 'schema_name' invalide")
Pierre Dittgen's avatar
Pierre Dittgen committed
71

72
73
            self.section_name, self.name = chunks
            self.section_title = self.find_section_title(self.section_name)
Pierre Dittgen's avatar
Pierre Dittgen committed
74
75

            # Look for schema catalog first
76
77
78
79
80
            try:
                table_schema_catalog = get_schema_catalog(self.section_name)
            except Exception as ex:
                log.exception(ex)
                abort(400, "Erreur de traitement du catalogue")
Pierre Dittgen's avatar
Pierre Dittgen committed
81
            if table_schema_catalog is None:
82
                abort(400, "Catalogue indisponible")
83
84
85

            schema_reference = table_schema_catalog.reference_by_name.get(self.name)
            if schema_reference is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
86
87
                abort(
                    400,
Christophe Benz's avatar
Christophe Benz committed
88
89
                    f"Schéma {self.name!r} non trouvé dans le catalogue de la "
                    f"section {self.section_name!r}",
Pierre Dittgen's avatar
Pierre Dittgen committed
90
                )
91
92

            if isinstance(schema_reference, GitSchemaReference):
Pierre Dittgen's avatar
Pierre Dittgen committed
93
94
95
                self.tags = sorted(
                    schema_reference.iter_tags(), key=by_commit_date, reverse=True
                )
96
                if self.ref is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
                    schema_ref = (
                        self.tags[0]
                        if self.tags
                        else schema_reference.get_default_branch()
                    )
                    abort(
                        redirect(
                            compute_validation_form_url(
                                {
                                    "schema_name": self.schema_and_section_name,
                                    "schema_ref": schema_ref.name,
                                }
                            )
                        )
                    )
112
                tag_names = [tag.name for tag in self.tags]
Pierre Dittgen's avatar
Pierre Dittgen committed
113
114
115
116
117
118
119
120
                self.branches = [
                    branch
                    for branch in schema_reference.iter_branches()
                    if branch.name not in tag_names
                ]
                self.doc_url = schema_reference.get_doc_url(
                    ref=self.ref
                ) or schema_reference.get_project_url(ref=self.ref)
121

122
            self.url = schema_reference.get_schema_url(ref=self.ref)
123
124

        else:
Pierre Dittgen's avatar
Pierre Dittgen committed
125
            flash_error("Erreur dans la récupération des informations de schéma")
Pierre Dittgen's avatar
Pierre Dittgen committed
126
            abort(redirect(url_for("home")))
127

128
        try:
Pierre Dittgen's avatar
Pierre Dittgen committed
129
            self.schema = fetch_schema(self.url)
130
131
        except json.JSONDecodeError as e:
            log.exception(e)
Pierre Dittgen's avatar
Pierre Dittgen committed
132
            flash_error("Le format du schéma n'est pas reconnu")
Pierre Dittgen's avatar
Pierre Dittgen committed
133
            abort(redirect(url_for("home")))
134
135
        except Exception as e:
            log.exception(e)
Pierre Dittgen's avatar
Pierre Dittgen committed
136
            flash_error("Impossible de récupérer le schéma")
Pierre Dittgen's avatar
Pierre Dittgen committed
137
            abort(redirect(url_for("home")))
138
139
140
141

    def request_parameters(self):
        if self.name:
            return {
Pierre Dittgen's avatar
Pierre Dittgen committed
142
143
                "schema_name": self.schema_and_section_name,
                "schema_ref": "" if self.ref is None else self.ref,
144
            }
Pierre Dittgen's avatar
Pierre Dittgen committed
145
        return {"schema_url": self.url}
146

147
    def find_section_title(self, section_name):
148
        if config.CONFIG:
Pierre Dittgen's avatar
Pierre Dittgen committed
149
150
151
            for section in config.CONFIG.homepage.sections:
                if section.name == section_name:
                    return section.title
Christophe Benz's avatar
Christophe Benz committed
152
153
        return None

154

155
156
157
def build_template_source_data(header, rows, preview_rows_nb=5):
    """Build source data information to preview in validation report page."""
    source_header_info = [(colname, False) for colname in header]
Pierre Dittgen's avatar
Pierre Dittgen committed
158

159
    rows_count = len(rows)
Pierre Dittgen's avatar
Pierre Dittgen committed
160
    preview_rows_count = min(preview_rows_nb, rows_count)
Pierre Dittgen's avatar
Pierre Dittgen committed
161
    return {
Pierre Dittgen's avatar
Pierre Dittgen committed
162
        "source_header_info": source_header_info,
163
        "header": header,
Pierre Dittgen's avatar
Pierre Dittgen committed
164
        "rows_nb": rows_count,
165
        "data_rows": rows,
Pierre Dittgen's avatar
Pierre Dittgen committed
166
        "preview_rows_count": preview_rows_count,
167
        "preview_rows": rows[:preview_rows_count],
Pierre Dittgen's avatar
Pierre Dittgen committed
168
    }
169
170


171
def build_ui_errors(errors):
Pierre Dittgen's avatar
Pierre Dittgen committed
172
    """Add context to errors, converts markdown content to HTML"""
173

Pierre Dittgen's avatar
Pierre Dittgen committed
174
175
    def improve_err(err):
        """Adds context info based on row-nb presence and converts content to HTML"""
176

Pierre Dittgen's avatar
Pierre Dittgen committed
177
178
        # Context
        update_keys = {
Pierre Dittgen's avatar
Pierre Dittgen committed
179
            "context": "body"
180
            if "row-number" in err and err["row-number"] is not None
Pierre Dittgen's avatar
Pierre Dittgen committed
181
            else "table",
Pierre Dittgen's avatar
Pierre Dittgen committed
182
        }
Pierre Dittgen's avatar
Pierre Dittgen committed
183

184
        # Set title
185
        if "title" not in err:
186
            update_keys["title"] = err["name"]
187

188
        # Set content
189
        content = "*content soon available*"
190
        if "message" in err:
191
            content = err["message"]
192
193
        elif "description" in err:
            content = err["description"]
194
        update_keys["content"] = commonmark(content)
195

Pierre Dittgen's avatar
Pierre Dittgen committed
196
        return {**err, **update_keys}
197

Pierre Dittgen's avatar
Pierre Dittgen committed
198
    return list(map(improve_err, errors))
199
200


Pierre Dittgen's avatar
Pierre Dittgen committed
201
def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict):
202
203
204
205
206
207
208
    """Creates an error report easier to handle and display in templates:
    - only one table
    - errors are contextualized
    - error-counts is ok
    - errors are grouped by lines
    - errors are separated into "structure" and "body"
    - error messages are improved
209
    """
210
    v_report = copy.deepcopy(validata_core_report)
211

212
213
214
    # Create a new UI report from information picked in validata report
    ui_report = {}
    ui_report["table"] = {}
215

216
    # source headers
Pierre Dittgen's avatar
Pierre Dittgen committed
217
    headers = v_report.table["header"]
218
    ui_report["table"]["header"] = headers
Pierre Dittgen's avatar
Pierre Dittgen committed
219

220
    # source dimension
Pierre Dittgen's avatar
Pierre Dittgen committed
221
    ui_report["table"]["col_count"] = len(headers)
Pierre Dittgen's avatar
Pierre Dittgen committed
222
    ui_report["table"]["row_count"] = rows_count
223
224

    # Computes column info from schema
Pierre Dittgen's avatar
Pierre Dittgen committed
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
    fields_dict = {
        f["name"]: (f.get("title", f["name"]), f.get("description", ""))
        for f in schema_dict.get("fields", [])
    }
    ui_report["table"]["headers_title"] = [
        fields_dict[h][0] if h in fields_dict else "Colonne inconnue" for h in headers
    ]
    ui_report["table"]["headers_description"] = [
        fields_dict[h][1]
        if h in fields_dict
        else "Cette colonne n'est pas définie dans le schema"
        for h in headers
    ]
    missing_headers = [
        err["message-data"]["column-name"]
        for err in v_report.table["errors"]
        if err["code"] == "missing-header"
    ]
    ui_report["table"]["cols_alert"] = [
        "table-danger" if h not in fields_dict or h in missing_headers else ""
        for h in headers
    ]
247

248
    # prepare error structure for UI needs
Pierre Dittgen's avatar
Pierre Dittgen committed
249
    errors = build_ui_errors(v_report.table["errors"])
250

Pierre Dittgen's avatar
Pierre Dittgen committed
251
    # Count errors and warnings
Pierre Dittgen's avatar
Pierre Dittgen committed
252
    ui_report["error_count"] = len(errors)
Pierre Dittgen's avatar
Pierre Dittgen committed
253
254
    ui_report["warn_count"] = len(v_report.table["structure_warnings"])
    ui_report["warnings"] = v_report.table["structure_warnings"]
255
256

    # Then group them in 2 groups : structure and body
Pierre Dittgen's avatar
Pierre Dittgen committed
257
    ui_report["table"]["errors"] = {"structure": [], "body": []}
258
    for err in errors:
259
        if is_structure_error(err):
Pierre Dittgen's avatar
Pierre Dittgen committed
260
            ui_report["table"]["errors"]["structure"].append(err)
261
        elif is_body_error(err):
Pierre Dittgen's avatar
Pierre Dittgen committed
262
            ui_report["table"]["errors"]["body"].append(err)
263

Pierre Dittgen's avatar
Pierre Dittgen committed
264
265
266
    # Group body errors by row id
    rows = []
    current_row_id = 0
Pierre Dittgen's avatar
Pierre Dittgen committed
267
    for err in ui_report["table"]["errors"]["body"]:
268
        if "rowPosition" not in err:
269
            continue
Pierre Dittgen's avatar
Pierre Dittgen committed
270
        row_id = err["rowPosition"]
Pierre Dittgen's avatar
Pierre Dittgen committed
271
272
        if row_id != current_row_id:
            current_row_id = row_id
Pierre Dittgen's avatar
Pierre Dittgen committed
273
            rows.append({"row_id": current_row_id, "errors": {}})
Pierre Dittgen's avatar
Pierre Dittgen committed
274

Pierre Dittgen's avatar
Pierre Dittgen committed
275
        column_id = err.get("fieldPosition")
Pierre Dittgen's avatar
Pierre Dittgen committed
276
        if column_id is not None:
Pierre Dittgen's avatar
Pierre Dittgen committed
277
            rows[-1]["errors"][column_id] = err
Pierre Dittgen's avatar
Pierre Dittgen committed
278
        else:
Pierre Dittgen's avatar
Pierre Dittgen committed
279
280
            rows[-1]["errors"]["row"] = err
    ui_report["table"]["errors"]["body_by_rows"] = rows
Pierre Dittgen's avatar
Pierre Dittgen committed
281

282
    # Sort by error names in statistics
283
    ui_report["table"]["count-by-code"] = {}
284
285
    stats = {}
    total_errors_count = 0
Pierre Dittgen's avatar
Pierre Dittgen committed
286
    for key in ("structure", "body"):
287
288
        # convert dict into tuples with french title instead of error code
        # and sorts by title
289
        key_errors = ui_report["table"]["errors"][key]
290
        key_errors_count = len(key_errors)
291
        ct = Counter(ke["name"] for ke in key_errors)
292
293
294

        error_stats = {
            "count": key_errors_count,
Pierre Dittgen's avatar
Pierre Dittgen committed
295
            "count-by-code": sorted((k, v) for k, v in ct.items()),
296
297
298
299
        }
        total_errors_count += key_errors_count

        # Add error rows count
Pierre Dittgen's avatar
Pierre Dittgen committed
300
301
302
303
        if key == "body":
            error_rows = {
                err["rowPosition"] for err in key_errors if "rowPosition" in err
            }
304
305
306
307
308
            error_stats["rows-count"] = len(error_rows)

        stats[f"{key}-errors"] = error_stats

    stats["count"] = total_errors_count
309
    ui_report["table"]["error-stats"] = stats
310

311
    return ui_report
Pierre Dittgen's avatar
Pierre Dittgen committed
312
313


Pierre Dittgen's avatar
Pierre Dittgen committed
314
315
def compute_badge_message_and_color(badge):
    """Computes message and color from badge information"""
Pierre Dittgen's avatar
Pierre Dittgen committed
316
317
    structure = badge["structure"]
    body = badge.get("body")
Pierre Dittgen's avatar
Pierre Dittgen committed
318
319

    # Bad structure, stop here
Pierre Dittgen's avatar
Pierre Dittgen committed
320
321
    if structure == "KO":
        return ("structure invalide", "red")
Pierre Dittgen's avatar
Pierre Dittgen committed
322
323

    # No body error
Pierre Dittgen's avatar
Pierre Dittgen committed
324
325
    if body == "OK":
        return (
Pierre Dittgen's avatar
Pierre Dittgen committed
326
            ("partiellement valide", "yellowgreen")
Pierre Dittgen's avatar
Pierre Dittgen committed
327
328
329
            if structure == "WARN"
            else ("valide", "green")
        )
Pierre Dittgen's avatar
Pierre Dittgen committed
330
331

    # else compute quality ratio percent
Pierre Dittgen's avatar
Pierre Dittgen committed
332
333
334
    p = (1 - badge["error-ratio"]) * 100.0
    msg = "cellules valides : {:.1f}%".format(p)
    return (msg, "red") if body == "KO" else (msg, "orange")
Pierre Dittgen's avatar
Pierre Dittgen committed
335
336
337
338
339
340


def get_badge_url_and_message(badge):
    """Gets badge url from badge information"""

    msg, color = compute_badge_message_and_color(badge)
Christophe Benz's avatar
Christophe Benz committed
341
    badge_url = "{}?{}".format(
Pierre Dittgen's avatar
Pierre Dittgen committed
342
343
        urljoin(config.SHIELDS_IO_BASE_URL, "/static/v1.svg"),
        urlencode({"label": "Validata", "message": msg, "color": color}),
Christophe Benz's avatar
Christophe Benz committed
344
345
    )
    return (badge_url, msg)
Pierre Dittgen's avatar
Pierre Dittgen committed
346
347


348
def validate(schema_instance: SchemaInstance, validata_resource: ValidataResource):
349
350
    """ Validate source and display report """

351
352
353
354
355
356
357
358
    def compute_resource_info(resource: ValidataResource):
        source = resource.get_source()
        return {
            "type": "url" if source.startswith("http") else "file",
            "url": source,
            "filename": Path(source).name,
        }

Pierre Dittgen's avatar
Pierre Dittgen committed
359
    # Parse source data once
360
    header, rows = validata_resource.extract_tabular_data()
Pierre Dittgen's avatar
Pierre Dittgen committed
361
    rows_count = len(rows)
Pierre Dittgen's avatar
Pierre Dittgen committed
362
363

    # Call validata_core with parsed data
Pierre Dittgen's avatar
Pierre Dittgen committed
364
    validata_core_report = validata_core.validate(
365
        [header] + rows, schema_instance.schema
Pierre Dittgen's avatar
Pierre Dittgen committed
366
    )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
367

368
    # disable badge
Pierre Dittgen's avatar
Pierre Dittgen committed
369
    badge_config = config.BADGE_CONFIG
370

Pierre Dittgen's avatar
Pierre Dittgen committed
371
    # Computes badge from report and badge configuration
Pierre Dittgen's avatar
Pierre Dittgen committed
372
    badge_url, badge_msg = None, None
Pierre Dittgen's avatar
Pierre Dittgen committed
373
    display_badge = badge_config and config.SHIELDS_IO_BASE_URL
Pierre Dittgen's avatar
Pierre Dittgen committed
374
    if display_badge:
Pierre Dittgen's avatar
Pierre Dittgen committed
375
376
377
        badge_stats = validata_core.compute_badge_metrics(
            validata_core_report, badge_config
        )
378
379
380
381
382
383
384
385
386
387
388
        if badge_stats:
            badge_url, badge_msg = get_badge_url_and_message(badge_stats)

    # Non table errors
    if validata_core_report["errors"]:
        non_table_error = validata_core_report["errors"][0]
        msg = non_table_error["note"]
        flash_error(f"Une erreur est survenue empêchant la validation : {msg}")
        return redirect(
            compute_validation_form_url(schema_instance.request_parameters())
        )
Pierre Dittgen's avatar
Pierre Dittgen committed
389

390
    # Source error
391
392
    source_errors = [
        err
Pierre Dittgen's avatar
Pierre Dittgen committed
393
394
        for err in validata_core_report["tables"][0]["errors"]
        if err["code"] in {"source-error", "unknown-csv-dialect"}
395
    ]
396
397
    if source_errors:
        err = source_errors[0]
Pierre Dittgen's avatar
Pierre Dittgen committed
398
399
400
401
402
403
404
        msg = (
            "l'encodage du fichier est invalide. Veuillez le corriger"
            if "charmap" in err["message"]
            else err["message"]
        )
        flash_error("Erreur de source : {}".format(msg))
        return redirect(url_for("custom_validator"))
405

Pierre Dittgen's avatar
Pierre Dittgen committed
406
    # handle report date
Pierre Dittgen's avatar
Pierre Dittgen committed
407
    report_datetime = datetime.fromisoformat(validata_core_report["date"]).astimezone()
408

409
    # create ui_report
Pierre Dittgen's avatar
Pierre Dittgen committed
410
411
412
    ui_report = create_validata_ui_report(
        rows_count, validata_core_report, schema_instance.schema
    )
Pierre Dittgen's avatar
Pierre Dittgen committed
413

Pierre Dittgen's avatar
Pierre Dittgen committed
414
    # Display report to the user
Pierre Dittgen's avatar
Pierre Dittgen committed
415
416
417
    validator_form_url = compute_validation_form_url(
        schema_instance.request_parameters()
    )
Christophe Benz's avatar
Christophe Benz committed
418
    schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
419
420

    # Build PDF report URL
421
422
423
    # PDF report is available if:
    # - a pdf_service has been configured
    # - tabular resource to validate is defined as an URL
424
    pdf_report_url = None
425
    if pdf_service and isinstance(validata_resource, URLValidataResource):
426
427
        base_url = url_for("pdf_report")
        query_string = urlencode(
428
429
430
431
            {
                **schema_instance.request_parameters(),
                "url": validata_resource.url,
            }
Pierre Dittgen's avatar
Pierre Dittgen committed
432
        )
433
        pdf_report_url = f"{base_url}?{query_string}"
Pierre Dittgen's avatar
Pierre Dittgen committed
434
435
436

    return render_template(
        "validation_report.html",
437
        config=config.CONFIG,
Pierre Dittgen's avatar
Pierre Dittgen committed
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
        badge_msg=badge_msg,
        badge_url=badge_url,
        breadcrumbs=[
            {"title": "Accueil", "url": url_for("home")},
            {"title": schema_instance.section_title},
            {"title": schema_info["title"], "url": validator_form_url},
            {"title": "Rapport de validation"},
        ],
        display_badge=display_badge,
        doc_url=schema_instance.doc_url,
        pdf_report_url=pdf_report_url,
        print_mode=request.args.get("print", "false") == "true",
        report=ui_report,
        schema_current_version=schema_instance.ref,
        schema_info=schema_info,
        section_title=schema_instance.section_title,
454
        source_data=build_template_source_data(header, rows),
455
        resource=compute_resource_info(validata_resource),
Pierre Dittgen's avatar
Pierre Dittgen committed
456
457
        validation_date=report_datetime.strftime("le %d/%m/%Y à %Hh%M"),
    )
458
459


460
461
def bytes_data(f):
    """ Gets bytes data from Werkzeug FileStorage instance """
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
462
    iob = io.BytesIO()
463
464
465
466
467
    f.save(iob)
    iob.seek(0)
    return iob.getvalue()


Pierre Dittgen's avatar
Pierre Dittgen committed
468
def retrieve_schema_catalog(section: Section):
Pierre Dittgen's avatar
Pierre Dittgen committed
469
    """Retrieve schema catalog and return formatted error if it fails."""
470
471
472
473

    def format_error_message(err_message, exc):
        """Prepare a bootstrap error message with details if wanted."""

Pierre Dittgen's avatar
Pierre Dittgen committed
474
        exception_text = "\n".join([str(arg) for arg in exc.args])
475

Pierre Dittgen's avatar
Pierre Dittgen committed
476
        return f"""{err_msg}
477
        <div class="float-right">
Christophe Benz's avatar
Christophe Benz committed
478
479
            <button type="button" class="btn btn-info btn-xs" data-toggle="collapse"
                data-target="#exception_info">détails</button>
480
481
482
483
484
485
486
        </div>
        <div id="exception_info" class="collapse">
                <pre>{exception_text}</pre>
        </div>
"""

    try:
Pierre Dittgen's avatar
Pierre Dittgen committed
487
        schema_catalog = get_schema_catalog(section.name)
488
        return (schema_catalog, None)
Pierre Dittgen's avatar
Pierre Dittgen committed
489

490
491
492
493
494
495
496
497
498
    except Exception as exc:
        log.exception(exc)
        err_msg = "une erreur s'est produite"
        if isinstance(exc, requests.ConnectionError):
            err_msg = "problème de connexion"
        elif isinstance(exc, json.decoder.JSONDecodeError):
            err_msg = "format JSON incorrect"
        elif isinstance(exc, jsonschema.exceptions.ValidationError):
            err_msg = "le catalogue ne respecte pas le schéma de référence"
Pierre Dittgen's avatar
Pierre Dittgen committed
499

500
        error_catalog = {
Pierre Dittgen's avatar
Pierre Dittgen committed
501
            **{k: v for k, v in section.dict().items() if k != "catalog"},
Pierre Dittgen's avatar
Pierre Dittgen committed
502
            "err": format_error_message(err_msg, exc),
503
504
505
506
        }
        return None, error_catalog


507
508
509
# Routes


Pierre Dittgen's avatar
Pierre Dittgen committed
510
@app.route("/")
511
512
def home():
    """ Home page """
513
514
515

    def iter_sections():
        """Yield sections of the home page, filled with schema metadata."""
Pierre Dittgen's avatar
Pierre Dittgen committed
516
517

        # Iterate on all sections
Pierre Dittgen's avatar
Pierre Dittgen committed
518
        for section in config.CONFIG.homepage.sections:
Pierre Dittgen's avatar
Pierre Dittgen committed
519

Pierre Dittgen's avatar
Pierre Dittgen committed
520
            # section with only links to external validators
Pierre Dittgen's avatar
Pierre Dittgen committed
521
            if section.links:
Pierre Dittgen's avatar
Pierre Dittgen committed
522
                yield section
523
                continue
524

Pierre Dittgen's avatar
Pierre Dittgen committed
525
            # section with catalog
Pierre Dittgen's avatar
Pierre Dittgen committed
526
            if section.catalog is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
527
528
529
                # skip section
                continue

530
531
532
533
534
            # retrieving schema catatalog
            schema_catalog, catalog_error = retrieve_schema_catalog(section)
            if schema_catalog is None:
                yield catalog_error
                continue
Pierre Dittgen's avatar
Pierre Dittgen committed
535
536
537
538

            # Working on catalog
            schema_info_list = []
            for schema_reference in schema_catalog.references:
539
540
541
                # retain tableschema only
                if schema_reference.get_schema_type() != "tableschema":
                    continue
Pierre Dittgen's avatar
Pierre Dittgen committed
542
                # Loads default table schema for each schema reference
Pierre Dittgen's avatar
Pierre Dittgen committed
543
                schema_info = {"name": schema_reference.name}
544
                try:
Pierre Dittgen's avatar
Pierre Dittgen committed
545
                    table_schema = fetch_schema(schema_reference.get_schema_url())
Pierre Dittgen's avatar
Pierre Dittgen committed
546
                except json.JSONDecodeError:
Pierre Dittgen's avatar
Pierre Dittgen committed
547
                    schema_info["err"] = True
Christophe Benz's avatar
Christophe Benz committed
548
549
550
551
                    schema_info["title"] = (
                        f"le format du schéma « {schema_info['name']} » "
                        "n'est pas reconnu"
                    )
552
                except Exception:
Pierre Dittgen's avatar
Pierre Dittgen committed
553
                    schema_info["err"] = True
Christophe Benz's avatar
Christophe Benz committed
554
555
556
                    schema_info["title"] = (
                        f"le schéma « {schema_info['name']} » " "n'est pas disponible"
                    )
557
                else:
Pierre Dittgen's avatar
Pierre Dittgen committed
558
559
560
                    schema_info["title"] = (
                        table_schema.get("title") or schema_info["name"]
                    )
Pierre Dittgen's avatar
Pierre Dittgen committed
561
562
                schema_info_list.append(schema_info)
            schema_info_list = sorted(
Pierre Dittgen's avatar
Pierre Dittgen committed
563
564
                schema_info_list, key=lambda sc: strip_accents(sc["title"].lower())
            )
Pierre Dittgen's avatar
Pierre Dittgen committed
565

Pierre Dittgen's avatar
Pierre Dittgen committed
566
            yield {
Pierre Dittgen's avatar
Pierre Dittgen committed
567
                **{k: v for k, v in section.dict().items() if k != "catalog"},
Pierre Dittgen's avatar
Pierre Dittgen committed
568
569
                "catalog": schema_info_list,
            }
570

Pierre Dittgen's avatar
Pierre Dittgen committed
571
572
573
    return render_template(
        "home.html", config=config.CONFIG, sections=list(iter_sections())
    )
574
575


Pierre Dittgen's avatar
Pierre Dittgen committed
576
@app.route("/pdf")
Pierre Dittgen's avatar
Pierre Dittgen committed
577
def pdf_report():
578
    """PDF report generation"""
Pierre Dittgen's avatar
Pierre Dittgen committed
579
    err_prefix = "Erreur de génération du rapport PDF"
580

Pierre Dittgen's avatar
Pierre Dittgen committed
581
    url_param = request.args.get("url")
582
    if not url_param:
Pierre Dittgen's avatar
Pierre Dittgen committed
583
584
        flash_error(err_prefix + " : URL non fournie")
        return redirect(url_for("home"))
585

586
587
588
589
590
    if pdf_service is None:
        flash_error(err_prefix + " : service de génération non configuré")
        return redirect(url_for("home"))

    # Compute validation report URL
591
    schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
592

Pierre Dittgen's avatar
Pierre Dittgen committed
593
    base_url = url_for("custom_validator", _external=True)
594
    parameter_dict = {
Pierre Dittgen's avatar
Pierre Dittgen committed
595
596
597
598
        "input": "url",
        "print": "true",
        "url": url_param,
        **schema_instance.request_parameters(),
599
    }
Christophe Benz's avatar
Christophe Benz committed
600
    validation_url = "{}?{}".format(base_url, urlencode(parameter_dict))
Pierre Dittgen's avatar
Pierre Dittgen committed
601
    log.info("Validation URL = %s", validation_url)
602

603
604
605
    # Ask for PDF report generation
    try:
        pdf_bytes_content = pdf_service.render(validation_url)
Christophe Benz's avatar
Christophe Benz committed
606
    except Exception:
607
608
609
        log.exception(err_prefix)
        flash_error(err_prefix + " : contactez votre administrateur")
        return redirect(url_for("home"))
610

611
612
613
614
    # Compute pdf filename
    pdf_filename = "Rapport de validation {}.pdf".format(
        datetime.now().strftime("%d-%m-%Y %Hh%M")
    )
615

616
617
618
619
620
621
    # Prepare and send response
    response = make_response(pdf_bytes_content)
    response.headers.set("Content-Disposition", "attachment", filename=pdf_filename)
    response.headers.set("Content-Length", len(pdf_bytes_content))
    response.headers.set("Content-Type", "application/pdf")
    return response
622
623


Pierre Dittgen's avatar
Pierre Dittgen committed
624
def extract_schema_metadata(table_schema: frictionless.Schema):
Pierre Dittgen's avatar
Pierre Dittgen committed
625
    """Gets author, contibutor, version...metadata from schema header"""
Pierre Dittgen's avatar
Pierre Dittgen committed
626
    return {k: v for k, v in table_schema.items() if k != "fields"}
Pierre Dittgen's avatar
Pierre Dittgen committed
627
628


Pierre Dittgen's avatar
Pierre Dittgen committed
629
def compute_schema_info(table_schema: frictionless.Schema, schema_url):
Pierre Dittgen's avatar
Pierre Dittgen committed
630
    """Factor code for validator form page"""
631

Pierre Dittgen's avatar
Pierre Dittgen committed
632
633
    # Schema URL + schema metadata info
    schema_info = {
Pierre Dittgen's avatar
Pierre Dittgen committed
634
        "path": schema_url,
Christophe Benz's avatar
Christophe Benz committed
635
636
        # a "path" metadata property can be found in Table Schema,
        # and we'd like it to override the `schema_url`
Christophe Benz's avatar
Christophe Benz committed
637
        # given by the user (in case schema was given by URL)
Pierre Dittgen's avatar
Pierre Dittgen committed
638
        **extract_schema_metadata(table_schema),
Pierre Dittgen's avatar
Pierre Dittgen committed
639
    }
Christophe Benz's avatar
Christophe Benz committed
640
    return schema_info
641
642


643
def compute_validation_form_url(request_parameters: dict):
644
    """Computes validation form url with schema URL parameter"""
Pierre Dittgen's avatar
Pierre Dittgen committed
645
    url = url_for("custom_validator")
Christophe Benz's avatar
Christophe Benz committed
646
    return "{}?{}".format(url, urlencode(request_parameters))
Pierre Dittgen's avatar
Pierre Dittgen committed
647
648


Pierre Dittgen's avatar
Pierre Dittgen committed
649
@app.route("/table-schema", methods=["GET", "POST"])
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
650
def custom_validator():
651
    """Validator form"""
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
652

Pierre Dittgen's avatar
Pierre Dittgen committed
653
    if request.method == "GET":
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
654

655
656
        # input is a hidden form parameter to know
        # if this is the initial page display or if the validation has been asked for
Pierre Dittgen's avatar
Pierre Dittgen committed
657
        input_param = request.args.get("input")
658
659

        # url of resource to be validated
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
660
661
        url_param = request.args.get("url")

662
        schema_instance = SchemaInstance(request.args)
Pierre Dittgen's avatar
Pierre Dittgen committed
663

Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
664
665
        # First form display
        if input_param is None:
Pierre Dittgen's avatar
Pierre Dittgen committed
666
667
668
669
670
            schema_info = compute_schema_info(
                schema_instance.schema, schema_instance.url
            )
            return render_template(
                "validation_form.html",
671
                config=config.CONFIG,
Pierre Dittgen's avatar
Pierre Dittgen committed
672
673
674
675
676
677
678
679
680
681
682
683
684
                branches=schema_instance.branches,
                breadcrumbs=[
                    {"url": url_for("home"), "title": "Accueil"},
                    {"title": schema_instance.section_title},
                    {"title": schema_info["title"]},
                ],
                doc_url=schema_instance.doc_url,
                schema_current_version=schema_instance.ref,
                schema_info=schema_info,
                schema_params=schema_instance.request_parameters(),
                section_title=schema_instance.section_title,
                tags=schema_instance.tags,
            )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
685
686
687

        # Process URL
        else:
688
            if not url_param:
Christophe Benz's avatar
Christophe Benz committed
689
                flash_error("Vous n'avez pas indiqué d'URL à valider")
Pierre Dittgen's avatar
Pierre Dittgen committed
690
691
692
                return redirect(
                    compute_validation_form_url(schema_instance.request_parameters())
                )
693
694
            return validate(schema_instance, URLValidataResource(url_param))

Pierre Dittgen's avatar
Pierre Dittgen committed
695
    elif request.method == "POST":
696

697
        schema_instance = SchemaInstance(request.form)
Pierre Dittgen's avatar
Pierre Dittgen committed
698

Pierre Dittgen's avatar
Pierre Dittgen committed
699
        input_param = request.form.get("input")
Pierre Dittgen's avatar
Pierre Dittgen committed
700
701
        if input_param is None:
            flash_error("Vous n'avez pas indiqué de fichier à valider")
Pierre Dittgen's avatar
Pierre Dittgen committed
702
703
704
            return redirect(
                compute_validation_form_url(schema_instance.request_parameters())
            )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
705
706

        # File validation
Pierre Dittgen's avatar
Pierre Dittgen committed
707
708
        if input_param == "file":
            f = request.files.get("file")
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
709
710
            if f is None:
                flash_warning("Vous n'avez pas indiqué de fichier à valider")
Pierre Dittgen's avatar
Pierre Dittgen committed
711
712
713
714
                return redirect(
                    compute_validation_form_url(schema_instance.request_parameters())
                )
            return validate(
715
                schema_instance, FileContentValidataResource(f.filename, bytes_data(f))
Pierre Dittgen's avatar
Pierre Dittgen committed
716
            )
Pierre Dittgen's avatar
wip    
Pierre Dittgen committed
717

Pierre Dittgen's avatar
Pierre Dittgen committed
718
        return "Combinaison de paramètres non supportée", 400
719
720
721

    else:
        return "Method not allowed", 405