Commit 68c26076 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

merge towards_frictionless_4

parents f17cc796 4695caa7
...@@ -2,6 +2,26 @@ ...@@ -2,6 +2,26 @@
## next ## next
## 0.5.0a4 (`towards_frictionless_4` branch)
- fix flake8 issues
- update requirements (validata-core 0.7.0a7)
- rephrase presentation in footer
## 0.5.0a3 (`towards_frictionless_4` branch)
- update requirements (frictionless 4.1.0)
## 0.5.0a2 (`towards_frictionless_4` branch)
- update requirements (validata-core 0.7.0a2)
- add explicit error message when a tabular data as url can't be retrieved (404)
- improve error message ([issue](https://git.opendatafrance.net/validata/validata-ui/-/issues/82)]
## 0.5.0a1 (`towards_frictionless_4` branch)
- update requirements
- adapt to changes in validation report
## 0.4.1 ## 0.4.1
- add explicit error message when a tabular data as an URL can't be retrieved (404) - add explicit error message when a tabular data as an URL can't be retrieved (404)
......
...@@ -23,7 +23,7 @@ commonmark==0.9.1 ...@@ -23,7 +23,7 @@ commonmark==0.9.1
# via validata_ui (setup.py) # via validata_ui (setup.py)
decorator==4.4.2 decorator==4.4.2
# via validators # via validators
deprecated==1.2.11 deprecated==1.2.12
# via pygithub # via pygithub
et-xmlfile==1.0.1 et-xmlfile==1.0.1
# via openpyxl # via openpyxl
...@@ -33,27 +33,25 @@ ezodf==0.3.2 ...@@ -33,27 +33,25 @@ ezodf==0.3.2
# validata_ui (setup.py) # validata_ui (setup.py)
flask==1.1.2 flask==1.1.2
# via validata_ui (setup.py) # via validata_ui (setup.py)
frictionless==3.48.0 frictionless==4.2.1
# via # via
# validata-core # validata-core
# validata_ui (setup.py) # validata_ui (setup.py)
idna==2.10 idna==2.10
# via requests # via requests
importlib-resources==5.1.0 importlib-resources==5.1.2
# via validata-core # via validata-core
isodate==0.6.0 isodate==0.6.0
# via frictionless # via frictionless
itsdangerous==1.1.0 itsdangerous==1.1.0
# via flask # via flask
jdcal==1.4.1
# via openpyxl
jinja2==2.11.3 jinja2==2.11.3
# via flask # via flask
jsonschema==3.2.0 jsonschema==3.2.0
# via # via
# frictionless # frictionless
# opendataschema # opendataschema
lxml==4.6.2 lxml==4.6.3
# via # via
# validata-core # validata-core
# validata_ui (setup.py) # validata_ui (setup.py)
...@@ -63,13 +61,13 @@ numpy==1.20.1 ...@@ -63,13 +61,13 @@ numpy==1.20.1
# via pandas # via pandas
opendataschema==0.6.0 opendataschema==0.6.0
# via validata_ui (setup.py) # via validata_ui (setup.py)
openpyxl==3.0.6 openpyxl==3.0.7
# via validata-core # via validata-core
pandas==1.2.1 pandas==1.2.3
# via tablib # via tablib
petl==1.7.2 petl==1.7.2
# via frictionless # via frictionless
pydantic==1.7.3 pydantic==1.8.1
# via validata_ui (setup.py) # via validata_ui (setup.py)
pygithub==1.54.1 pygithub==1.54.1
# via opendataschema # via opendataschema
...@@ -109,6 +107,8 @@ requests==2.25.1 ...@@ -109,6 +107,8 @@ requests==2.25.1
# requests-toolbelt # requests-toolbelt
# validata-core # validata-core
# validata_ui (setup.py) # validata_ui (setup.py)
rfc3986==1.4.0
# via frictionless
shellingham==1.4.0 shellingham==1.4.0
# via typer # via typer
simpleeval==0.9.10 simpleeval==0.9.10
...@@ -131,9 +131,11 @@ toolz==0.11.1 ...@@ -131,9 +131,11 @@ toolz==0.11.1
# via validata-core # via validata-core
typer[all]==0.3.2 typer[all]==0.3.2
# via frictionless # via frictionless
urllib3==1.26.3 typing-extensions==3.7.4.3
# via pydantic
urllib3==1.26.4
# via requests # via requests
validata-core==0.6.0 validata-core==0.7.0a7
# via validata_ui (setup.py) # via validata_ui (setup.py)
validators==0.18.2 validators==0.18.2
# via frictionless # via frictionless
......
...@@ -13,7 +13,7 @@ with readme_filepath.open("rt", encoding="utf-8") as fd: ...@@ -13,7 +13,7 @@ with readme_filepath.open("rt", encoding="utf-8") as fd:
setup( setup(
name="validata_ui", name="validata_ui",
version="0.4.1", version="0.5.0a4",
description="Validata Web UI", description="Validata Web UI",
long_description=LONG_DESCRIPTION, long_description=LONG_DESCRIPTION,
long_description_content_type="text/markdown", long_description_content_type="text/markdown",
......
"""Validata UI."""
import logging import logging
from datetime import timedelta from datetime import timedelta
......
...@@ -87,7 +87,7 @@ BROWSERLESS_API_TOKEN = os.getenv("BROWSERLESS_API_TOKEN") or None ...@@ -87,7 +87,7 @@ BROWSERLESS_API_TOKEN = os.getenv("BROWSERLESS_API_TOKEN") or None
# Cache backend (default is SQLite) # Cache backend (default is SQLite)
CACHE_BACKEND = os.getenv("CACHE_BACKEND") or "sqlite" CACHE_BACKEND = os.getenv("CACHE_BACKEND") or "sqlite"
log.info(f"Cache backend: {CACHE_BACKEND}") log.info("Cache backend: %r", CACHE_BACKEND)
# Caching time for schema requests in minutes # Caching time for schema requests in minutes
CACHE_EXPIRE_AFTER = os.getenv("CACHE_EXPIRE_AFTER") or None CACHE_EXPIRE_AFTER = os.getenv("CACHE_EXPIRE_AFTER") or None
......
"""Config model using pydantic""" """Config model using pydantic."""
from typing import List, Optional, Union from typing import List, Optional, Union
from pydantic import BaseModel, HttpUrl, root_validator from pydantic import BaseModel, HttpUrl, root_validator
class Link(BaseModel): class Link(BaseModel):
"""Link class."""
title: str title: str
url: HttpUrl url: HttpUrl
class ExternalLink(BaseModel): class ExternalLink(BaseModel):
"""ExternalLink class."""
name: str name: str
type: str type: str
title: str title: str
...@@ -18,16 +22,22 @@ class ExternalLink(BaseModel): ...@@ -18,16 +22,22 @@ class ExternalLink(BaseModel):
class Schema(BaseModel): class Schema(BaseModel):
"""Schema class."""
name: str name: str
repo_url: HttpUrl repo_url: HttpUrl
class Catalog(BaseModel): class Catalog(BaseModel):
"""Catalog class."""
version: int version: int
schemas: List[Schema] schemas: List[Schema]
class Section(BaseModel): class Section(BaseModel):
"""Section class."""
name: str name: str
title: str title: str
description: Optional[str] = None description: Optional[str] = None
...@@ -35,7 +45,8 @@ class Section(BaseModel): ...@@ -35,7 +45,8 @@ class Section(BaseModel):
links: Optional[List[ExternalLink]] = None links: Optional[List[ExternalLink]] = None
@root_validator @root_validator
def check_catalog_or_links(cls, values): def check_catalog_or_links(cls, values): # noqa
"""Check that catalog or links attributes is defined but not both."""
catalog, links = values.get("catalog"), values.get("links") catalog, links = values.get("catalog"), values.get("links")
if catalog is None and links is None: if catalog is None and links is None:
raise ValueError("catalog or links field must be defined") raise ValueError("catalog or links field must be defined")
...@@ -45,18 +56,26 @@ class Section(BaseModel): ...@@ -45,18 +56,26 @@ class Section(BaseModel):
class Footer(BaseModel): class Footer(BaseModel):
"""Footer section."""
links: List[Link] links: List[Link]
class Header(BaseModel): class Header(BaseModel):
"""Header section."""
links: List[Link] links: List[Link]
class Homepage(BaseModel): class Homepage(BaseModel):
"""Homepage section."""
sections: List[Section] sections: List[Section]
class Config(BaseModel): class Config(BaseModel):
"""Config class defines header, footer and homepage sections."""
footer: Footer footer: Footer
header: Header header: Header
homepage: Homepage homepage: Homepage
...@@ -38,11 +38,13 @@ class BrowserlessPDFRenderer(PDFRenderer): ...@@ -38,11 +38,13 @@ class BrowserlessPDFRenderer(PDFRenderer):
"""Browserless IO implementation.""" """Browserless IO implementation."""
def __init__(self, api_url: str, api_token: str): def __init__(self, api_url: str, api_token: str):
"""Create browserless.io implementation."""
log.info("BrowserlessPDFRenderer: creating instance with api_url = %r", api_url) log.info("BrowserlessPDFRenderer: creating instance with api_url = %r", api_url)
self.api_url = api_url self.api_url = api_url
self.api_token = api_token self.api_token = api_token
def render(self, url: str) -> bytes: def render(self, url: str) -> bytes:
"""Call browserless.io service to generate PDF."""
headers = { headers = {
"Cache-Control": "no-cache", "Cache-Control": "no-cache",
"Content-Type": "application/json", "Content-Type": "application/json",
...@@ -82,6 +84,7 @@ class ChromiumHeadlessPDFRenderer(PDFRenderer): ...@@ -82,6 +84,7 @@ class ChromiumHeadlessPDFRenderer(PDFRenderer):
"""Chromium implementation.""" """Chromium implementation."""
def render(self, url: str) -> bytes: def render(self, url: str) -> bytes:
"""Render PDF document using Chromium headless."""
# Create temp file to save validation report # Create temp file to save validation report
# This temp file will be automatically deleted on context exit # This temp file will be automatically deleted on context exit
with tempfile.NamedTemporaryFile( with tempfile.NamedTemporaryFile(
......
...@@ -66,7 +66,7 @@ ...@@ -66,7 +66,7 @@
<footer class="footer hidden-print"> <footer class="footer hidden-print">
<p> <p>
Le <a href="https://www.validata.fr/">projet Validata</a> est <a href="https://www.validata.fr/">Validata</a> est
une initiative d'<a href="http://www.opendatafrance.net/">OpenDataFrance</a> une initiative d'<a href="http://www.opendatafrance.net/">OpenDataFrance</a>
développée par <a href="https://jailbreak.paris">Jailbreak</a>. développée par <a href="https://jailbreak.paris">Jailbreak</a>.
</p> </p>
......
""" """Util UI functions."""
Util UI functions
"""
from flask import flash from flask import flash
def flash_error(msg): def flash_error(msg):
""" Flash bootstrap error message """ """Flash bootstrap error message."""
flash(msg, "danger") flash(msg, "danger")
def flash_warning(msg): def flash_warning(msg):
""" Flash bootstrap warning message """ """Flash bootstrap warning message."""
flash(msg, "warning") flash(msg, "warning")
def flash_success(msg): def flash_success(msg):
""" Flash bootstrap success message """ """Flash bootstrap success message."""
flash(msg, "success") flash(msg, "success")
def flash_info(msg): def flash_info(msg):
""" Flash bootstrap info message """ """Flash bootstrap info message."""
flash(msg, "info") flash(msg, "info")
"""Utility functions""" """Utility functions."""
import unicodedata import unicodedata
def strip_accents(s): def strip_accents(s):
"""Remove accents from string, used to sort normalized strings""" """Remove accents from string, used to sort normalized strings."""
return "".join( return "".join(
c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn" c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"
) )
""" """Routes."""
Routes
"""
import copy import copy
import io import io
import json import json
...@@ -15,15 +13,12 @@ import jsonschema ...@@ -15,15 +13,12 @@ import jsonschema
import requests import requests
import validata_core import validata_core
from commonmark import commonmark from commonmark import commonmark
from flask import abort, make_response, redirect, render_template, request, url_for from flask import (abort, make_response, redirect, render_template, request,
url_for)
from opendataschema import GitSchemaReference, by_commit_date from opendataschema import GitSchemaReference, by_commit_date
from validata_core.helpers import ( from validata_core.helpers import (FileContentValidataResource,
FileContentValidataResource, URLValidataResource, ValidataResource,
URLValidataResource, is_body_error, is_structure_error)
ValidataResource,
is_body_error,
is_structure_error,
)
from . import app, config, fetch_schema, pdf_service, schema_catalog_registry from . import app, config, fetch_schema, pdf_service, schema_catalog_registry
from .model import Section from .model import Section
...@@ -34,16 +29,15 @@ log = logging.getLogger(__name__) ...@@ -34,16 +29,15 @@ log = logging.getLogger(__name__)
def get_schema_catalog(section_name): def get_schema_catalog(section_name):
"""Return a schema catalog associated to a section_name""" """Return a schema catalog associated to a section_name."""
return schema_catalog_registry.build_schema_catalog(section_name) return schema_catalog_registry.build_schema_catalog(section_name)
class SchemaInstance: class SchemaInstance:
"""Handy class to handle schema information""" """Handy class to handle schema information."""
def __init__(self, parameter_dict): def __init__(self, parameter_dict):
"""Initializes schema instance from requests dict and tableschema Catalog """Initialize schema instance and tableschema catalog."""
(for name ref)."""
self.section_name = None self.section_name = None
self.section_title = None self.section_title = None
self.name = None self.name = None
...@@ -75,8 +69,8 @@ class SchemaInstance: ...@@ -75,8 +69,8 @@ class SchemaInstance:
# Look for schema catalog first # Look for schema catalog first
try: try:
table_schema_catalog = get_schema_catalog(self.section_name) table_schema_catalog = get_schema_catalog(self.section_name)
except Exception as ex: except Exception:
log.exception(ex) log.exception("")
abort(400, "Erreur de traitement du catalogue") abort(400, "Erreur de traitement du catalogue")
if table_schema_catalog is None: if table_schema_catalog is None:
abort(400, "Catalogue indisponible") abort(400, "Catalogue indisponible")
...@@ -127,24 +121,26 @@ class SchemaInstance: ...@@ -127,24 +121,26 @@ class SchemaInstance:
try: try:
self.schema = fetch_schema(self.url) self.schema = fetch_schema(self.url)
except json.JSONDecodeError as e: except json.JSONDecodeError:
log.exception(e) err_msg = "Le format du schéma n'est pas reconnu"
flash_error("Le format du schéma n'est pas reconnu") log.exception(err_msg)
flash_error(err_msg)
abort(redirect(url_for("home"))) abort(redirect(url_for("home")))
except Exception as e: except Exception:
log.exception(e) err_msg = "Impossible de récupérer le schéma"
flash_error("Impossible de récupérer le schéma") log.exception(err_msg)
flash_error(err_msg)
abort(redirect(url_for("home"))) abort(redirect(url_for("home")))
def request_parameters(self): def request_parameters(self):
if self.name: """Build request parameter dict to identify schema."""
return { return {
"schema_name": self.schema_and_section_name, "schema_name": self.schema_and_section_name,
"schema_ref": "" if self.ref is None else self.ref, "schema_ref": "" if self.ref is None else self.ref,
} } if self.name else {"schema_url": self.url}
return {"schema_url": self.url}
def find_section_title(self, section_name): def find_section_title(self, section_name):
"""Return section title or None if not found."""
if config.CONFIG: if config.CONFIG:
for section in config.CONFIG.homepage.sections: for section in config.CONFIG.homepage.sections:
if section.name == section_name: if section.name == section_name:
...@@ -169,11 +165,10 @@ def build_template_source_data(header, rows, preview_rows_nb=5): ...@@ -169,11 +165,10 @@ def build_template_source_data(header, rows, preview_rows_nb=5):
def build_ui_errors(errors): def build_ui_errors(errors):
"""Add context to errors, converts markdown content to HTML""" """Add context to errors, converts markdown content to HTML."""
def improve_err(err): def improve_err(err):
"""Adds context info based on row-nb presence and converts content to HTML""" """Add context info based on row-nb presence and converts content to HTML."""
# Context # Context
update_keys = { update_keys = {
"context": "body" "context": "body"
...@@ -199,7 +194,9 @@ def build_ui_errors(errors): ...@@ -199,7 +194,9 @@ def build_ui_errors(errors):
def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict): def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict):
"""Creates an error report easier to handle and display in templates: """Create an error report easier to handle and display using templates.
improvements done:
- only one table - only one table
- errors are contextualized - errors are contextualized
- error-counts is ok - error-counts is ok
...@@ -207,14 +204,14 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict ...@@ -207,14 +204,14 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict
- errors are separated into "structure" and "body" - errors are separated into "structure" and "body"
- error messages are improved - error messages are improved
""" """
v_report = copy.deepcopy(validata_core_report) v_report = copy.deepcopy(validata_core_report.to_dict())
# Create a new UI report from information picked in validata report # Create a new UI report from information picked in validata report
ui_report = {} ui_report = {}
ui_report["table"] = {} ui_report["table"] = {}
# source headers # source headers
headers = v_report.table["header"] headers = v_report["tasks"][0]["resource"]["data"][0]
ui_report["table"]["header"] = headers ui_report["table"]["header"] = headers
# source dimension # source dimension
...@@ -235,9 +232,10 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict ...@@ -235,9 +232,10 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict
else "Cette colonne n'est pas définie dans le schema" else "Cette colonne n'est pas définie dans le schema"
for h in headers for h in headers
] ]
v_report_table = v_report["tasks"][0]
missing_headers = [ missing_headers = [
err["message-data"]["column-name"] err["message-data"]["column-name"]
for err in v_report.table["errors"] for err in v_report_table["errors"]
if err["code"] == "missing-header" if err["code"] == "missing-header"
] ]
ui_report["table"]["cols_alert"] = [ ui_report["table"]["cols_alert"] = [
...@@ -246,12 +244,12 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict ...@@ -246,12 +244,12 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict
] ]
# prepare error structure for UI needs # prepare error structure for UI needs
errors = build_ui_errors(v_report.table["errors"]) errors = build_ui_errors(v_report_table["errors"])