merge towards_frictionless_4

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 ## next
 
+## 0.5.0a4 (`towards_frictionless_4` branch)
+
+- fix flake8 issues
+- update requirements (validata-core 0.7.0a7)
+- rephrase presentation in footer
+
+## 0.5.0a3 (`towards_frictionless_4` branch)
+
+- update requirements (frictionless 4.1.0)
+
+## 0.5.0a2 (`towards_frictionless_4` branch)
+
+- update requirements (validata-core 0.7.0a2)
+- add explicit error message when a tabular data as url can't be retrieved (404)
+- improve error message ([issue](https://git.opendatafrance.net/validata/validata-ui/-/issues/82)]
+
+## 0.5.0a1 (`towards_frictionless_4` branch)
+
+- update requirements
+- adapt to changes in validation report
 ## 0.4.1
 
 - add explicit error message when a tabular data as an URL can't be retrieved (404)

requirements.txt

@@ -23,7 +23,7 @@ commonmark==0.9.1
     # via validata_ui (setup.py)
 decorator==4.4.2
     # via validators
-deprecated==1.2.11
+deprecated==1.2.12
     # via pygithub
 et-xmlfile==1.0.1
     # via openpyxl
@@ -33,27 +33,25 @@ ezodf==0.3.2
     #   validata_ui (setup.py)
 flask==1.1.2
     # via validata_ui (setup.py)
-frictionless==3.48.0
+frictionless==4.2.1
     # via
     #   validata-core
     #   validata_ui (setup.py)
 idna==2.10
     # via requests
-importlib-resources==5.1.0
+importlib-resources==5.1.2
     # via validata-core
 isodate==0.6.0
     # via frictionless
 itsdangerous==1.1.0
     # via flask
-jdcal==1.4.1
-    # via openpyxl
 jinja2==2.11.3
     # via flask
 jsonschema==3.2.0
     # via
     #   frictionless
     #   opendataschema
-lxml==4.6.2
+lxml==4.6.3
     # via
     #   validata-core
     #   validata_ui (setup.py)
@@ -63,13 +61,13 @@ numpy==1.20.1
     # via pandas
 opendataschema==0.6.0
     # via validata_ui (setup.py)
-openpyxl==3.0.6
+openpyxl==3.0.7
     # via validata-core
-pandas==1.2.1
+pandas==1.2.3
     # via tablib
 petl==1.7.2
     # via frictionless
-pydantic==1.7.3
+pydantic==1.8.1
     # via validata_ui (setup.py)
 pygithub==1.54.1
     # via opendataschema
@@ -109,6 +107,8 @@ requests==2.25.1
     #   requests-toolbelt
     #   validata-core
     #   validata_ui (setup.py)
+rfc3986==1.4.0
+    # via frictionless
 shellingham==1.4.0
     # via typer
 simpleeval==0.9.10
@@ -131,9 +131,11 @@ toolz==0.11.1
     # via validata-core
 typer[all]==0.3.2
     # via frictionless
-urllib3==1.26.3
+typing-extensions==3.7.4.3
+    # via pydantic
+urllib3==1.26.4
     # via requests
-validata-core==0.6.0
+validata-core==0.7.0a7
     # via validata_ui (setup.py)
 validators==0.18.2
     # via frictionless

setup.py

@@ -13,7 +13,7 @@ with readme_filepath.open("rt", encoding="utf-8") as fd:
 
 setup(
     name="validata_ui",
-    version="0.4.1",
+    version="0.5.0a4",
     description="Validata Web UI",
     long_description=LONG_DESCRIPTION,
     long_description_content_type="text/markdown",

validata_ui/__init__.py

+"""Validata UI."""
 import logging
 from datetime import timedelta
 

validata_ui/config.py

@@ -87,7 +87,7 @@ BROWSERLESS_API_TOKEN = os.getenv("BROWSERLESS_API_TOKEN") or None
 
 # Cache backend (default is SQLite)
 CACHE_BACKEND = os.getenv("CACHE_BACKEND") or "sqlite"
-log.info(f"Cache backend: {CACHE_BACKEND}")
+log.info("Cache backend: %r", CACHE_BACKEND)
 
 # Caching time for schema requests in minutes
 CACHE_EXPIRE_AFTER = os.getenv("CACHE_EXPIRE_AFTER") or None

validata_ui/model.py

-"""Config model using pydantic"""
+"""Config model using pydantic."""
 from typing import List, Optional, Union
 
 from pydantic import BaseModel, HttpUrl, root_validator
 
 
 class Link(BaseModel):
+    """Link class."""
+
     title: str
     url: HttpUrl
 
 
 class ExternalLink(BaseModel):
+    """ExternalLink class."""
+
     name: str
     type: str
     title: str
@@ -18,16 +22,22 @@ class ExternalLink(BaseModel):
 
 
 class Schema(BaseModel):
+    """Schema class."""
+
     name: str
     repo_url: HttpUrl
 
 
 class Catalog(BaseModel):
+    """Catalog class."""
+
     version: int
     schemas: List[Schema]
 
 
 class Section(BaseModel):
+    """Section class."""
+
     name: str
     title: str
     description: Optional[str] = None
@@ -35,7 +45,8 @@ class Section(BaseModel):
     links: Optional[List[ExternalLink]] = None
 
     @root_validator
-    def check_catalog_or_links(cls, values):
+    def check_catalog_or_links(cls, values): # noqa
+        """Check that catalog or links attributes is defined but not both."""
         catalog, links = values.get("catalog"), values.get("links")
         if catalog is None and links is None:
             raise ValueError("catalog or links field must be defined")
@@ -45,18 +56,26 @@ class Section(BaseModel):
 
 
 class Footer(BaseModel):
+    """Footer section."""
+
     links: List[Link]
 
 
 class Header(BaseModel):
+    """Header section."""
+
     links: List[Link]
 
 
 class Homepage(BaseModel):
+    """Homepage section."""
+
     sections: List[Section]
 
 
 class Config(BaseModel):
+    """Config class defines header, footer and homepage sections."""
+
     footer: Footer
     header: Header
     homepage: Homepage

validata_ui/pdf_renderer.py

@@ -38,11 +38,13 @@ class BrowserlessPDFRenderer(PDFRenderer):
     """Browserless IO implementation."""
 
     def __init__(self, api_url: str, api_token: str):
+        """Create browserless.io implementation."""
         log.info("BrowserlessPDFRenderer: creating instance with api_url = %r", api_url)
         self.api_url = api_url
         self.api_token = api_token
 
     def render(self, url: str) -> bytes:
+        """Call browserless.io service to generate PDF."""
         headers = {
             "Cache-Control": "no-cache",
             "Content-Type": "application/json",
@@ -82,6 +84,7 @@ class ChromiumHeadlessPDFRenderer(PDFRenderer):
     """Chromium implementation."""
 
     def render(self, url: str) -> bytes:
+        """Render PDF document using Chromium headless."""
         # Create temp file to save validation report
         # This temp file will be automatically deleted on context exit
         with tempfile.NamedTemporaryFile(

validata_ui/templates/base_template.html

@@ -66,7 +66,7 @@
 
     <footer class="footer hidden-print">
         <p>
-            Le <a href="https://www.validata.fr/">projet Validata</a> est
+            <a href="https://www.validata.fr/">Validata</a> est
             une initiative d'<a href="http://www.opendatafrance.net/">OpenDataFrance</a>
             développée par <a href="https://jailbreak.paris">Jailbreak</a>.
         </p>

validata_ui/ui_util.py

-"""
-    Util UI functions
-"""
+"""Util UI functions."""
 from flask import flash
 
 
 def flash_error(msg):
-    """ Flash bootstrap error message """
+    """Flash bootstrap error message."""
     flash(msg, "danger")
 
 
 def flash_warning(msg):
-    """ Flash bootstrap warning message """
+    """Flash bootstrap warning message."""
     flash(msg, "warning")
 
 
 def flash_success(msg):
-    """ Flash bootstrap success message """
+    """Flash bootstrap success message."""
     flash(msg, "success")
 
 
 def flash_info(msg):
-    """ Flash bootstrap info message """
+    """Flash bootstrap info message."""
     flash(msg, "info")

validata_ui/validata_util.py

-"""Utility functions"""
+"""Utility functions."""
 
 import unicodedata
 
 
 def strip_accents(s):
-    """Remove accents from string, used to sort normalized strings"""
+    """Remove accents from string, used to sort normalized strings."""
     return "".join(
         c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"
     )

validata_ui/views.py

-"""
-    Routes
-"""
+"""Routes."""
 import copy
 import io
 import json
@@ -15,15 +13,12 @@ import jsonschema
 import requests
 import validata_core
 from commonmark import commonmark
-from flask import abort, make_response, redirect, render_template, request, url_for
+from flask import (abort, make_response, redirect, render_template, request,
+                   url_for)
 from opendataschema import GitSchemaReference, by_commit_date
-from validata_core.helpers import (
-    FileContentValidataResource,
-    URLValidataResource,
-    ValidataResource,
-    is_body_error,
-    is_structure_error,
-)
+from validata_core.helpers import (FileContentValidataResource,
+                                   URLValidataResource, ValidataResource,
+                                   is_body_error, is_structure_error)
 
 from . import app, config, fetch_schema, pdf_service, schema_catalog_registry
 from .model import Section
@@ -34,16 +29,15 @@ log = logging.getLogger(__name__)
 
 
 def get_schema_catalog(section_name):
-    """Return a schema catalog associated to a section_name"""
+    """Return a schema catalog associated to a section_name."""
     return schema_catalog_registry.build_schema_catalog(section_name)
 
 
 class SchemaInstance:
-    """Handy class to handle schema information"""
+    """Handy class to handle schema information."""
 
     def __init__(self, parameter_dict):
-        """Initializes schema instance from requests dict and tableschema Catalog
-        (for name ref)."""
+        """Initialize schema instance and tableschema catalog."""
         self.section_name = None
         self.section_title = None
         self.name = None
@@ -75,8 +69,8 @@ class SchemaInstance:
             # Look for schema catalog first
             try:
                 table_schema_catalog = get_schema_catalog(self.section_name)
-            except Exception as ex:
-                log.exception(ex)
+            except Exception:
+                log.exception("")
                 abort(400, "Erreur de traitement du catalogue")
             if table_schema_catalog is None:
                 abort(400, "Catalogue indisponible")
@@ -127,24 +121,26 @@ class SchemaInstance:
 
         try:
             self.schema = fetch_schema(self.url)
-        except json.JSONDecodeError as e:
-            log.exception(e)
-            flash_error("Le format du schéma n'est pas reconnu")
+        except json.JSONDecodeError:
+            err_msg = "Le format du schéma n'est pas reconnu"
+            log.exception(err_msg)
+            flash_error(err_msg)
             abort(redirect(url_for("home")))
-        except Exception as e:
-            log.exception(e)
-            flash_error("Impossible de récupérer le schéma")
+        except Exception:
+            err_msg = "Impossible de récupérer le schéma"
+            log.exception(err_msg)
+            flash_error(err_msg)
             abort(redirect(url_for("home")))
 
     def request_parameters(self):
-        if self.name:
-            return {
-                "schema_name": self.schema_and_section_name,
-                "schema_ref": "" if self.ref is None else self.ref,
-            }
-        return {"schema_url": self.url}
+        """Build request parameter dict to identify schema."""
+        return {
+            "schema_name": self.schema_and_section_name,
+            "schema_ref": "" if self.ref is None else self.ref,
+        } if self.name else {"schema_url": self.url}
 
     def find_section_title(self, section_name):
+        """Return section title or None if not found."""
         if config.CONFIG:
             for section in config.CONFIG.homepage.sections:
                 if section.name == section_name:
@@ -169,11 +165,10 @@ def build_template_source_data(header, rows, preview_rows_nb=5):
 
 
 def build_ui_errors(errors):
-    """Add context to errors, converts markdown content to HTML"""
+    """Add context to errors, converts markdown content to HTML."""
 
     def improve_err(err):
-        """Adds context info based on row-nb presence and converts content to HTML"""
-
+        """Add context info based on row-nb presence and converts content to HTML."""
         # Context
         update_keys = {
             "context": "body"
@@ -199,7 +194,9 @@ def build_ui_errors(errors):
 
 
 def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict):
-    """Creates an error report easier to handle and display in templates:
+    """Create an error report easier to handle and display using templates.
+
+    improvements done:
     - only one table
     - errors are contextualized
     - error-counts is ok
@@ -207,14 +204,14 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict
     - errors are separated into "structure" and "body"
     - error messages are improved
     """
-    v_report = copy.deepcopy(validata_core_report)
+    v_report = copy.deepcopy(validata_core_report.to_dict())
 
     # Create a new UI report from information picked in validata report
     ui_report = {}
     ui_report["table"] = {}
 
     # source headers
-    headers = v_report.table["header"]
+    headers = v_report["tasks"][0]["resource"]["data"][0]
     ui_report["table"]["header"] = headers
 
     # source dimension
@@ -235,9 +232,10 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict
         else "Cette colonne n'est pas définie dans le schema"
         for h in headers
     ]
+    v_report_table = v_report["tasks"][0]
     missing_headers = [
         err["message-data"]["column-name"]
-        for err in v_report.table["errors"]
+        for err in v_report_table["errors"]
         if err["code"] == "missing-header"
     ]
     ui_report["table"]["cols_alert"] = [
@@ -246,12 +244,12 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict
     ]
 
     # prepare error structure for UI needs
-    errors = build_ui_errors(v_report.table["errors"])
+    errors = build_ui_errors(v_report_table["errors"])
 
     # Count errors and warnings
     ui_report["error_count"] = len(errors)
-    ui_report["warn_count"] = len(v_report.table["structure_warnings"])
-    ui_report["warnings"] = v_report.table["structure_warnings"]
+    ui_report["warn_count"] = len(v_report_table["structure_warnings"])
+    ui_report["warnings"] = v_report_table["structure_warnings"]
 
     # Then group them in 2 groups : structure and body
     ui_report["table"]["errors"] = {"structure": [], "body": []}
@@ -312,7 +310,7 @@ def create_validata_ui_report(rows_count: int, validata_core_report, schema_dict
 
 
 def compute_badge_message_and_color(badge):
-    """Computes message and color from badge information"""
+    """Compute message and color from badge information."""
     structure = badge["structure"]
     body = badge.get("body")
 
@@ -335,8 +333,7 @@ def compute_badge_message_and_color(badge):
 
 
 def get_badge_url_and_message(badge):
-    """Gets badge url from badge information"""
-
+    """Get badge url from badge information."""
     msg, color = compute_badge_message_and_color(badge)
     badge_url = "{}?{}".format(
         urljoin(config.SHIELDS_IO_BASE_URL, "/static/v1.svg"),
@@ -346,8 +343,7 @@ def get_badge_url_and_message(badge):
 
 
 def validate(schema_instance: SchemaInstance, validata_resource: ValidataResource):
-    """ Validate source and display report """
-
+    """Validate source and display report."""
     def compute_resource_info(resource: ValidataResource):
         source = resource.get_source()
         return {
@@ -397,7 +393,7 @@ def validate(schema_instance: SchemaInstance, validata_resource: ValidataResourc
     # Source error
     source_errors = [
         err
-        for err in validata_core_report["tables"][0]["errors"]
+        for err in validata_core_report["tasks"][0]["errors"]
         if err["code"] in {"source-error", "unknown-csv-dialect"}
     ]
     if source_errors:
@@ -465,7 +461,7 @@ def validate(schema_instance: SchemaInstance, validata_resource: ValidataResourc
 
 
 def bytes_data(f):
-    """ Gets bytes data from Werkzeug FileStorage instance """
+    """Get bytes data from Werkzeug FileStorage instance."""
     iob = io.BytesIO()
     f.save(iob)
     iob.seek(0)
@@ -477,7 +473,6 @@ def retrieve_schema_catalog(section: Section):
 
     def format_error_message(err_message, exc):
         """Prepare a bootstrap error message with details if wanted."""
-
         exception_text = "\n".join([str(arg) for arg in exc.args])
 
         return f"""{err_msg}
@@ -495,7 +490,6 @@ def retrieve_schema_catalog(section: Section):
         return (schema_catalog, None)
 
     except Exception as exc:
-        log.exception(exc)
         err_msg = "une erreur s'est produite"
         if isinstance(exc, requests.ConnectionError):
             err_msg = "problème de connexion"
@@ -503,6 +497,7 @@ def retrieve_schema_catalog(section: Section):
             err_msg = "format JSON incorrect"
         elif isinstance(exc, jsonschema.exceptions.ValidationError):
             err_msg = "le catalogue ne respecte pas le schéma de référence"
+        log.exception(err_msg)
 
         error_catalog = {
             **{k: v for k, v in section.dict().items() if k != "catalog"},
@@ -516,11 +511,10 @@ def retrieve_schema_catalog(section: Section):
 
 @app.route("/")
 def home():
-    """ Home page """
+    """Home page."""
 
     def iter_sections():
         """Yield sections of the home page, filled with schema metadata."""
-
         # Iterate on all sections
         for section in config.CONFIG.homepage.sections:
 
@@ -582,7 +576,7 @@ def home():
 
 @app.route("/pdf")
 def pdf_report():
-    """PDF report generation"""
+    """PDF report generation."""
     err_prefix = "Erreur de génération du rapport PDF"
 
     url_param = request.args.get("url")
@@ -629,13 +623,12 @@ def pdf_report():
 
 
 def extract_schema_metadata(table_schema: frictionless.Schema):
-    """Gets author, contibutor, version...metadata from schema header"""
+    """Get author, contibutor, version...metadata from schema header."""
     return {k: v for k, v in table_schema.items() if k != "fields"}
 
 
 def compute_schema_info(table_schema: frictionless.Schema, schema_url):
-    """Factor code for validator form page"""
-
+    """Factor code for validator form page."""
     # Schema URL + schema metadata info
     schema_info = {
         "path": schema_url,
@@ -648,15 +641,14 @@ def compute_schema_info(table_schema: frictionless.Schema, schema_url):
 
 
 def compute_validation_form_url(request_parameters: dict):
-    """Computes validation form url with schema URL parameter"""
+    """Compute validation form url with schema URL parameter."""
     url = url_for("custom_validator")
     return "{}?{}".format(url, urlencode(request_parameters))
 
 
 @app.route("/table-schema", methods=["GET", "POST"])
 def custom_validator():
-    """Validator form"""
-
+    """Display validator form."""
     if request.method == "GET":
 
         # input is a hidden form parameter to know