Commit 69ab0b10 authored by Christophe Benz's avatar Christophe Benz
Browse files

Merge branch '75_custom_links' into 'frictionless_scratch'

Amélioration du système de personnalisation de Validata

See merge request !7
parents f18e56e2 bbd18e9b
......@@ -12,8 +12,8 @@ BADGE_CONFIG_URL="https://git.opendatafrance.net/validata/validata-badge/raw/mas
# Comment the following line to disable badge display
SHIELDS_IO_BASE_URL="https://img.shields.io/"
# Homepage sections and blocks config file path
HOMEPAGE_CONFIG_FILE=homepage_config.json.example
# Configuration file
CONFIG=config.example.yaml
# Uncomment to enable [Matomo](https://matomo.org/) analytics
# MATOMO_BASE_URL=""
......
......@@ -2,8 +2,8 @@ __pycache__/
validata_ui.egg-info/
.env
.vscode/
homepage_config.json
src/
config.yaml
# Cypress ui tests
screenshotsFolder
......
# Changelog
## next
- replace `homepage_config.json` file by `config.yaml` that contains homepage schemas and catalog
settings but also header and footer links
## 0.4.0a1
- Fix badge display
......
footer:
links:
- title: API
url: https://go.validata.fr/api/v1/apidocs
- title: Codes sources
url: https://git.opendatafrance.net/validata
- title: Salon de discussion public
url: https://riot.im/app/#/room/#validata:jailbreak.paris
header:
links:
- title: Mode d'emploi
url: https://validata.fr/doku.php?id=utilisation
- title: Documentation SCDL
url: https://scdl.opendatafrance.net/docs
- title: Forum
url: https://teamopendata.org/c/socle-commun-des-donnees-locales
homepage:
sections:
- name: scdl
title: Socle Commun des Données Locales
description: Ces schémas respectent <a href='https://scdl.opendatafrance.net/docs/CONTRIBUTING.html' target='_blank'>les standards du SCDL</a>.
catalog: https://git.opendatafrance.net/scdl/catalog/raw/master/catalog.json
- name: other
title: Autres schémas
description: Ces schémas sont proches du SCDL mais n'en font pas encore partie.
catalog:
version: 1
schemas:
- name: lieux-covoiturage
repo_url: https://github.com/etalab/schema-lieux-covoiturage
- name: decp-dpa
repo_url: https://github.com/etalab/schema-decp-dpa
- name: stationnement
repo_url: https://github.com/etalab/schema-stationnement
- name: external,
title: Validateurs externes
description: Sélection d'outils de validation de données complémentaires à Validata.
links:
- name: inspire
type: external
title: INSPIRE
description: proposé par la Commission Européenne pour tester des ressources géographiques (données, services ou métadonnées)
website: http://inspire-sandbox.jrc.ec.europa.eu/validator/
- name: bal
type: external
title: BAL
description: proposé par la mission Etalab pour tester des données Adresse produites localement (format BAL 1.1)
website: https://adresse.data.gouv.fr/bases-locales/validateur
- name: cvdtc
type: external
title: CVDTC
description: proposé par l'AFIMB dans le cadre du projet Chouette pour tester des données de transport collectif (GTFS ou NEPTUNE)
website: http://www.conversion-validation-donnees-tc.org/
{
"sections": [
{
"name": "schema-catalog",
"title": "A catalog of schemas",
"description": "This is a catalog of schemas that is published somewhere. (Note: the catalog must be conform to this schema: https://opendataschema.frama.io/catalog/schema-catalog.json",
"catalog": "https://git.opendatafrance.net/scdl/catalog/raw/master/catalog.json"
},
{
"name": "schema-list",
"title": "A list of schemas",
"description": "This is a list of other schemas.",
"catalog": {
"version": 1,
"schemas": [
{
"repo_url": "https://github.com/CharlesNepote/liste-prenoms-nouveaux-nes/",
"schema_filename": "prenom-schema.json",
"doc_url_by_ref": {
"~latest_tag": "https://scdl.opendatafrance.net/docs/schemas/prenoms.html",
"next": "https://scdl.opendatafrance.net/docs-next/schemas/prenoms.html"
}
},
{
"name": "irve",
"repo_url": "https://github.com/etalab/schema-irve/",
"doc_url_by_ref": {
"~latest_tag": "https://scdl.opendatafrance.net/docs/schemas/irve.html",
"next": "https://scdl.opendatafrance.net/docs-next/schemas/irve.html"
}
}
]
}
},
{
"name": "external",
"title": "A list of links to external validators",
"description": "This is a selection of validation tools that complement Validata.",
"links": [
{
"name": "inspire",
"type": "external",
"title": "INSPIRE",
"description": "proposé par la Commission Européenne pour tester des ressources géographiques (données, services ou métadonnées)",
"website": "http://inspire-sandbox.jrc.ec.europa.eu/validator/"
},
{
"name": "bal",
"type": "external",
"title": "BAL",
"description": "proposé par la mission Etalab pour tester des données Adresse produites localement (format BAL 1.1)",
"website": "https://adresse.data.gouv.fr/bases-locales/validateur"
}
]
}
]
}
\ No newline at end of file
......@@ -5,7 +5,9 @@ ezodf
flask
frictionless >= 3.39.0
lxml
pydantic
python-dotenv
pyyaml
requests
sentry-sdk[flask] == 0.14.3
toml
......
......@@ -20,6 +20,7 @@ ezodf==0.3.2 # via -r requirements.in, validata-core
flask==1.1.2 # via -r requirements.in, sentry-sdk
frictionless==3.46.0 # via -r requirements.in, validata-core
idna==2.10 # via requests
importlib-metadata==3.3.0 # via jsonschema
importlib-resources==3.3.0 # via validata-core
isodate==0.6.0 # via frictionless
itsdangerous==1.1.0 # via flask
......@@ -34,6 +35,7 @@ opendataschema==0.5.5 # via -r requirements.in
openpyxl==3.0.5 # via validata-core
pandas==1.1.5 # via tablib
petl==1.6.8 # via frictionless
pydantic==1.7.3 # via -r requirements.in
pygithub==1.54 # via opendataschema
pyjwt==1.7.1 # via pygithub
pyrsistent==0.17.3 # via jsonschema
......@@ -43,7 +45,7 @@ python-gitlab==2.5.0 # via opendataschema
python-slugify==4.0.1 # via frictionless
python-stdnum==1.14 # via validata-core
pytz==2020.4 # via pandas
pyyaml==5.3.1 # via frictionless
pyyaml==5.3.1 # via -r requirements.in, frictionless
requests==2.24.0 # via -r requirements.in, cachecontrol, frictionless, opendataschema, pygithub, python-gitlab, validata-core
sentry-sdk[flask]==0.14.3 # via -r requirements.in
shellingham==1.3.2 # via typer
......@@ -55,10 +57,12 @@ text-unidecode==1.3 # via python-slugify
toml==0.10.2 # via -r requirements.in
toolz==0.11.1 # via validata-core
typer[all]==0.3.2 # via frictionless
typing-extensions==3.7.4.3 # via importlib-metadata
urllib3==1.25.11 # via requests, sentry-sdk
validators==0.18.1 # via frictionless
werkzeug==1.0.1 # via flask
wrapt==1.12.1 # via deprecated
zipp==3.4.0 # via importlib-metadata, importlib-resources
# The following packages are considered to be unsafe in a requirements file:
# setuptools
......@@ -12,6 +12,7 @@ import opendataschema
import pkg_resources
import requests
from commonmark import commonmark
from pydantic import HttpUrl
from . import config
......@@ -51,15 +52,15 @@ class SchemaCatalogRegistry:
caching_session = cachecontrol.CacheControl(requests.Session())
fetch_schema = generate_schema_from_url_func(caching_session)
# And load schema catalogs which URLs are found in homepage_config.json
# And load schema catalogs which URLs are found in 'homepage' key of config.yaml
schema_catalog_registry = SchemaCatalogRegistry(caching_session)
if config.HOMEPAGE_CONFIG:
if config.CONFIG:
log.info("Initializing homepage sections...")
for section in config.HOMEPAGE_CONFIG["sections"]:
name = section["name"]
for section in config.CONFIG.homepage.sections:
name = section.name
log.info('Initializing homepage section "{}"...'.format(name))
catalog_ref = section.get("catalog")
if catalog_ref:
if section.catalog:
catalog_ref = str(section.catalog) if isinstance(section.catalog, HttpUrl) else section.catalog.dict()
schema_catalog_registry.add_ref(name, catalog_ref)
log.info("...done")
......
import json
import yaml
import logging
import os
import sys
......@@ -7,6 +7,9 @@ from pathlib import Path
import requests
import toml
from dotenv import load_dotenv
from pydantic.error_wrappers import ValidationError
from . model import Config
log = logging.getLogger(__name__)
......@@ -44,17 +47,19 @@ SHIELDS_IO_BASE_URL = os.environ.get("SHIELDS_IO_BASE_URL") or None
if SHIELDS_IO_BASE_URL:
SHIELDS_IO_BASE_URL = without_trailing_slash(SHIELDS_IO_BASE_URL)
HOMEPAGE_CONFIG_FILE = os.environ.get("HOMEPAGE_CONFIG_FILE") or None
HOMEPAGE_CONFIG = None
if HOMEPAGE_CONFIG_FILE:
HOMEPAGE_CONFIG_FILE = Path(HOMEPAGE_CONFIG_FILE)
with HOMEPAGE_CONFIG_FILE.open() as fd:
CONFIG_FILE = os.environ.get("CONFIG_FILE") or None
CONFIG = None
if CONFIG_FILE:
CONFIG_FILE = Path(CONFIG_FILE)
with CONFIG_FILE.open() as fd:
try:
config_dict = yaml.full_load(fd)
except yaml.scanner.ScannerError as exc:
raise ValueError(f"Could not load YAML config file {CONFIG_FILE}") from exc
try:
HOMEPAGE_CONFIG = json.load(fd)
except ValueError as exc:
raise ValueError(
f"Could not load homepage config JSON file {HOMEPAGE_CONFIG_FILE}"
) from exc
CONFIG = Config.parse_obj(config_dict)
except ValidationError as exc:
raise ValueError(f"Invalid config from {CONFIG_FILE}") from exc
MATOMO_BASE_URL = os.getenv("MATOMO_BASE_URL") or None
if MATOMO_BASE_URL:
......
"""Config model using pydantic"""
from typing import List, Optional, Union
from pydantic import BaseModel, HttpUrl, root_validator
class Link(BaseModel):
title: str
url: HttpUrl
class ExternalLink(BaseModel):
name: str
type: str
title: str
description: str
website: HttpUrl
class Schema(BaseModel):
name: str
repo_url: HttpUrl
class Catalog(BaseModel):
version: int
schemas: List[Schema]
class Section(BaseModel):
name: str
title: str
description: Optional[str] = ""
catalog: Optional[Union[HttpUrl, Catalog]] = None
links: Optional[List[ExternalLink]] = None
@root_validator
def check_catalog_or_links(cls, values):
catalog, links = values.get("catalog"), values.get("links")
if catalog is None and links is None:
raise ValueError("catalog or links field must be defined")
if catalog is not None and links is not None:
raise ValueError("only one of catalog and links must be defined")
return values
class Footer(BaseModel):
links: List[Link]
class Header(BaseModel):
links: List[Link]
class Homepage(BaseModel):
sections: List[Section]
class Config(BaseModel):
footer: Footer
header: Header
homepage: Homepage
\ No newline at end of file
......@@ -23,9 +23,9 @@
<a class="navbar-brand" href="{{ url_for('home') }}">
<img src="{{ url_for('static', filename='img/logo-horizontal.png') }}" height="15" alt="Validata" />
</a>
<a class="nav-link" href="https://validata.fr/doku.php?id=utilisation" target="_blank">Mode d'emploi</a>
<a class="nav-link" href="https://scdl.opendatafrance.net/docs" target="_blank">Documentation SCDL</a>
<a class="nav-link" href="https://teamopendata.org/c/socle-commun-des-donnees-locales" target="_blank">Forum</a>
{% for link in config.header.links %}
<a class="nav-link" href="{{ link.url }}" target="_blank">{{ link.title }}</a>
{% endfor %}
<a class="btn btn-outline-danger ml-auto" rel="external" target="_blank" href="https://git.opendatafrance.net/validata/validata-ui/issues/new?issuable_template=Probl%C3%A8me">
Signaler un problème
</a>
......@@ -66,18 +66,18 @@
<footer class="footer hidden-print">
<p>
Le service de validation du <a href="https://www.validata.fr/">projet Validata</a>
est mis à disposition par
<a href="http://www.opendatafrance.net/">OpenDataFrance</a> avec
l'aide de <a href="https://jailbreak.paris">Jailbreak</a>.
Le <a href="https://www.validata.fr/">projet Validata</a> est
une initiative d'<a href="http://www.opendatafrance.net/">OpenDataFrance</a>
développée par <a href="https://jailbreak.paris">Jailbreak</a>.
</p>
<p>
<a href="/api/v1/apidocs" target="_blank">API</a>
| <a href="https://git.opendatafrance.net/validata" target="_blank">Codes sources</a>
| <a href="https://riot.im/app/#/room/#validata:jailbreak.paris" target="_blank">Salon de discussion public</a>
{% for link in config.footer.links %}
<a href="{{ link.url }}" target="_blank">{{ link.title }}</a>
{{ " | " if not loop.last }}
{% endfor %}
</p>
<p class="text-muted">
(version <a href="https://git.opendatafrance.net/validata/validata-ui/tree/v{{ validata_ui_version }}" target="_blank">{{ validata_ui_version }}</a>)
version <a href="https://git.opendatafrance.net/validata/validata-ui/tree/v{{ validata_ui_version }}" target="_blank">{{ validata_ui_version }}</a>
</p>
</footer>
......
......@@ -23,6 +23,7 @@ from opendataschema import GitSchemaReference, by_commit_date
import validata_core
from . import app, config, schema_catalog_registry, fetch_schema
from .model import Section
from .ui_util import flash_error, flash_warning
from .validata_util import (
UploadedFileValidataResource,
......@@ -146,10 +147,10 @@ class SchemaInstance:
return {"schema_url": self.url}
def find_section_title(self, section_name):
if config.HOMEPAGE_CONFIG:
for section in config.HOMEPAGE_CONFIG["sections"]:
if section["name"] == section_name:
return section.get("title")
if config.CONFIG:
for section in config.CONFIG.homepage.sections:
if section.name == section_name:
return section.title
return None
......@@ -642,6 +643,7 @@ def validate(schema_instance: SchemaInstance, validata_source: ValidataResource)
return render_template(
"validation_report.html",
config=config.CONFIG,
badge_msg=badge_msg,
badge_url=badge_url,
breadcrumbs=[
......@@ -673,7 +675,7 @@ def bytes_data(f):
return iob.getvalue()
def retrieve_schema_catalog(section):
def retrieve_schema_catalog(section: Section):
"""Retrieve schema catalog and return formatted error if it fails."""
def format_error_message(err_message, exc):
......@@ -691,7 +693,7 @@ def retrieve_schema_catalog(section):
"""
try:
schema_catalog = get_schema_catalog(section["name"])
schema_catalog = get_schema_catalog(section.name)
return (schema_catalog, None)
except Exception as exc:
......@@ -705,7 +707,7 @@ def retrieve_schema_catalog(section):
err_msg = "le catalogue ne respecte pas le schéma de référence"
error_catalog = {
**{k: v for k, v in section.items() if k != "catalog"},
**{k: v for k, v in section.dict().items() if k != "catalog"},
"err": format_error_message(err_msg, exc),
}
return None, error_catalog
......@@ -720,19 +722,17 @@ def home():
def iter_sections():
"""Yield sections of the home page, filled with schema metadata."""
if not config.HOMEPAGE_CONFIG:
return
# Iterate on all sections
for section in config.HOMEPAGE_CONFIG["sections"]:
for section in config.CONFIG.homepage.sections:
# section with only links to external validators
if "links" in section:
if section.links:
yield section
continue
# section with catalog
if not "catalog" in section:
if section.catalog is None:
# skip section
continue
......@@ -774,11 +774,11 @@ def home():
)
yield {
**{k: v for k, v in section.items() if k != "catalog"},
**{k: v for k, v in section.dict().items() if k != "catalog"},
"catalog": schema_info_list,
}
return render_template("home.html", sections=list(iter_sections()))
return render_template("home.html", config=config.CONFIG, sections=list(iter_sections()))
@app.route("/pdf")
......@@ -887,6 +887,7 @@ def custom_validator():
)
return render_template(
"validation_form.html",
config=config.CONFIG,
branches=schema_instance.branches,
breadcrumbs=[
{"url": url_for("home"), "title": "Accueil"},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment