Commit 1f9a72b4 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

validate config with pydantic

parent a43f121d
......@@ -5,6 +5,7 @@ ezodf
flask
frictionless >= 3.39.0
lxml
pydantic
python-dotenv
pyyaml
requests
......
......@@ -35,6 +35,7 @@ opendataschema==0.5.5 # via -r requirements.in
openpyxl==3.0.5 # via validata-core
pandas==1.1.5 # via tablib
petl==1.6.8 # via frictionless
pydantic==1.7.3 # via -r requirements.in
pygithub==1.54 # via opendataschema
pyjwt==1.7.1 # via pygithub
pyrsistent==0.17.3 # via jsonschema
......
......@@ -51,14 +51,14 @@ class SchemaCatalogRegistry:
caching_session = cachecontrol.CacheControl(requests.Session())
fetch_schema = generate_schema_from_url_func(caching_session)
# And load schema catalogs which URLs are found in homepage_config.json
# And load schema catalogs which URLs are found in 'homepage' key of config.yaml
schema_catalog_registry = SchemaCatalogRegistry(caching_session)
if config.CONFIG:
log.info("Initializing homepage sections...")
for section in config.CONFIG["homepage"]["sections"]:
name = section["name"]
for section in config.CONFIG.homepage.sections:
name = section.name
log.info('Initializing homepage section "{}"...'.format(name))
catalog_ref = section.get("catalog")
catalog_ref = section.catalog
if catalog_ref:
schema_catalog_registry.add_ref(name, catalog_ref)
log.info("...done")
......
......@@ -8,6 +8,8 @@ import requests
import toml
from dotenv import load_dotenv
from . model import Config
log = logging.getLogger(__name__)
load_dotenv()
......@@ -49,12 +51,8 @@ CONFIG = None
if CONFIG_FILE:
CONFIG_FILE = Path(CONFIG_FILE)
with CONFIG_FILE.open() as fd:
try:
CONFIG = yaml.full_load(fd)
except ValueError as exc:
raise ValueError(
f"Could not load config YAML file {CONFIG_FILE}"
) from exc
config_dict = yaml.full_load(fd)
CONFIG = Config.parse_obj(config_dict)
MATOMO_BASE_URL = os.getenv("MATOMO_BASE_URL") or None
if MATOMO_BASE_URL:
......
"""Config model using pydantic"""
from typing import List, Union
from pydantic import BaseModel, HttpUrl
class Link(BaseModel):
title: str
url: HttpUrl
class ExternalLink(BaseModel):
name: str
type: str
title: str
description: str
website: HttpUrl
class Schema(BaseModel):
name: str
repo_url: HttpUrl
class Catalog(BaseModel):
version: str
schemas: List[Schema]
class Section(BaseModel):
name: str
title: str
description: str = ""
catalog: Union[HttpUrl, Catalog] = None
links: List[ExternalLink] = None
class Footer(BaseModel):
links: List[Link]
class Header(BaseModel):
links: List[Link]
class Homepage(BaseModel):
sections: List[Section]
class Config(BaseModel):
footer: Footer
header: Header
homepage: Homepage
\ No newline at end of file
......@@ -23,6 +23,7 @@ from opendataschema import GitSchemaReference, by_commit_date
import validata_core
from . import app, config, schema_catalog_registry, fetch_schema
from .model import Section
from .ui_util import flash_error, flash_warning
from .validata_util import (
UploadedFileValidataResource,
......@@ -147,9 +148,9 @@ class SchemaInstance:
def find_section_title(self, section_name):
if config.CONFIG:
for section in config.CONFIG["homepage"]["sections"]:
if section["name"] == section_name:
return section.get("title")
for section in config.CONFIG.homepage.sections:
if section.name == section_name:
return section.title
return None
......@@ -673,7 +674,7 @@ def bytes_data(f):
return iob.getvalue()
def retrieve_schema_catalog(section):
def retrieve_schema_catalog(section: Section):
"""Retrieve schema catalog and return formatted error if it fails."""
def format_error_message(err_message, exc):
......@@ -691,7 +692,7 @@ def retrieve_schema_catalog(section):
"""
try:
schema_catalog = get_schema_catalog(section["name"])
schema_catalog = get_schema_catalog(section.name)
return (schema_catalog, None)
except Exception as exc:
......@@ -705,7 +706,7 @@ def retrieve_schema_catalog(section):
err_msg = "le catalogue ne respecte pas le schéma de référence"
error_catalog = {
**{k: v for k, v in section.items() if k != "catalog"},
**{k: v for k, v in section.dict().items() if k != "catalog"},
"err": format_error_message(err_msg, exc),
}
return None, error_catalog
......@@ -720,19 +721,17 @@ def home():
def iter_sections():
"""Yield sections of the home page, filled with schema metadata."""
if not config.CONFIG["homepage"]:
return
# Iterate on all sections
for section in config.CONFIG["homepage"]["sections"]:
for section in config.CONFIG.homepage.sections:
# section with only links to external validators
if "links" in section:
if section.links:
yield section
continue
# section with catalog
if not "catalog" in section:
if section.catalog is None:
# skip section
continue
......@@ -774,7 +773,7 @@ def home():
)
yield {
**{k: v for k, v in section.items() if k != "catalog"},
**{k: v for k, v in section.dict().items() if k != "catalog"},
"catalog": schema_info_list,
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment