Commit 04f1214f authored by Pierre Dittgen's avatar Pierre Dittgen

Merge branch 'directcatalog' into next

parents 90536e49 47c3ecbf
Pipeline #1084 passed with stage
in 4 minutes and 59 seconds
import json import json
import logging import logging
import os import os
import re
from pathlib import Path from pathlib import Path
from urllib.parse import quote_plus from urllib.parse import quote_plus
...@@ -29,20 +30,41 @@ def generate_schema_from_url_func(session): ...@@ -29,20 +30,41 @@ def generate_schema_from_url_func(session):
return tableschema_from_url return tableschema_from_url
def is_http_url(ref) -> bool:
return isinstance(ref, str) and re.match("https?://", ref)
class SchemaCatalogRegistry:
"""Retain section_name -> catalog url matching
and creates SchemaCatalog instance on demand"""
def __init__(self, session):
self.session = session
self.url_map = {}
def add_ref(self, name, url):
self.url_map[name] = url
def build_schema_catalog(self, name):
if name in self.url_map:
catalog_url = self.url_map[name]
return opendataschema.SchemaCatalog(catalog_url, session=self.session)
return None
caching_session = cachecontrol.CacheControl(requests.Session()) caching_session = cachecontrol.CacheControl(requests.Session())
tableschema_from_url = generate_schema_from_url_func(caching_session) tableschema_from_url = generate_schema_from_url_func(caching_session)
# And load schema catalogs which URLs are found in homepage_config.json # And load schema catalogs which URLs are found in homepage_config.json
schema_catalog_map = {} schema_catalog_registry = SchemaCatalogRegistry(caching_session)
if config.HOMEPAGE_CONFIG: if config.HOMEPAGE_CONFIG:
log.info("Initializing homepage sections...") log.info("Initializing homepage sections...")
for section in config.HOMEPAGE_CONFIG['sections']: for section in config.HOMEPAGE_CONFIG['sections']:
name = section['name'] name = section['name']
log.info('Initializing homepage section "{}"...'.format(name)) log.info('Initializing homepage section "{}"...'.format(name))
catalog = section.get('catalog') catalog_ref = section.get('catalog')
if catalog and isinstance(catalog, str): if is_http_url(catalog_ref):
schema_catalog_map[name] = opendataschema.SchemaCatalog(catalog, session=caching_session) schema_catalog_registry.add_ref(name, catalog_ref)
log.info("...done") log.info("...done")
# Flask things # Flask things
......
...@@ -23,7 +23,7 @@ from validata_core import messages ...@@ -23,7 +23,7 @@ from validata_core import messages
from opendataschema import GitSchemaReference, by_semver from opendataschema import GitSchemaReference, by_semver
from . import app, config, schema_catalog_map, tableschema_from_url from . import app, config, schema_catalog_registry, tableschema_from_url
from .ui_util import flash_error, flash_warning from .ui_util import flash_error, flash_warning
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
...@@ -32,10 +32,15 @@ MonkeyPatch.patch_fromisoformat() ...@@ -32,10 +32,15 @@ MonkeyPatch.patch_fromisoformat()
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def get_schema_catalog(section_name):
"""Return a schema catalog associated to a section_name"""
return schema_catalog_registry.build_schema_catalog(section_name)
class SchemaInstance: class SchemaInstance:
"""Handy class to handle schema information""" """Handy class to handle schema information"""
def __init__(self, parameter_dict, schema_catalog_map): def __init__(self, parameter_dict):
"""Initializes schema instance from requests dict and tableschema catalog (for name ref)""" """Initializes schema instance from requests dict and tableschema catalog (for name ref)"""
self.section_name = None self.section_name = None
self.section_title = None self.section_title = None
...@@ -66,7 +71,7 @@ class SchemaInstance: ...@@ -66,7 +71,7 @@ class SchemaInstance:
self.section_title = self.find_section_title(self.section_name) self.section_title = self.find_section_title(self.section_name)
# Look for schema catalog first # Look for schema catalog first
table_schema_catalog = schema_catalog_map.get(self.section_name) table_schema_catalog = get_schema_catalog(self.section_name)
if table_schema_catalog is None: if table_schema_catalog is None:
abort(400, "Section '{}' non trouvée dans la configuration".format(self.section_name)) abort(400, "Section '{}' non trouvée dans la configuration".format(self.section_name))
...@@ -413,16 +418,16 @@ def bytes_data(f): ...@@ -413,16 +418,16 @@ def bytes_data(f):
return iob.getvalue() return iob.getvalue()
def homepage_config_with_schema_metadata(ui_config, schema_catalog_map): def homepage_config_with_schema_metadata(ui_config):
"""Replace catalog url within ui_config by schema references """Replace catalog url within ui_config by schema references
containing schema metadata properties""" containing schema metadata properties"""
extended_ui_config = ui_config.copy() extended_ui_config = ui_config.copy()
for section in extended_ui_config['sections']: for section in extended_ui_config['sections']:
section_name = section['name'] section_name = section['name']
if section_name not in schema_catalog_map: schema_catalog = get_schema_catalog(section_name)
if schema_catalog is None:
continue continue
schema_catalog = schema_catalog_map[section_name]
schema_list = [] schema_list = []
for ref in schema_catalog.references: for ref in schema_catalog.references:
# Loads default table schema for each schema reference # Loads default table schema for each schema reference
...@@ -441,7 +446,7 @@ def homepage_config_with_schema_metadata(ui_config, schema_catalog_map): ...@@ -441,7 +446,7 @@ def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
@app.route('/') @app.route('/')
def home(): def home():
""" Home page """ """ Home page """
home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map) home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG)
return render_template('home.html', config=home_config) return render_template('home.html', config=home_config)
...@@ -455,7 +460,7 @@ def pdf_report(): ...@@ -455,7 +460,7 @@ def pdf_report():
flash_error(err_prefix + ' : URL non fournie') flash_error(err_prefix + ' : URL non fournie')
return redirect(url_for('home')) return redirect(url_for('home'))
schema_instance = SchemaInstance(request.args, schema_catalog_map) schema_instance = SchemaInstance(request.args)
# Compute pdf url report # Compute pdf url report
base_url = url_for('custom_validator', _external=True) base_url = url_for('custom_validator', _external=True)
...@@ -532,7 +537,7 @@ def custom_validator(): ...@@ -532,7 +537,7 @@ def custom_validator():
# url of resource to be validated # url of resource to be validated
url_param = request.args.get("url") url_param = request.args.get("url")
schema_instance = SchemaInstance(request.args, schema_catalog_map) schema_instance = SchemaInstance(request.args)
# First form display # First form display
if input_param is None: if input_param is None:
...@@ -561,7 +566,7 @@ def custom_validator(): ...@@ -561,7 +566,7 @@ def custom_validator():
elif request.method == 'POST': elif request.method == 'POST':
schema_instance = SchemaInstance(request.form, schema_catalog_map) schema_instance = SchemaInstance(request.form)
input_param = request.form.get('input') input_param = request.form.get('input')
if input_param is None: if input_param is None:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment