Commit 47c3ecbf authored by Pierre Dittgen's avatar Pierre Dittgen

Create schema_catalog on demand

parent 6e476a35
import json import json
import logging import logging
import os import os
import re
from pathlib import Path from pathlib import Path
from urllib.parse import quote_plus from urllib.parse import quote_plus
...@@ -29,20 +30,41 @@ def generate_schema_from_url_func(session): ...@@ -29,20 +30,41 @@ def generate_schema_from_url_func(session):
return tableschema_from_url return tableschema_from_url
def is_http_url(ref) -> bool:
return isinstance(ref, str) and re.match("https?://", ref)
class SchemaCatalogRegistry:
"""Retain section_name -> catalog url matching
and creates SchemaCatalog instance on demand"""
def __init__(self, session):
self.session = session
self.url_map = {}
def add_ref(self, name, url):
self.url_map[name] = url
def build_schema_catalog(self, name):
if name in self.url_map:
catalog_url = self.url_map[name]
return opendataschema.SchemaCatalog(catalog_url, session=self.session)
return None
caching_session = cachecontrol.CacheControl(requests.Session()) caching_session = cachecontrol.CacheControl(requests.Session())
tableschema_from_url = generate_schema_from_url_func(caching_session) tableschema_from_url = generate_schema_from_url_func(caching_session)
# And load schema catalogs which URLs are found in homepage_config.json # And load schema catalogs which URLs are found in homepage_config.json
schema_catalog_map = {} schema_catalog_registry = SchemaCatalogRegistry(caching_session)
if config.HOMEPAGE_CONFIG: if config.HOMEPAGE_CONFIG:
log.info("Initializing homepage sections...") log.info("Initializing homepage sections...")
for section in config.HOMEPAGE_CONFIG['sections']: for section in config.HOMEPAGE_CONFIG['sections']:
name = section['name'] name = section['name']
log.info('Initializing homepage section "{}"...'.format(name)) log.info('Initializing homepage section "{}"...'.format(name))
catalog = section.get('catalog') catalog_ref = section.get('catalog')
if catalog and isinstance(catalog, str): if is_http_url(catalog_ref):
schema_catalog_map[name] = opendataschema.SchemaCatalog(catalog, session=caching_session) schema_catalog_registry.add_ref(name, catalog_ref)
log.info("...done") log.info("...done")
# Flask things # Flask things
......
...@@ -23,7 +23,7 @@ from validata_core import messages ...@@ -23,7 +23,7 @@ from validata_core import messages
from opendataschema import GitSchemaReference from opendataschema import GitSchemaReference
from . import app, config, schema_catalog_map, tableschema_from_url from . import app, config, schema_catalog_registry, tableschema_from_url
from .ui_util import flash_error, flash_warning from .ui_util import flash_error, flash_warning
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
...@@ -32,10 +32,15 @@ MonkeyPatch.patch_fromisoformat() ...@@ -32,10 +32,15 @@ MonkeyPatch.patch_fromisoformat()
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def get_schema_catalog(section_name):
"""Return a schema catalog associated to a section_name"""
return schema_catalog_registry.build_schema_catalog(section_name)
class SchemaInstance: class SchemaInstance:
"""Handy class to handle schema information""" """Handy class to handle schema information"""
def __init__(self, parameter_dict, schema_catalog_map): def __init__(self, parameter_dict):
"""Initializes schema instance from requests dict and tableschema catalog (for name ref)""" """Initializes schema instance from requests dict and tableschema catalog (for name ref)"""
self.section_name = None self.section_name = None
self.section_title = None self.section_title = None
...@@ -66,7 +71,7 @@ class SchemaInstance: ...@@ -66,7 +71,7 @@ class SchemaInstance:
self.section_title = self.find_section_title(self.section_name) self.section_title = self.find_section_title(self.section_name)
# Look for schema catalog first # Look for schema catalog first
table_schema_catalog = schema_catalog_map.get(self.section_name) table_schema_catalog = get_schema_catalog(self.section_name)
if table_schema_catalog is None: if table_schema_catalog is None:
abort(400, "Section '{}' non trouvée dans la configuration".format(self.section_name)) abort(400, "Section '{}' non trouvée dans la configuration".format(self.section_name))
...@@ -413,16 +418,16 @@ def bytes_data(f): ...@@ -413,16 +418,16 @@ def bytes_data(f):
return iob.getvalue() return iob.getvalue()
def homepage_config_with_schema_metadata(ui_config, schema_catalog_map): def homepage_config_with_schema_metadata(ui_config):
"""Replace catalog url within ui_config by schema references """Replace catalog url within ui_config by schema references
containing schema metadata properties""" containing schema metadata properties"""
extended_ui_config = ui_config.copy() extended_ui_config = ui_config.copy()
for section in extended_ui_config['sections']: for section in extended_ui_config['sections']:
section_name = section['name'] section_name = section['name']
if section_name not in schema_catalog_map: schema_catalog = get_schema_catalog(section_name)
if schema_catalog is None:
continue continue
schema_catalog = schema_catalog_map[section_name]
schema_list = [] schema_list = []
for ref in schema_catalog.references: for ref in schema_catalog.references:
# Loads default table schema for each schema reference # Loads default table schema for each schema reference
...@@ -441,7 +446,7 @@ def homepage_config_with_schema_metadata(ui_config, schema_catalog_map): ...@@ -441,7 +446,7 @@ def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
@app.route('/') @app.route('/')
def home(): def home():
""" Home page """ """ Home page """
home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map) home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG)
return render_template('home.html', config=home_config) return render_template('home.html', config=home_config)
...@@ -455,7 +460,7 @@ def pdf_report(): ...@@ -455,7 +460,7 @@ def pdf_report():
flash_error(err_prefix + ' : URL non fournie') flash_error(err_prefix + ' : URL non fournie')
return redirect(url_for('home')) return redirect(url_for('home'))
schema_instance = SchemaInstance(request.args, schema_catalog_map) schema_instance = SchemaInstance(request.args)
# Compute pdf url report # Compute pdf url report
base_url = url_for('custom_validator', _external=True) base_url = url_for('custom_validator', _external=True)
...@@ -532,7 +537,7 @@ def custom_validator(): ...@@ -532,7 +537,7 @@ def custom_validator():
# url of resource to be validated # url of resource to be validated
url_param = request.args.get("url") url_param = request.args.get("url")
schema_instance = SchemaInstance(request.args, schema_catalog_map) schema_instance = SchemaInstance(request.args)
# First form display # First form display
if input_param is None: if input_param is None:
...@@ -561,7 +566,7 @@ def custom_validator(): ...@@ -561,7 +566,7 @@ def custom_validator():
elif request.method == 'POST': elif request.method == 'POST':
schema_instance = SchemaInstance(request.form, schema_catalog_map) schema_instance = SchemaInstance(request.form)
input_param = request.form.get('input') input_param = request.form.get('input')
if input_param is None: if input_param is None:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment