Commit ac14b236 authored by Christophe Benz's avatar Christophe Benz

Support tags and branches with latest tag, enhance homepage config file

parent 304c0a8f
Pipeline #1066 passed with stage
in 4 minutes and 55 seconds
......@@ -8,7 +8,7 @@
{
"name": "external",
"title": "Validateurs externes",
"catalog": [
"links": [
{
"name": "inspire",
"type": "external",
......
......@@ -5,7 +5,7 @@ ezodf==0.3.2
Flask==1.0.2
Flask-Matomo==1.2.0
lxml==4.2.5
opendataschema==0.2.0
opendataschema==0.4.0
python-dotenv==0.10.1
requests==2.22.0
toml==0.10.0
......
......@@ -64,7 +64,7 @@ setup(
'tabulator',
'opendataschema >= 0.3.0, < 0.4',
'opendataschema >= 0.4.0, < 0.5',
'validata_core >= 0.3.0, < 0.4',
],
......
......@@ -20,27 +20,29 @@ log = logging.getLogger(__name__)
def generate_schema_from_url_func(session):
"""Generates a function that encloses session"""
def schema_from_url(url):
req = session.get(url)
req.raise_for_status()
schema_dict = req.json()
return tableschema.Schema(schema_dict)
def tableschema_from_url(url):
response = session.get(url)
response.raise_for_status()
descriptor = response.json()
return tableschema.Schema(descriptor)
return schema_from_url
return tableschema_from_url
caching_session = cachecontrol.CacheControl(requests.Session())
tableschema_from_url = generate_schema_from_url_func(caching_session)
# And load schema catalogs which URLs are found in homepage_config.json
schema_catalog_map = {}
if config.HOMEPAGE_CONFIG:
log.info("Initializing homepage sections...")
caching_session = cachecontrol.CacheControl(requests.Session())
schema_from_url = generate_schema_from_url_func(caching_session)
for section in config.HOMEPAGE_CONFIG['sections']:
name = section['name']
log.info('Initializing homepage section "{}"...'.format(name))
if isinstance(section['catalog'], str) and section['catalog'].startswith('http'):
url = section['catalog']
schema_catalog_map[name] = opendataschema.SchemaCatalog(url, session=caching_session)
catalog = section.get('catalog')
if catalog and isinstance(catalog, str) and catalog.startswith('http'):
schema_catalog_map[name] = opendataschema.SchemaCatalog(catalog, session=caching_session)
log.info("...done")
# Flask things
......
......@@ -5,7 +5,7 @@
{% block content %}
{% for section in config.sections %}
{% if section.name != "external" %}
{% if section.catalog %}
<div class="mb-5">
<h2 class="my-4">{{section.title}}</h2>
<form action="{{ url_for('custom_validator') }}" method="GET">
......@@ -13,8 +13,8 @@
<div class="form-group col-lg-9">
<select required name="schema_name" class="form-control">
<option disabled selected value="">Choisissez un schéma</option>
{% for val in section.catalog | sort(attribute='title') %}
<option value="{{ val.name }}">{{ val.title }}</option>
{% for item in section.catalog | sort(attribute='title') %}
<option value="{{ item.name }}">{{ item.title }}</option>
{% endfor %}
</select>
</div>
......@@ -22,7 +22,6 @@
<button type="submit" class="btn btn-primary">Valider un fichier</button>
</div>
</div>
<input type="hidden" name="schema_ref" value="master" />
</form>
</div>
{% endif %}
......@@ -55,18 +54,18 @@
</div>
{% for section in config.sections %}
{% if section.name == "external" %}
{% if section.links %}
<div class="my-5">
<h2 class="my-4">{{ section.title }}</h2>
<div class="row my-4">
{% for val in section.catalog %}
{% for item in section.links %}
<div class="col-sm-4 mb-4">
<div class="card h-100">
<div class="card-body d-flex flex-column">
<h4 class="card-title">{{ val.title }}</h4>
<p class="card-text">{{ val.description }}</p>
<h4 class="card-title">{{ item.title }}</h4>
<p class="card-text">{{ item.description }}</p>
<a
href="{{ val.website }}" target="_blank"
href="{{ item.website }}" target="_blank"
class="btn btn-outline-secondary mt-auto"
>Utiliser
<i class="fas fa-external-link-alt ml-1"></i>
......
{% macro html(schema_info, schema_versions, schema_current_version, doc_url) %}
{% macro html(schema_info, branches, tags, schema_current_version, doc_url) %}
<h2 class="card-title">
{% if schema_info.title %}
{{ schema_info.title }}
......@@ -8,14 +8,25 @@
{% endif %}
</h2>
{% if schema_versions %}
{% if branches or tags %}
<form class="form-inline my-2">
<div class="form-group">
<label for="git_ref" class="mr-2">Version</label>
<select class="form-control" id="version_select" name="git_ref">
{% for sv in schema_versions %}
<option{% if sv.name == schema_current_version %} selected="selected"{% endif %}>{{ sv.name }}</option>
{% if tags %}
<optgroup label="Tags">
{% for tag in tags %}
<option{% if tag.name == schema_current_version %} selected{% endif %}>{{ tag.name }}</option>
{% endfor %}
</optgroup>
{% endif %}
{% if branches %}
<optgroup label="Branches">
{% for branch in branches %}
<option{% if branch.name == schema_current_version %} selected{% endif %}>{{ branch.name }}</option>
{% endfor %}
</optgroup>
{% endif %}
</select>
</div>
</form>
......
......@@ -9,7 +9,7 @@
<div class="col-lg-6 mb-4">
<div class="card">
<div class="card-body">
{{ schema_info_macros.html(schema_info, schema_versions, schema_current_version, doc_url) }}
{{ schema_info_macros.html(schema_info, branches, tags, schema_current_version, doc_url) }}
</div>
</div>
</div>
......@@ -87,7 +87,7 @@
{% endblock %}
{% block page_scripts %}
{% if schema_versions %}
{% if branches or tags %}
{{ schema_info_macros.script() }}
{% endif %}
{% endblock %}
\ No newline at end of file
......@@ -42,7 +42,7 @@
<div class="col-lg-6 mb-4">
<div class="card">
<div class="card-body">
{{ schema_info_macros.html(schema_info, None, schema_current_version, doc_url) }}
{{ schema_info_macros.html(schema_info, None, None, schema_current_version, doc_url) }}
</div>
</div>
</div>
......
......@@ -18,11 +18,12 @@ import tableschema
import tabulator
from backports.datetime_fromisoformat import MonkeyPatch
from commonmark import commonmark
from flask import make_response, redirect, render_template, request, url_for
from flask import abort, make_response, redirect, render_template, request, url_for
from validata_core import messages
from . import app, config, schema_catalog_map, schema_from_url
from opendataschema import GitSchemaReference
from . import app, config, schema_catalog_map, tableschema_from_url
from .ui_util import flash_error, flash_warning
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
......@@ -31,80 +32,90 @@ MonkeyPatch.patch_fromisoformat()
log = logging.getLogger(__name__)
class SchemaInstance():
"""Handly class to handle schema information"""
def __init__(self, url=None, section_name=None, name=None, ref=None, schema=None, versions=None, doc_url=None):
"""This function is not intended to be called directly
but via from_parameters() static method!"""
self.url = url
self.section_name = section_name
self.section_title = self.get_section_title(section_name) \
if section_name \
else "Autre schéma"
self.name = name
self.ref = ref
self.schema = schema
self.versions = versions
self.doc_url = doc_url
@staticmethod
def from_parameters(parameter_dict, schema_catalog_map):
"""Initializes schema instance from requests dict and tableschema catalog (for name ref)
"""
schema_url, section_name, schema_name, schema_ref, versions, doc_url = None, None, None, None, None, None
class SchemaInstance:
"""Handy class to handle schema information"""
def __init__(self, parameter_dict, schema_catalog_map):
"""Initializes schema instance from requests dict and tableschema catalog (for name ref)"""
self.section_name = None
self.section_title = None
self.name = None
self.url = None
self.ref = None
self.reference = None
self.doc_url = None
self.branches = None
self.tags = None
# From schema_url
if parameter_dict.get("schema_url"):
schema_url = parameter_dict["schema_url"]
self.url = parameter_dict["schema_url"]
self.section_title = "Autre schéma"
# from schema_name (and schema_ref)
elif parameter_dict.get('schema_name'):
schema_name = parameter_dict['schema_name']
schema_ref = parameter_dict.get('schema_ref')
self.schema_and_section_name = parameter_dict['schema_name']
self.ref = parameter_dict.get('schema_ref')
# Check schema name
chunks = schema_name.split('.')
chunks = self.schema_and_section_name.split('.')
if len(chunks) != 2:
return None
abort(400, "Paramètre 'schema_name' invalide")
section_name, ref_name = chunks
self.section_name, self.name = chunks
self.section_title = self.find_section_title(self.section_name)
# Look for schema catalog first
table_schema_catalog = schema_catalog_map.get(section_name)
table_schema_catalog = schema_catalog_map.get(self.section_name)
if table_schema_catalog is None:
return None
abort(400, "Section '{}' non trouvée dans la configuration".format(self.section_name))
schema_reference = table_schema_catalog.reference_by_name.get(self.name)
if schema_reference is None:
abort(400, "Schéma '{}' non trouvé dans le catalogue de la section '{}'".format(self.name, self.section_name))
self.doc_url = schema_reference.doc_url
# Unknown schema name?
table_schema_reference = table_schema_catalog.reference_by_name.get(ref_name)
if table_schema_reference is None:
return None
if isinstance(schema_reference, GitSchemaReference):
if self.ref is None:
schema_ref = schema_reference.get_latest_tag() or schema_reference.get_default_branch()
abort(redirect(compute_validation_form_url({
'schema_name': self.schema_and_section_name,
'schema_ref': schema_ref.name
})))
self.tags = list(schema_reference.iter_tags())
tag_names = [tag.name for tag in self.tags]
self.branches = [branch for branch in schema_reference.iter_branches()
if branch.name not in tag_names]
# Git refs
if table_schema_reference:
versions = table_schema_reference.get_refs()
doc_url = table_schema_reference.doc_url
options = {'ref': schema_ref} if schema_ref else {}
schema_url = table_schema_reference.get_schema_url(**options)
self.url = schema_reference.get_schema_url(ref=self.ref)
# else???
else:
return None
# Neither schema_name or schema_url were given.
abort(400, "L'un des paramètres est nécessaire : 'schema_name', 'schema_url'")
return SchemaInstance(url=schema_url, section_name=section_name, name=schema_name, ref=schema_ref,
schema=schema_from_url(schema_url), versions=versions, doc_url=doc_url)
try:
self.schema = tableschema_from_url(self.url)
except json.JSONDecodeError as e:
log.exception(e)
flash_error("Format de schéma non reconnu")
abort(redirect(url_for('home')))
except Exception as e:
log.exception(e)
flash_error("Erreur lors de l'obtention du schéma")
abort(redirect(url_for('home')))
def request_parameters(self):
if self.name:
return {
'schema_name': self.name,
'schema_name': self.schema_and_section_name,
'schema_ref': '' if self.ref is None else self.ref
}
return {
'schema_url': self.url
}
def get_section_title(self, section_name):
def find_section_title(self, section_name):
if config.HOMEPAGE_CONFIG:
for section in config.HOMEPAGE_CONFIG['sections']:
if section["name"] == section_name:
......@@ -290,8 +301,7 @@ def compute_badge_message_and_color(badge):
# Bad structure, stop here
if structure == 'KO':
return (
'structure invalide', 'red')
return ('structure invalide', 'red')
# No body error
if body == 'OK':
......@@ -314,8 +324,6 @@ def get_badge_url_and_message(badge):
def validate(schema_instance: SchemaInstance, source: ValidataResource):
""" Validate source and display report """
api_url = config.API_VALIDATE_ENDPOINT
# Useful to receive response as JSON
headers = {"Accept": "application/json"}
......@@ -325,22 +333,21 @@ def validate(schema_instance: SchemaInstance, source: ValidataResource):
"schema": schema_instance.url,
"url": source.url,
}
req = requests.get(api_url, params=params, headers=headers)
response = requests.get(config.API_VALIDATE_ENDPOINT, params=params, headers=headers)
else:
files = {'file': (source.filename, source.build_reader())}
data = {"schema": schema_instance.url}
req = requests.post(api_url, data=data, files=files, headers=headers)
response = requests.post(config.API_VALIDATE_ENDPOINT, data=data, files=files, headers=headers)
except requests.ConnectionError as err:
logging.exception(err)
flash_error("Erreur technique lors de la validation")
return redirect(url_for('home'))
if not req.ok:
flash_error("Une erreur s'est produite côté serveur :-(")
return redirect(compute_validation_form_url(schema_instance))
if not response.ok:
flash_error("Erreur technique lors de la validation")
return redirect(compute_validation_form_url(schema_instance.request_parameters()))
json_response = req.json()
json_response = response.json()
validata_core_report = json_response['report']
badge_info = json_response.get('badge')
......@@ -371,26 +378,32 @@ def validate(schema_instance: SchemaInstance, source: ValidataResource):
validata_report = create_validata_ui_report(validata_core_report, schema_instance.schema.descriptor)
# Display report to the user
validator_form_url = compute_validation_form_url(schema_instance)
validator_form_url = compute_validation_form_url(schema_instance.request_parameters())
schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
pdf_report_url = url_for('pdf_report')+'?'+urlencode(schema_instance.request_parameters())
pdf_report_url = url_for('pdf_report') + '?' + urlencode(schema_instance.request_parameters())
return render_template('validation_report.html',
schema_info=schema_info, report=validata_report,
pdf_report_url=pdf_report_url,
validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
source=source, source_data=source_data,
schema_current_version=schema_instance.ref,
badge_msg=badge_msg,
badge_url=badge_url,
breadcrumbs=[
{'title': 'Accueil', 'url': url_for('home')},
{'title': schema_instance.section_title},
{'title': schema_info['title'], 'url': validator_form_url},
{'title': 'Rapport de validation'},
],
display_badge=display_badge,
doc_url=schema_instance.doc_url,
pdf_report_url=pdf_report_url,
print_mode=request.args.get('print', 'false') == 'true',
display_badge=display_badge, badge_url=badge_url, badge_msg=badge_msg,
report_str=json.dumps(validata_report, sort_keys=True, indent=2),
report=validata_report,
schema_current_version=schema_instance.ref,
schema_info=schema_info,
section_title=schema_instance.section_title,
breadcrumbs=[
{'url': url_for('home'), 'title': 'Accueil'},
{'title': schema_instance.section_title},
{'url': validator_form_url, 'title': schema_info['title']},
{'title': 'Rapport de validation'},
])
source_data=source_data,
source=source,
validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
)
def bytes_data(f):
......@@ -414,7 +427,7 @@ def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
schema_list = []
for ref in schema_catalog.references:
# Loads default table schema for each schema reference
table_schema = schema_from_url(ref.get_schema_url(check_exists=False))
table_schema = tableschema_from_url(ref.get_schema_url())
schema_list.append({
'name': '{}.{}'.format(section_name, ref.name),
# Extracts title, description, ...
......@@ -440,13 +453,10 @@ def pdf_report():
url_param = request.args.get('url')
if not url_param:
flash_error(err_prefix + ': URL non fournie')
flash_error(err_prefix + ' : URL non fournie')
return redirect(url_for('home'))
schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
if schema_instance is None:
flash_error(err_prefix + ': Information de schema non fournie')
return redirect(url_for('home'))
schema_instance = SchemaInstance(request.args, schema_catalog_map)
# Compute pdf url report
base_url = url_for('custom_validator', _external=True)
......@@ -503,11 +513,10 @@ def compute_schema_info(table_schema: tableschema.Schema, schema_url):
return schema_info
def compute_validation_form_url(schema_instance: SchemaInstance):
def compute_validation_form_url(request_parameters: dict):
"""Computes validation form url with schema URL parameter"""
url = url_for('custom_validator')
param_list = ['{}={}'.format(k, quote_plus(v))
for k, v in schema_instance.request_parameters().items()]
param_list = ['{}={}'.format(k, quote_plus(v)) for k, v in request_parameters.items()]
return "{}?{}".format(url, '&'.join(param_list))
......@@ -524,28 +533,15 @@ def custom_validator():
# url of resource to be validated
url_param = request.args.get("url")
schema_instance = None
try:
schema_instance = SchemaInstance.from_parameters(request.args, schema_catalog_map)
except json.JSONDecodeError as e:
flash_error("Format de schéma non reconnu")
return redirect(url_for('home'))
except Exception as e:
log.exception(e)
flash_error("Erreur lors de l'obtention du schéma")
return redirect(url_for('home'))
if schema_instance is None:
flash_error("Aucun schéma passé en paramètre")
return redirect(url_for('home'))
schema_instance = SchemaInstance(request.args, schema_catalog_map)
# First form display
if input_param is None:
schema_versions = schema_instance.versions
schema_info = compute_schema_info(schema_instance.schema, schema_instance.url)
return render_template('validation_form.html',
schema_info=schema_info,
schema_versions=schema_versions,
branches=schema_instance.branches,
tags=schema_instance.tags,
schema_current_version=schema_instance.ref,
doc_url=schema_instance.doc_url,
schema_params=schema_instance.request_parameters(),
......@@ -558,39 +554,30 @@ def custom_validator():
# Process URL
else:
if url_param is None or url_param == '':
if not url_param:
flash_error("Vous n'avez pas indiqué d'URL à valider")
return redirect(compute_validation_form_url(schema_instance))
try:
return validate(schema_instance, URLValidataResource(url_param))
except tabulator.exceptions.FormatError as e:
flash_error('Erreur : Format de ressource non supporté')
log.info(e)
return redirect(compute_validation_form_url(schema_instance))
except tabulator.exceptions.HTTPError as e:
flash_error('Erreur : impossible d\'accéder au fichier source en ligne')
log.info(e)
return redirect(compute_validation_form_url(schema_instance))
else: # POST
schema_instance = SchemaInstance.from_parameters(request.form, schema_catalog_map)
if schema_instance is None:
flash_error('Aucun schéma défini')
return redirect(url_for('home'))
return redirect(compute_validation_form_url(schema_instance.request_parameters()))
return validate(schema_instance, URLValidataResource(url_param))
elif request.method == 'POST':
schema_instance = SchemaInstance(request.form, schema_catalog_map)
input_param = request.form.get('input')
if input_param is None:
flash_error("Vous n'avez pas indiqué de fichier à valider")
return redirect(compute_validation_form_url(schema_instance))
return redirect(compute_validation_form_url(schema_instance.request_parameters()))
# File validation
if input_param == 'file':
f = request.files.get('file')
if f is None:
flash_warning("Vous n'avez pas indiqué de fichier à valider")
return redirect(compute_validation_form_url(schema_instance))
return redirect(compute_validation_form_url(schema_instance.request_parameters()))
return validate(schema_instance, UploadedFileValidataResource(f.filename, bytes_data(f)))
return 'Bizarre, vous avez dit bizarre ?'
return 'Combinaison de paramètres non supportée', 400
else:
return "Method not allowed", 405
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment