Commit a6e7f236 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

wip connect to validata_core / build ui_report from scratch

parent 1a9328c2
......@@ -21,6 +21,7 @@ from commonmark import commonmark
from flask import abort, make_response, redirect, render_template, request, url_for
from opendataschema import GitSchemaReference, by_commit_date
import validata_core
from validata_core.helpers import translate_error_code
from . import app, config, schema_catalog_registry, fetch_schema
......@@ -132,7 +133,7 @@ class SchemaInstance:
return None
def extract_source_data(source: ValidataResource, schema_descriptor, preview_rows_nb=5):
def extract_source_data(validata_source: ValidataResource, schema_descriptor, preview_rows_nb=5):
"""Computes table preview"""
def stringify(val):
......@@ -155,7 +156,7 @@ def extract_source_data(source: ValidataResource, schema_descriptor, preview_row
rows = []
nb_rows = 0
table_source, table_options = source.build_stream_args()
table_source, table_options = validata_source.build_stream_args()
# Gets original source, header and rows
source_header = None
......@@ -206,7 +207,7 @@ def extract_source_data(source: ValidataResource, schema_descriptor, preview_row
}
def improve_errors(errors):
def build_ui_errors(errors):
"""Add context to errors, converts markdown content to HTML"""
def improve_err(err):
......@@ -388,58 +389,56 @@ def create_validata_ui_report(validata_core_report, schema_dict):
- errors are separated into "structure" and "body"
- error messages are improved
"""
report = copy.deepcopy(validata_core_report)
v_report = copy.deepcopy(validata_core_report)
with open("frictionless_report.json", "wt", encoding="utf-8") as fd:
fd.write(json.dumps(v_report, ensure_ascii=True, sort_keys=True, indent=2))
# One table is enough
report['table'] = report['tables'][0]
del report['tables']
del report['table']['time']
del report['table']['valid']
del report['valid']
# use _ instead of - to ease information picking in jinja2 template
report['table']['row_count'] = report['table']['stats']["rows"]
# Create a new UI report from information picked in validata report
ui_report = {}
ui_report["table"] = {}
# Handy col_count info
headers = report['table']['header']
report['table']['col_count'] = len(headers)
# source headers
headers = v_report.table['header']
ui_report["table"]["header"] = headers
# Computes column info
# source dimension
ui_report['table']['col_count'] = len(headers)
ui_report['table']['row_count'] = v_report.table['stats']["rows"]
# Computes column info from schema
fields_dict = {f['name']: (f.get('title', f['name']), f.get('description', ''))
for f in schema_dict.get('fields', [])}
report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'Colonne inconnue' for h in headers]
report['table']['headers_description'] = [fields_dict[h][1]
ui_report['table']['headers_title'] = [fields_dict[h][0] if h in fields_dict else 'Colonne inconnue' for h in headers]
ui_report['table']['headers_description'] = [fields_dict[h][1]
if h in fields_dict else 'Cette colonne n\'est pas définie dans le schema' for h in headers]
missing_headers = [err['message-data']['column-name']
for err in report['table']['errors']
for err in v_report.table['errors']
if err['code'] == 'missing-header']
report['table']['cols_alert'] = ['table-danger' if h not in fields_dict or h in missing_headers else ''
ui_report['table']['cols_alert'] = ['table-danger' if h not in fields_dict or h in missing_headers else ''
for h in headers]
# Provide better (french) messages
errors = improve_errors(report['table']['errors'])
del report['table']['errors']
# prepare error structure for UI needs
errors = build_ui_errors(v_report.table['errors'])
# Count errors
report['error_count'] = len(errors)
ui_report['error_count'] = len(errors)
# Then group them in 2 groups : structure and body
report['table']['errors'] = {'structure': [], 'body': []}
ui_report['table']['errors'] = {'structure': [], 'body': []}
for err in errors:
if '#head' in err['tags'] or '#structure' in err['tags']:
report['table']['errors']['structure'].append(err)
ui_report['table']['errors']['structure'].append(err)
else:
report['table']['errors']['body'].append(err)
ui_report['table']['errors']['body'].append(err)
# Group body errors by row id
rows = []
current_row_id = 0
for err in report['table']['errors']['body']:
for err in ui_report['table']['errors']['body']:
if not 'rowPosition' in err:
print('ERR', err)
row_id = err['rowPosition']
# del err['row-number']
# del err['context']
if row_id != current_row_id:
current_row_id = row_id
rows.append({'row_id': current_row_id, 'errors': {}})
......@@ -449,18 +448,18 @@ def create_validata_ui_report(validata_core_report, schema_dict):
rows[-1]['errors'][column_id] = err
else:
rows[-1]['errors']['row'] = err
report['table']['errors']['body_by_rows'] = rows
ui_report['table']['errors']['body_by_rows'] = rows
report['repair_actions'] = compute_repair_actions(report['table']['errors']['structure'])
ui_report['repair_actions'] = compute_repair_actions(ui_report['table']['errors']['structure'])
# Sort by error names in statistics
report["table"]["count-by-code"] = {}
ui_report["table"]["count-by-code"] = {}
stats = {}
total_errors_count = 0
for key in ('structure', 'body'):
# convert dict into tuples with french title instead of error code
# and sorts by title
key_errors = report["table"]["errors"][key]
key_errors = ui_report["table"]["errors"][key]
key_errors_count = len(key_errors)
ct = Counter(ke["code"] for ke in key_errors)
......@@ -478,9 +477,9 @@ def create_validata_ui_report(validata_core_report, schema_dict):
stats[f"{key}-errors"] = error_stats
stats["count"] = total_errors_count
report["table"]["error-stats"] = stats
ui_report["table"]["error-stats"] = stats
return report
return ui_report
def compute_badge_message_and_color(badge):
......@@ -513,36 +512,15 @@ def get_badge_url_and_message(badge):
return (badge_url, msg)
def validate(schema_instance: SchemaInstance, source: ValidataResource):
def validate(schema_instance: SchemaInstance, validata_source: ValidataResource):
""" Validate source and display report """
# Useful to receive response as JSON
headers = {"Accept": "application/json"}
try:
if source.type == 'url':
params = {
"schema": schema_instance.url,
"url": source.url,
"repair": True,
}
response = requests.get(config.API_VALIDATE_ENDPOINT, params=params, headers=headers)
else:
files = {'file': (source.filename, source.build_reader())}
data = {"schema": schema_instance.url, "repair": True}
response = requests.post(config.API_VALIDATE_ENDPOINT, data=data, files=files, headers=headers)
except requests.ConnectionError as err:
logging.exception(err)
flash_error("Une erreur est survenue lors de la validation")
return redirect(url_for('home'))
if not response.ok:
flash_error("Une erreur est survenue lors de la validation")
return redirect(compute_validation_form_url(schema_instance.request_parameters()))
# Call validata_core to validate source
source, source_options = validata_source.build_stream_args()
validata_core_report = validata_core.validate(source, schema_instance.url, **source_options)
json_response = response.json()
validata_core_report = json_response['report']
badge_info = json_response.get('badge')
# disable badge
badge_info = None
# Computes badge from report and badge configuration
badge_url, badge_msg = None, None
......@@ -562,15 +540,15 @@ def validate(schema_instance: SchemaInstance, source: ValidataResource):
flash_error('Erreur de source : {}'.format(msg))
return redirect(url_for('custom_validator'))
source_data = extract_source_data(source, schema_instance.schema)
source_data = extract_source_data(validata_source, schema_instance.schema)
# handle report date
report_datetime = datetime.fromisoformat(validata_core_report['date']).astimezone()
# Enhance validata_core_report
validata_report = create_validata_ui_report(validata_core_report, schema_instance.schema)
# create ui_report
ui_report = create_validata_ui_report(validata_core_report, schema_instance.schema)
#open("err_validata_core_report.json", "wt", encoding="utf-8").write(json.dumps(validata_report, indent=2, sort_keys=True, ensure_ascii=False))
#open("err_validata_core_report.json", "wt", encoding="utf-8").write(json.dumps(ui_report, indent=2, sort_keys=True, ensure_ascii=False))
# Display report to the user
......@@ -579,8 +557,8 @@ def validate(schema_instance: SchemaInstance, source: ValidataResource):
pdf_report_url = "{}?{}".format(url_for('pdf_report'),
urlencode({
**schema_instance.request_parameters(),
"url": source.url,
})) if source.type == 'url' else None
"url": validata_source.url,
})) if validata_source.type == 'url' else None
return render_template('validation_report.html',
badge_msg=badge_msg,
......@@ -595,13 +573,13 @@ def validate(schema_instance: SchemaInstance, source: ValidataResource):
doc_url=schema_instance.doc_url,
pdf_report_url=pdf_report_url,
print_mode=request.args.get('print', 'false') == 'true',
report_str=json.dumps(validata_report, sort_keys=True, indent=2),
report=validata_report,
report_str=json.dumps(ui_report, sort_keys=True, indent=2),
report=ui_report,
schema_current_version=schema_instance.ref,
schema_info=schema_info,
section_title=schema_instance.section_title,
source_data=source_data,
source=source,
source=validata_source,
validation_date=report_datetime.strftime('le %d/%m/%Y à %Hh%M'),
)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment