Commit 6f13f896 authored by Pierre Dittgen's avatar Pierre Dittgen

Use get_tabulator_params from validata_code

parent 1a2ecbe2
#!/usr/bin/env python3
"""
Util UI functions
"""
from flask import flash
def flash_error(msg):
""" Flash bootstrap error message """
flash(msg, 'danger')
def flash_warning(msg):
""" Flash bootstrap warning message """
flash(msg, 'warning')
def flash_success(msg):
""" Flash bootstrap success message """
flash(msg, 'success')
def flash_info(msg):
""" Flash bootstrap info message """
flash(msg, 'info')
#!/usr/bin/env python3
"""
Util functions
"""
import json
from collections import namedtuple
from io import BytesIO
from flask import flash
from tabulator import helpers
from tabulator.loader import Loader
def flash_error(msg):
""" Flash bootstrap error message """
flash(msg, 'danger')
def flash_warning(msg):
""" Flash bootstrap warning message """
flash(msg, 'warning')
def flash_success(msg):
""" Flash bootstrap success message """
flash(msg, 'success')
def flash_info(msg):
""" Flash bootstrap info message """
flash(msg, 'info')
class ValidataSource():
""" Handy class to handle different sort of data source """
def __init__(self, data, name, type):
""" Initialization """
self.data = data
self.name = name
self.type = type
self.scheme = None
self.format = None
def get_goodtables_source(self):
""" Creates source ready to be ingested by tabulator """
self.scheme, self.format = helpers.detect_scheme_and_format(self.name)
# In case of uploaded file (we work with bytes string)
if self.type == 'file':
# CSV: converts to string
if self.format == 'csv':
self.scheme = 'text'
encoding = helpers.detect_encoding(self.data)
self.data = self.data.decode(encoding)
# Else use custom BytesLoader
else:
self.scheme = 'bytes'
return {'source': self.data, 'format': self.format, 'scheme': self.scheme}
#!/usr/bin/env python3
""" Call validation code """
import logging
from io import BytesIO
import validata_core
from validata_core.source_helpers import build_tabulator_params
log = logging.getLogger(__name__)
class ValidataSource():
""" Handy class to handle different sort of data source """
def __init__(self, type, name, source):
""" Initialization """
self.type = type
self.name = name
self.source = source
info = build_tabulator_params(type, name, source)
self.source = info.get('source')
self.format = info.get('format')
self.scheme = info.get('scheme')
def get_tabulator_params(self):
""" Creates source ready to be ingested by tabulator """
return {'source': self.source, 'format': self.format, 'scheme': self.scheme}
def bytes_data(f):
"""Gets bytes data from Werkzeug FileStorage instance"""
iob = BytesIO()
f.save(iob)
iob.seek(0)
return iob.getvalue()
def validate(schema_code, **args):
""" Validate source against schema using custom-checks and pre-checks"""
return validata_core.validate(
source=args['source'],
schema=schema_code,
force_strings=True,
**{k: v for k, v in args.items() if k != 'source'}
)
......@@ -5,8 +5,6 @@ import logging
import requests
from validata_core import validate
#
# MEMENTO: json.load(pkg_resources.resource_stream('validata_core', 'spec.json'))
#
......@@ -73,23 +71,3 @@ class ValidatorHelper:
def schema_info_list(cls):
""" Computes and return schema info list """
return [cls.schema_info(code) for code in sorted(cls.schema_dict.keys())]
@classmethod
def validate(cls, schema_code, **args):
""" Validate source against schema using custom-checks """
# Gets schema info
sc_info = cls.schema_dict[schema_code]
# Build checks configuration
pre_checks_conf = None
if 'goodtables_checks' in sc_info:
pre_checks_conf = sc_info['goodtables_checks'].get('pre_checks')
return validate(
source=args['source'],
schema=schema_code,
pre_checks_conf=pre_checks_conf,
force_strings=True,
**{k: v for k, v in args.items() if k != 'source'}
)
......@@ -4,22 +4,20 @@
"""
import copy
import json
import os
from collections import OrderedDict
from datetime import datetime
from io import BytesIO
from pathlib import Path
from commonmark import commonmark
from flask import Flask, jsonify, redirect, render_template, request, url_for
from flask import redirect, render_template, request, url_for
import tabulator
from validata_ui import app
from validata_ui.util import (ValidataSource, flash_error, flash_info,
flash_success, flash_warning)
from validata_ui.validate_helper import ValidatorHelper
import validata_ui.validata_util as vu
from validata_core import csv_helpers
from validata_core.loaders import custom_loaders
from validata_ui import app
from validata_ui.ui_util import (flash_error, flash_warning)
from validata_ui.validata_util import ValidataSource
from validata_ui.validate_helper import ValidatorHelper
def extract_source_data(source: ValidataSource, preview_rows_nb=5):
......@@ -35,9 +33,9 @@ def extract_source_data(source: ValidataSource, preview_rows_nb=5):
options = {}
if source.format == "csv":
options['delimiter'] = csv_helpers.detect_dialect(source.data, format=source.format, scheme=source.scheme,
options['delimiter'] = csv_helpers.detect_dialect(source.source, format=source.format, scheme=source.scheme,
custom_loaders=custom_loaders).delimiter
with tabulator.Stream(source.data, format=source.format, scheme=source.scheme, custom_loaders=custom_loaders,
with tabulator.Stream(source.source, format=source.format, scheme=source.scheme, custom_loaders=custom_loaders,
**options) as stream:
for row in stream:
if header is None:
......@@ -169,7 +167,7 @@ def validate(schema_code, source: ValidataSource):
""" Validate source and display report """
try:
goodtables_report = ValidatorHelper.validate(schema_code, **source.get_goodtables_source())
goodtables_report = vu.validate(schema_code, **source.get_tabulator_params())
except tabulator.exceptions.FormatError:
flash_error('Erreur : format de fichier non supporté')
return redirect(url_for('scdl_validator', val_code=schema_code))
......@@ -238,11 +236,11 @@ def scdl_validator(val_code):
# Process URL
else:
url = request.args.get('url')
if url is None or url == '':
url_param = request.args.get('url')
if url_param is None or url_param == '':
flash_error("Vous n'avez pas indiqué d'url à valider")
return redirect(url_for('scdl_validator', val_code=val_code))
return validate(val_code, ValidataSource(url, url, 'url'))
return validate(val_code, ValidataSource('url', url_param, url_param))
else: # POST
input_param = request.form.get('input')
......@@ -257,6 +255,6 @@ def scdl_validator(val_code):
flash_warning("Vous n'avez pas indiqué de fichier à valider")
return redirect(url_for('scdl_validator', val_code=val_code))
return validate(val_code, ValidataSource(bytes_data(f), f.filename, 'file'))
return validate(val_code, ValidataSource('file', f.filename, bytes_data(f)))
return 'Bizarre, vous avez dit bizarre ?'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment