Commit 524bfe39 authored by Pierre Dittgen's avatar Pierre Dittgen

Make uploaded file in memory works

parent 2f409dd6
......@@ -2,7 +2,13 @@
"""
Util functions
"""
import json
from collections import namedtuple
from io import BytesIO
from flask import flash
from tabulator import helpers
from tabulator.loader import Loader
def flash_error(msg):
......@@ -23,3 +29,48 @@ def flash_success(msg):
def flash_info(msg):
""" Flash bootstrap info message """
flash(msg, 'info')
class ValidataSource():
""" Handy class to handle different sort of data source """
def __init__(self, data, name, type):
""" Initialization """
self.data = data
self.name = name
self.type = type
self.scheme = None
self.format = None
self.custom_loaders = {}
def get_goodtables_source(self):
""" Creates source ready to be ingested by tabulator """
self.scheme, self.format = helpers.detect_scheme_and_format(self.name)
# In case of uploaded file (we work with bytes string)
if self.type == 'file':
# CSV: converts to string
if self.format == 'csv':
self.scheme = 'text'
encoding = helpers.detect_encoding(self.data)
self.data = self.data.decode(encoding)
# Else use custom BytesLoader
else:
self.scheme = 'custom'
self.custom_loaders = {'custom': BytesLoader}
return {'source': self.data, 'format': self.format, 'scheme': self.scheme, "custom_loaders": self.custom_loaders}
class BytesLoader(Loader):
""" Custom loader for bytes string """
options = []
def __init__(self, bytes_sample_size, **options):
pass
def load(self, source, mode='t', encoding=None):
return BytesIO(source)
......@@ -84,7 +84,7 @@ class ValidatorHelper:
return [cls.schema_info(code) for code in sorted(cls.schema_dict.keys())]
@classmethod
def validate(cls, schema_code, source, res_type):
def validate(cls, schema_code, **args):
""" Validate source against schema using custom-checks """
# Gets schema info
......@@ -103,8 +103,9 @@ class ValidatorHelper:
inspector = Inspector(checks=checks, row_limit=VALIDATA_MAX_ROWS)
return validate(
source=source,
source=args['source'],
inspector=inspector,
schema=sc_info['schema'],
pre_checks_conf=pre_checks_conf,
**{k: v for k, v in args.items() if k != 'source'}
)
......@@ -8,15 +8,18 @@ import os
from collections import OrderedDict
from pathlib import Path
from validata_validate import csv_helpers
from validata_ui_next import app
from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning
from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning, ValidataSource
from validata_ui_next.validate_helper import ValidatorHelper
from flask import Flask, jsonify, redirect, render_template, request, url_for
import tabulator
from io import BytesIO
def extract_source_data(source, preview_rows_nb=5):
def extract_source_data(source: ValidataSource, preview_rows_nb=5):
""" Computes table preview """
def stringify(val):
......@@ -28,7 +31,12 @@ def extract_source_data(source, preview_rows_nb=5):
header = None
rows = []
nb_rows = 0
with tabulator.Stream(source) as stream:
delimiter = None
if source.format == "csv":
delimiter = csv_helpers.detect_dialect(source.data, format=source.format,
scheme=source.scheme, custom_loaders=source.custom_loaders).delimiter
with tabulator.Stream(source.data, format=source.format, scheme=source.scheme, custom_loaders=source.custom_loaders, delimiter=delimiter) as stream:
for row in stream:
if header is None:
header = row
......@@ -179,27 +187,35 @@ def create_validata_report(goodtables_report):
return report
def validate(schema_code, source, source_type):
def validate(schema_code, source: ValidataSource):
""" Validate source and display report """
goodtables_report = ValidatorHelper.validate(schema_code, source, source_type)
goodtables_report = ValidatorHelper.validate(schema_code, **source.get_goodtables_source())
source_data = extract_source_data(source)
validata_report = create_validata_report(goodtables_report)
# return jsonify(validata_report)
source_data = extract_source_data(source)
# Complete report
val_info = ValidatorHelper.schema_info(schema_code)
return render_template('validation_report.html', title='Rapport de validation',
val_info=ValidatorHelper.schema_info(schema_code), report=validata_report,
source=source, source_type=source_type, source_data=source_data,
source=source, source_type=source.type, source_data=source_data,
report_str=json.dumps(validata_report, sort_keys=True, indent=2),
breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
{'url': url_for('scdl_validator', val_code=schema_code), 'title': val_info['title']}])
def bytes_data(f):
""" Gets bytes data from Werkzeug FileStorage instance """
iob = BytesIO()
f.save(iob)
iob.seek(0)
return iob.getvalue()
# Routes
......@@ -248,7 +264,7 @@ def scdl_validator(val_code):
if url is None or url == '':
flash_error("Vous n'avez pas indiqué d'url à valider")
return redirect(url_for('scdl_validator', val_code=val_code))
return validate(val_code, url, 'url')
return validate(val_code, ValidataSource(url, url, 'url'))
else: # POST
input_param = request.form.get('input')
......@@ -262,8 +278,7 @@ def scdl_validator(val_code):
if f is None:
flash_warning("Vous n'avez pas indiqué de fichier à valider")
return redirect(url_for('scdl_validator', val_code=val_code))
fpath = os.path.join('/tmp', f.filename)
f.save(fpath)
return validate(val_code, fpath, 'file')
return validate(val_code, ValidataSource(bytes_data(f), f.filename, 'file'))
return 'Bizarre, vous avez dit bizarre ?'
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment