Commit 524bfe39 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Make uploaded file in memory works

parent 2f409dd6
...@@ -2,7 +2,13 @@ ...@@ -2,7 +2,13 @@
""" """
Util functions Util functions
""" """
import json
from collections import namedtuple
from io import BytesIO
from flask import flash from flask import flash
from tabulator import helpers
from tabulator.loader import Loader
def flash_error(msg): def flash_error(msg):
...@@ -23,3 +29,48 @@ def flash_success(msg): ...@@ -23,3 +29,48 @@ def flash_success(msg):
def flash_info(msg): def flash_info(msg):
""" Flash bootstrap info message """ """ Flash bootstrap info message """
flash(msg, 'info') flash(msg, 'info')
class ValidataSource():
""" Handy class to handle different sort of data source """
def __init__(self, data, name, type):
""" Initialization """
self.data = data
self.name = name
self.type = type
self.scheme = None
self.format = None
self.custom_loaders = {}
def get_goodtables_source(self):
""" Creates source ready to be ingested by tabulator """
self.scheme, self.format = helpers.detect_scheme_and_format(self.name)
# In case of uploaded file (we work with bytes string)
if self.type == 'file':
# CSV: converts to string
if self.format == 'csv':
self.scheme = 'text'
encoding = helpers.detect_encoding(self.data)
self.data = self.data.decode(encoding)
# Else use custom BytesLoader
else:
self.scheme = 'custom'
self.custom_loaders = {'custom': BytesLoader}
return {'source': self.data, 'format': self.format, 'scheme': self.scheme, "custom_loaders": self.custom_loaders}
class BytesLoader(Loader):
""" Custom loader for bytes string """
options = []
def __init__(self, bytes_sample_size, **options):
pass
def load(self, source, mode='t', encoding=None):
return BytesIO(source)
...@@ -84,7 +84,7 @@ class ValidatorHelper: ...@@ -84,7 +84,7 @@ class ValidatorHelper:
return [cls.schema_info(code) for code in sorted(cls.schema_dict.keys())] return [cls.schema_info(code) for code in sorted(cls.schema_dict.keys())]
@classmethod @classmethod
def validate(cls, schema_code, source, res_type): def validate(cls, schema_code, **args):
""" Validate source against schema using custom-checks """ """ Validate source against schema using custom-checks """
# Gets schema info # Gets schema info
...@@ -103,8 +103,9 @@ class ValidatorHelper: ...@@ -103,8 +103,9 @@ class ValidatorHelper:
inspector = Inspector(checks=checks, row_limit=VALIDATA_MAX_ROWS) inspector = Inspector(checks=checks, row_limit=VALIDATA_MAX_ROWS)
return validate( return validate(
source=source, source=args['source'],
inspector=inspector, inspector=inspector,
schema=sc_info['schema'], schema=sc_info['schema'],
pre_checks_conf=pre_checks_conf, pre_checks_conf=pre_checks_conf,
**{k: v for k, v in args.items() if k != 'source'}
) )
...@@ -8,15 +8,18 @@ import os ...@@ -8,15 +8,18 @@ import os
from collections import OrderedDict from collections import OrderedDict
from pathlib import Path from pathlib import Path
from validata_validate import csv_helpers
from validata_ui_next import app from validata_ui_next import app
from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning, ValidataSource
from validata_ui_next.validate_helper import ValidatorHelper from validata_ui_next.validate_helper import ValidatorHelper
from flask import Flask, jsonify, redirect, render_template, request, url_for from flask import Flask, jsonify, redirect, render_template, request, url_for
import tabulator import tabulator
from io import BytesIO
def extract_source_data(source, preview_rows_nb=5):
def extract_source_data(source: ValidataSource, preview_rows_nb=5):
""" Computes table preview """ """ Computes table preview """
def stringify(val): def stringify(val):
...@@ -28,7 +31,12 @@ def extract_source_data(source, preview_rows_nb=5): ...@@ -28,7 +31,12 @@ def extract_source_data(source, preview_rows_nb=5):
header = None header = None
rows = [] rows = []
nb_rows = 0 nb_rows = 0
with tabulator.Stream(source) as stream:
delimiter = None
if source.format == "csv":
delimiter = csv_helpers.detect_dialect(source.data, format=source.format,
scheme=source.scheme, custom_loaders=source.custom_loaders).delimiter
with tabulator.Stream(source.data, format=source.format, scheme=source.scheme, custom_loaders=source.custom_loaders, delimiter=delimiter) as stream:
for row in stream: for row in stream:
if header is None: if header is None:
header = row header = row
...@@ -179,27 +187,35 @@ def create_validata_report(goodtables_report): ...@@ -179,27 +187,35 @@ def create_validata_report(goodtables_report):
return report return report
def validate(schema_code, source, source_type): def validate(schema_code, source: ValidataSource):
""" Validate source and display report """ """ Validate source and display report """
goodtables_report = ValidatorHelper.validate(schema_code, source, source_type) goodtables_report = ValidatorHelper.validate(schema_code, **source.get_goodtables_source())
source_data = extract_source_data(source)
validata_report = create_validata_report(goodtables_report) validata_report = create_validata_report(goodtables_report)
# return jsonify(validata_report) # return jsonify(validata_report)
source_data = extract_source_data(source)
# Complete report # Complete report
val_info = ValidatorHelper.schema_info(schema_code) val_info = ValidatorHelper.schema_info(schema_code)
return render_template('validation_report.html', title='Rapport de validation', return render_template('validation_report.html', title='Rapport de validation',
val_info=ValidatorHelper.schema_info(schema_code), report=validata_report, val_info=ValidatorHelper.schema_info(schema_code), report=validata_report,
source=source, source_type=source_type, source_data=source_data, source=source, source_type=source.type, source_data=source_data,
report_str=json.dumps(validata_report, sort_keys=True, indent=2), report_str=json.dumps(validata_report, sort_keys=True, indent=2),
breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
{'url': url_for('scdl_validator', val_code=schema_code), 'title': val_info['title']}]) {'url': url_for('scdl_validator', val_code=schema_code), 'title': val_info['title']}])
def bytes_data(f):
""" Gets bytes data from Werkzeug FileStorage instance """
iob = BytesIO()
f.save(iob)
iob.seek(0)
return iob.getvalue()
# Routes # Routes
...@@ -248,7 +264,7 @@ def scdl_validator(val_code): ...@@ -248,7 +264,7 @@ def scdl_validator(val_code):
if url is None or url == '': if url is None or url == '':
flash_error("Vous n'avez pas indiqué d'url à valider") flash_error("Vous n'avez pas indiqué d'url à valider")
return redirect(url_for('scdl_validator', val_code=val_code)) return redirect(url_for('scdl_validator', val_code=val_code))
return validate(val_code, url, 'url') return validate(val_code, ValidataSource(url, url, 'url'))
else: # POST else: # POST
input_param = request.form.get('input') input_param = request.form.get('input')
...@@ -262,8 +278,7 @@ def scdl_validator(val_code): ...@@ -262,8 +278,7 @@ def scdl_validator(val_code):
if f is None: if f is None:
flash_warning("Vous n'avez pas indiqué de fichier à valider") flash_warning("Vous n'avez pas indiqué de fichier à valider")
return redirect(url_for('scdl_validator', val_code=val_code)) return redirect(url_for('scdl_validator', val_code=val_code))
fpath = os.path.join('/tmp', f.filename)
f.save(fpath) return validate(val_code, ValidataSource(bytes_data(f), f.filename, 'file'))
return validate(val_code, fpath, 'file')
return 'Bizarre, vous avez dit bizarre ?' return 'Bizarre, vous avez dit bizarre ?'
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment