Make uploaded file in memory works

validata_ui_next/util.py

@@ -2,7 +2,13 @@
 """
     Util functions
 """
+import json
+from collections import namedtuple
+from io import BytesIO
+
 from flask import flash
+from tabulator import helpers
+from tabulator.loader import Loader
 
 
 def flash_error(msg):
@@ -23,3 +29,48 @@ def flash_success(msg):
 def flash_info(msg):
     """ Flash bootstrap info message """
     flash(msg, 'info')
+
+
+class ValidataSource():
+    """ Handy class to handle different sort of data source """
+
+    def __init__(self, data, name, type):
+        """ Initialization """
+        self.data = data
+        self.name = name
+        self.type = type
+        self.scheme = None
+        self.format = None
+        self.custom_loaders = {}
+
+    def get_goodtables_source(self):
+        """ Creates source ready to be ingested by tabulator """
+
+        self.scheme, self.format = helpers.detect_scheme_and_format(self.name)
+
+        # In case of uploaded file (we work with bytes string)
+        if self.type == 'file':
+
+            # CSV: converts to string
+            if self.format == 'csv':
+                self.scheme = 'text'
+                encoding = helpers.detect_encoding(self.data)
+                self.data = self.data.decode(encoding)
+
+            # Else use custom BytesLoader
+            else:
+                self.scheme = 'custom'
+                self.custom_loaders = {'custom': BytesLoader}
+
+        return {'source': self.data, 'format': self.format, 'scheme': self.scheme, "custom_loaders": self.custom_loaders}
+
+
+class BytesLoader(Loader):
+    """ Custom loader for bytes string """
+    options = []
+
+    def __init__(self, bytes_sample_size, **options):
+        pass
+
+    def load(self, source, mode='t', encoding=None):
+        return BytesIO(source)

validata_ui_next/validate_helper.py

@@ -84,7 +84,7 @@ class ValidatorHelper:
         return [cls.schema_info(code) for code in sorted(cls.schema_dict.keys())]
 
     @classmethod
-    def validate(cls, schema_code, source, res_type):
+    def validate(cls, schema_code, **args):
         """ Validate source against schema using custom-checks """
 
         # Gets schema info
@@ -103,8 +103,9 @@ class ValidatorHelper:
         inspector = Inspector(checks=checks, row_limit=VALIDATA_MAX_ROWS)
 
         return validate(
-            source=source,
+            source=args['source'],
             inspector=inspector,
             schema=sc_info['schema'],
             pre_checks_conf=pre_checks_conf,
+            **{k: v for k, v in args.items() if k != 'source'}
         )

validata_ui_next/views.py

@@ -8,15 +8,18 @@ import os
 from collections import OrderedDict
 from pathlib import Path
 
+from validata_validate import csv_helpers
 from validata_ui_next import app
-from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning
+from validata_ui_next.util import flash_error, flash_info, flash_success, flash_warning, ValidataSource
 from validata_ui_next.validate_helper import ValidatorHelper
 
 from flask import Flask, jsonify, redirect, render_template, request, url_for
 import tabulator
 
+from io import BytesIO
 
-def extract_source_data(source, preview_rows_nb=5):
+
+def extract_source_data(source: ValidataSource, preview_rows_nb=5):
     """ Computes table preview """
 
     def stringify(val):
@@ -28,7 +31,12 @@ def extract_source_data(source, preview_rows_nb=5):
     header = None
     rows = []
     nb_rows = 0
-    with tabulator.Stream(source) as stream:
+
+    delimiter = None
+    if source.format == "csv":
+        delimiter = csv_helpers.detect_dialect(source.data, format=source.format,
+                                               scheme=source.scheme, custom_loaders=source.custom_loaders).delimiter
+    with tabulator.Stream(source.data, format=source.format, scheme=source.scheme, custom_loaders=source.custom_loaders, delimiter=delimiter) as stream:
         for row in stream:
             if header is None:
                 header = row
@@ -179,27 +187,35 @@ def create_validata_report(goodtables_report):
     return report
 
 
-def validate(schema_code, source, source_type):
+def validate(schema_code, source: ValidataSource):
     """ Validate source and display report """
 
-    goodtables_report = ValidatorHelper.validate(schema_code, source, source_type)
+    goodtables_report = ValidatorHelper.validate(schema_code, **source.get_goodtables_source())
+
+    source_data = extract_source_data(source)
 
     validata_report = create_validata_report(goodtables_report)
 
     # return jsonify(validata_report)
 
-    source_data = extract_source_data(source)
-
     # Complete report
     val_info = ValidatorHelper.schema_info(schema_code)
     return render_template('validation_report.html', title='Rapport de validation',
                            val_info=ValidatorHelper.schema_info(schema_code), report=validata_report,
-                           source=source, source_type=source_type, source_data=source_data,
+                           source=source, source_type=source.type, source_data=source_data,
                            report_str=json.dumps(validata_report, sort_keys=True, indent=2),
                            breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'},
                                         {'url': url_for('scdl_validator', val_code=schema_code), 'title': val_info['title']}])
 
 
+def bytes_data(f):
+    """ Gets bytes data from Werkzeug FileStorage instance """
+    iob = BytesIO()
+    f.save(iob)
+    iob.seek(0)
+    return iob.getvalue()
+
+
 # Routes
 
 
@@ -248,7 +264,7 @@ def scdl_validator(val_code):
             if url is None or url == '':
                 flash_error("Vous n'avez pas indiqué d'url à valider")
                 return redirect(url_for('scdl_validator', val_code=val_code))
-            return validate(val_code, url, 'url')
+            return validate(val_code, ValidataSource(url, url, 'url'))
 
     else:  # POST
         input_param = request.form.get('input')
@@ -262,8 +278,7 @@ def scdl_validator(val_code):
             if f is None:
                 flash_warning("Vous n'avez pas indiqué de fichier à valider")
                 return redirect(url_for('scdl_validator', val_code=val_code))
-            fpath = os.path.join('/tmp', f.filename)
-            f.save(fpath)
-            return validate(val_code, fpath, 'file')
+
+            return validate(val_code, ValidataSource(bytes_data(f), f.filename, 'file'))
 
         return 'Bizarre, vous avez dit bizarre ?'