Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Validata
Validata UI
Commits
524bfe39
Commit
524bfe39
authored
Oct 02, 2018
by
Pierre Dittgen
Browse files
Make uploaded file in memory works
parent
2f409dd6
Changes
3
Hide whitespace changes
Inline
Side-by-side
validata_ui_next/util.py
View file @
524bfe39
...
...
@@ -2,7 +2,13 @@
"""
Util functions
"""
import
json
from
collections
import
namedtuple
from
io
import
BytesIO
from
flask
import
flash
from
tabulator
import
helpers
from
tabulator.loader
import
Loader
def
flash_error
(
msg
):
...
...
@@ -23,3 +29,48 @@ def flash_success(msg):
def
flash_info
(
msg
):
""" Flash bootstrap info message """
flash
(
msg
,
'info'
)
class
ValidataSource
():
""" Handy class to handle different sort of data source """
def
__init__
(
self
,
data
,
name
,
type
):
""" Initialization """
self
.
data
=
data
self
.
name
=
name
self
.
type
=
type
self
.
scheme
=
None
self
.
format
=
None
self
.
custom_loaders
=
{}
def
get_goodtables_source
(
self
):
""" Creates source ready to be ingested by tabulator """
self
.
scheme
,
self
.
format
=
helpers
.
detect_scheme_and_format
(
self
.
name
)
# In case of uploaded file (we work with bytes string)
if
self
.
type
==
'file'
:
# CSV: converts to string
if
self
.
format
==
'csv'
:
self
.
scheme
=
'text'
encoding
=
helpers
.
detect_encoding
(
self
.
data
)
self
.
data
=
self
.
data
.
decode
(
encoding
)
# Else use custom BytesLoader
else
:
self
.
scheme
=
'custom'
self
.
custom_loaders
=
{
'custom'
:
BytesLoader
}
return
{
'source'
:
self
.
data
,
'format'
:
self
.
format
,
'scheme'
:
self
.
scheme
,
"custom_loaders"
:
self
.
custom_loaders
}
class
BytesLoader
(
Loader
):
""" Custom loader for bytes string """
options
=
[]
def
__init__
(
self
,
bytes_sample_size
,
**
options
):
pass
def
load
(
self
,
source
,
mode
=
't'
,
encoding
=
None
):
return
BytesIO
(
source
)
validata_ui_next/validate_helper.py
View file @
524bfe39
...
...
@@ -84,7 +84,7 @@ class ValidatorHelper:
return
[
cls
.
schema_info
(
code
)
for
code
in
sorted
(
cls
.
schema_dict
.
keys
())]
@
classmethod
def
validate
(
cls
,
schema_code
,
source
,
res_type
):
def
validate
(
cls
,
schema_code
,
**
args
):
""" Validate source against schema using custom-checks """
# Gets schema info
...
...
@@ -103,8 +103,9 @@ class ValidatorHelper:
inspector
=
Inspector
(
checks
=
checks
,
row_limit
=
VALIDATA_MAX_ROWS
)
return
validate
(
source
=
source
,
source
=
args
[
'
source
'
]
,
inspector
=
inspector
,
schema
=
sc_info
[
'schema'
],
pre_checks_conf
=
pre_checks_conf
,
**
{
k
:
v
for
k
,
v
in
args
.
items
()
if
k
!=
'source'
}
)
validata_ui_next/views.py
View file @
524bfe39
...
...
@@ -8,15 +8,18 @@ import os
from
collections
import
OrderedDict
from
pathlib
import
Path
from
validata_validate
import
csv_helpers
from
validata_ui_next
import
app
from
validata_ui_next.util
import
flash_error
,
flash_info
,
flash_success
,
flash_warning
from
validata_ui_next.util
import
flash_error
,
flash_info
,
flash_success
,
flash_warning
,
ValidataSource
from
validata_ui_next.validate_helper
import
ValidatorHelper
from
flask
import
Flask
,
jsonify
,
redirect
,
render_template
,
request
,
url_for
import
tabulator
from
io
import
BytesIO
def
extract_source_data
(
source
,
preview_rows_nb
=
5
):
def
extract_source_data
(
source
:
ValidataSource
,
preview_rows_nb
=
5
):
""" Computes table preview """
def
stringify
(
val
):
...
...
@@ -28,7 +31,12 @@ def extract_source_data(source, preview_rows_nb=5):
header
=
None
rows
=
[]
nb_rows
=
0
with
tabulator
.
Stream
(
source
)
as
stream
:
delimiter
=
None
if
source
.
format
==
"csv"
:
delimiter
=
csv_helpers
.
detect_dialect
(
source
.
data
,
format
=
source
.
format
,
scheme
=
source
.
scheme
,
custom_loaders
=
source
.
custom_loaders
).
delimiter
with
tabulator
.
Stream
(
source
.
data
,
format
=
source
.
format
,
scheme
=
source
.
scheme
,
custom_loaders
=
source
.
custom_loaders
,
delimiter
=
delimiter
)
as
stream
:
for
row
in
stream
:
if
header
is
None
:
header
=
row
...
...
@@ -179,27 +187,35 @@ def create_validata_report(goodtables_report):
return
report
def
validate
(
schema_code
,
source
,
source_typ
e
):
def
validate
(
schema_code
,
source
:
ValidataSourc
e
):
""" Validate source and display report """
goodtables_report
=
ValidatorHelper
.
validate
(
schema_code
,
source
,
source_type
)
goodtables_report
=
ValidatorHelper
.
validate
(
schema_code
,
**
source
.
get_goodtables_source
())
source_data
=
extract_source_data
(
source
)
validata_report
=
create_validata_report
(
goodtables_report
)
# return jsonify(validata_report)
source_data
=
extract_source_data
(
source
)
# Complete report
val_info
=
ValidatorHelper
.
schema_info
(
schema_code
)
return
render_template
(
'validation_report.html'
,
title
=
'Rapport de validation'
,
val_info
=
ValidatorHelper
.
schema_info
(
schema_code
),
report
=
validata_report
,
source
=
source
,
source_type
=
source
_
type
,
source_data
=
source_data
,
source
=
source
,
source_type
=
source
.
type
,
source_data
=
source_data
,
report_str
=
json
.
dumps
(
validata_report
,
sort_keys
=
True
,
indent
=
2
),
breadcrumbs
=
[{
'url'
:
url_for
(
'home'
),
'title'
:
'Accueil'
},
{
'url'
:
url_for
(
'scdl_validator'
,
val_code
=
schema_code
),
'title'
:
val_info
[
'title'
]}])
def
bytes_data
(
f
):
""" Gets bytes data from Werkzeug FileStorage instance """
iob
=
BytesIO
()
f
.
save
(
iob
)
iob
.
seek
(
0
)
return
iob
.
getvalue
()
# Routes
...
...
@@ -248,7 +264,7 @@ def scdl_validator(val_code):
if
url
is
None
or
url
==
''
:
flash_error
(
"Vous n'avez pas indiqué d'url à valider"
)
return
redirect
(
url_for
(
'scdl_validator'
,
val_code
=
val_code
))
return
validate
(
val_code
,
url
,
'url'
)
return
validate
(
val_code
,
ValidataSource
(
url
,
url
,
'url'
)
)
else
:
# POST
input_param
=
request
.
form
.
get
(
'input'
)
...
...
@@ -262,8 +278,7 @@ def scdl_validator(val_code):
if
f
is
None
:
flash_warning
(
"Vous n'avez pas indiqué de fichier à valider"
)
return
redirect
(
url_for
(
'scdl_validator'
,
val_code
=
val_code
))
fpath
=
os
.
path
.
join
(
'/tmp'
,
f
.
filename
)
f
.
save
(
fpath
)
return
validate
(
val_code
,
fpath
,
'file'
)
return
validate
(
val_code
,
ValidataSource
(
bytes_data
(
f
),
f
.
filename
,
'file'
))
return
'Bizarre, vous avez dit bizarre ?'
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment