Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
Validata
validata-ui
Commits
524bfe39
Commit
524bfe39
authored
Oct 02, 2018
by
Pierre Dittgen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Make uploaded file in memory works
parent
2f409dd6
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
81 additions
and
14 deletions
+81
-14
validata_ui_next/util.py
validata_ui_next/util.py
+51
-0
validata_ui_next/validate_helper.py
validata_ui_next/validate_helper.py
+3
-2
validata_ui_next/views.py
validata_ui_next/views.py
+27
-12
No files found.
validata_ui_next/util.py
View file @
524bfe39
...
...
@@ -2,7 +2,13 @@
"""
Util functions
"""
import
json
from
collections
import
namedtuple
from
io
import
BytesIO
from
flask
import
flash
from
tabulator
import
helpers
from
tabulator.loader
import
Loader
def
flash_error
(
msg
):
...
...
@@ -23,3 +29,48 @@ def flash_success(msg):
def
flash_info
(
msg
):
""" Flash bootstrap info message """
flash
(
msg
,
'info'
)
class
ValidataSource
():
""" Handy class to handle different sort of data source """
def
__init__
(
self
,
data
,
name
,
type
):
""" Initialization """
self
.
data
=
data
self
.
name
=
name
self
.
type
=
type
self
.
scheme
=
None
self
.
format
=
None
self
.
custom_loaders
=
{}
def
get_goodtables_source
(
self
):
""" Creates source ready to be ingested by tabulator """
self
.
scheme
,
self
.
format
=
helpers
.
detect_scheme_and_format
(
self
.
name
)
# In case of uploaded file (we work with bytes string)
if
self
.
type
==
'file'
:
# CSV: converts to string
if
self
.
format
==
'csv'
:
self
.
scheme
=
'text'
encoding
=
helpers
.
detect_encoding
(
self
.
data
)
self
.
data
=
self
.
data
.
decode
(
encoding
)
# Else use custom BytesLoader
else
:
self
.
scheme
=
'custom'
self
.
custom_loaders
=
{
'custom'
:
BytesLoader
}
return
{
'source'
:
self
.
data
,
'format'
:
self
.
format
,
'scheme'
:
self
.
scheme
,
"custom_loaders"
:
self
.
custom_loaders
}
class
BytesLoader
(
Loader
):
""" Custom loader for bytes string """
options
=
[]
def
__init__
(
self
,
bytes_sample_size
,
**
options
):
pass
def
load
(
self
,
source
,
mode
=
't'
,
encoding
=
None
):
return
BytesIO
(
source
)
validata_ui_next/validate_helper.py
View file @
524bfe39
...
...
@@ -84,7 +84,7 @@ class ValidatorHelper:
return
[
cls
.
schema_info
(
code
)
for
code
in
sorted
(
cls
.
schema_dict
.
keys
())]
@
classmethod
def
validate
(
cls
,
schema_code
,
source
,
res_type
):
def
validate
(
cls
,
schema_code
,
**
args
):
""" Validate source against schema using custom-checks """
# Gets schema info
...
...
@@ -103,8 +103,9 @@ class ValidatorHelper:
inspector
=
Inspector
(
checks
=
checks
,
row_limit
=
VALIDATA_MAX_ROWS
)
return
validate
(
source
=
source
,
source
=
args
[
'
source
'
]
,
inspector
=
inspector
,
schema
=
sc_info
[
'schema'
],
pre_checks_conf
=
pre_checks_conf
,
**
{
k
:
v
for
k
,
v
in
args
.
items
()
if
k
!=
'source'
}
)
validata_ui_next/views.py
View file @
524bfe39
...
...
@@ -8,15 +8,18 @@ import os
from
collections
import
OrderedDict
from
pathlib
import
Path
from
validata_validate
import
csv_helpers
from
validata_ui_next
import
app
from
validata_ui_next.util
import
flash_error
,
flash_info
,
flash_success
,
flash_warning
from
validata_ui_next.util
import
flash_error
,
flash_info
,
flash_success
,
flash_warning
,
ValidataSource
from
validata_ui_next.validate_helper
import
ValidatorHelper
from
flask
import
Flask
,
jsonify
,
redirect
,
render_template
,
request
,
url_for
import
tabulator
from
io
import
BytesIO
def
extract_source_data
(
source
,
preview_rows_nb
=
5
):
def
extract_source_data
(
source
:
ValidataSource
,
preview_rows_nb
=
5
):
""" Computes table preview """
def
stringify
(
val
):
...
...
@@ -28,7 +31,12 @@ def extract_source_data(source, preview_rows_nb=5):
header
=
None
rows
=
[]
nb_rows
=
0
with
tabulator
.
Stream
(
source
)
as
stream
:
delimiter
=
None
if
source
.
format
==
"csv"
:
delimiter
=
csv_helpers
.
detect_dialect
(
source
.
data
,
format
=
source
.
format
,
scheme
=
source
.
scheme
,
custom_loaders
=
source
.
custom_loaders
).
delimiter
with
tabulator
.
Stream
(
source
.
data
,
format
=
source
.
format
,
scheme
=
source
.
scheme
,
custom_loaders
=
source
.
custom_loaders
,
delimiter
=
delimiter
)
as
stream
:
for
row
in
stream
:
if
header
is
None
:
header
=
row
...
...
@@ -179,27 +187,35 @@ def create_validata_report(goodtables_report):
return
report
def
validate
(
schema_code
,
source
,
source_typ
e
):
def
validate
(
schema_code
,
source
:
ValidataSourc
e
):
""" Validate source and display report """
goodtables_report
=
ValidatorHelper
.
validate
(
schema_code
,
source
,
source_type
)
goodtables_report
=
ValidatorHelper
.
validate
(
schema_code
,
**
source
.
get_goodtables_source
())
source_data
=
extract_source_data
(
source
)
validata_report
=
create_validata_report
(
goodtables_report
)
# return jsonify(validata_report)
source_data
=
extract_source_data
(
source
)
# Complete report
val_info
=
ValidatorHelper
.
schema_info
(
schema_code
)
return
render_template
(
'validation_report.html'
,
title
=
'Rapport de validation'
,
val_info
=
ValidatorHelper
.
schema_info
(
schema_code
),
report
=
validata_report
,
source
=
source
,
source_type
=
source
_
type
,
source_data
=
source_data
,
source
=
source
,
source_type
=
source
.
type
,
source_data
=
source_data
,
report_str
=
json
.
dumps
(
validata_report
,
sort_keys
=
True
,
indent
=
2
),
breadcrumbs
=
[{
'url'
:
url_for
(
'home'
),
'title'
:
'Accueil'
},
{
'url'
:
url_for
(
'scdl_validator'
,
val_code
=
schema_code
),
'title'
:
val_info
[
'title'
]}])
def
bytes_data
(
f
):
""" Gets bytes data from Werkzeug FileStorage instance """
iob
=
BytesIO
()
f
.
save
(
iob
)
iob
.
seek
(
0
)
return
iob
.
getvalue
()
# Routes
...
...
@@ -248,7 +264,7 @@ def scdl_validator(val_code):
if
url
is
None
or
url
==
''
:
flash_error
(
"Vous n'avez pas indiqué d'url à valider"
)
return
redirect
(
url_for
(
'scdl_validator'
,
val_code
=
val_code
))
return
validate
(
val_code
,
url
,
'url'
)
return
validate
(
val_code
,
ValidataSource
(
url
,
url
,
'url'
)
)
else
:
# POST
input_param
=
request
.
form
.
get
(
'input'
)
...
...
@@ -262,8 +278,7 @@ def scdl_validator(val_code):
if
f
is
None
:
flash_warning
(
"Vous n'avez pas indiqué de fichier à valider"
)
return
redirect
(
url_for
(
'scdl_validator'
,
val_code
=
val_code
))
fpath
=
os
.
path
.
join
(
'/tmp'
,
f
.
filename
)
f
.
save
(
fpath
)
return
validate
(
val_code
,
fpath
,
'file'
)
return
validate
(
val_code
,
ValidataSource
(
bytes_data
(
f
),
f
.
filename
,
'file'
))
return
'Bizarre, vous avez dit bizarre ?'
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment