Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
Validata
validata-ui
Commits
84c0a3a8
Commit
84c0a3a8
authored
Jun 03, 2019
by
Pierre Dittgen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
wip
parent
af1a8f31
Changes
10
Show whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
335 additions
and
79 deletions
+335
-79
.env.example
.env.example
+3
-0
CHANGELOG.md
CHANGELOG.md
+7
-0
validata_ui/__init__.py
validata_ui/__init__.py
+6
-4
validata_ui/config.py
validata_ui/config.py
+3
-0
validata_ui/templates/custom_validation_form.html
validata_ui/templates/custom_validation_form.html
+146
-0
validata_ui/templates/home.html
validata_ui/templates/home.html
+38
-29
validata_ui/templates/scdl_validation_form.html
validata_ui/templates/scdl_validation_form.html
+0
-0
validata_ui/validata_util.py
validata_ui/validata_util.py
+12
-7
validata_ui/validate_helper.py
validata_ui/validate_helper.py
+3
-6
validata_ui/views.py
validata_ui/views.py
+117
-33
No files found.
.env.example
View file @
84c0a3a8
...
...
@@ -6,3 +6,6 @@ SECRET_KEY="dev"
# Comment the two following lines to disable "badge" generation.
BADGE_CONFIG_URL="https://git.opendatafrance.net/validata/validata-badge/raw/master/badge_conf.toml"
SHIELDS_IO_BASE_URL="https://img.shields.io/"
# Validata API endpoint
API_VALIDATE_ENDPOINT=http://127.0.0.1:5600/validate
\ No newline at end of file
CHANGELOG.md
View file @
84c0a3a8
## 0.1.0 -> next
Non-breaking changes:
-
New feature: validate a CSV against a schema URL
-
UI now depends on validata-api, no more on validata-core
## 0.0.1 -> 0.1.0
Non-breaking changes:
...
...
validata_ui/__init__.py
View file @
84c0a3a8
...
...
@@ -5,15 +5,17 @@ from urllib.parse import quote_plus
import
flask
import
jinja2
import
validata_core
#
import validata_core
# Let this import after app initialisation
from
.
import
config
from
.validate_helper
import
ValidatorHelper
#
from .validate_helper import ValidatorHelper
# Schemas settings
schemas_config
=
validata_core
.
get_schemas_config
()
ValidatorHelper
.
init
(
schemas_config
)
# schemas_config = validata_core.get_schemas_config()
# ValidatorHelper.init(schemas_config)
# TODO: load config.toml
# Flask things
app
=
flask
.
Flask
(
__name__
)
...
...
validata_ui/config.py
View file @
84c0a3a8
...
...
@@ -12,6 +12,9 @@ load_dotenv()
SECRET_KEY
=
os
.
environ
.
get
(
"SECRET_KEY"
)
or
None
API_VALIDATE_ENDPOINT
=
os
.
environ
.
get
(
"API_VALIDATE_ENDPOINT"
)
or
None
if
API_VALIDATE_ENDPOINT
is
None
:
log
.
error
(
"API_VALIDATE_ENDPOINT environment variable is not set, validation is not possible"
)
BADGE_CONFIG_URL
=
os
.
environ
.
get
(
"BADGE_CONFIG_URL"
)
or
None
BADGE_CONFIG
=
None
...
...
validata_ui/templates/custom_validation_form.html
0 → 100644
View file @
84c0a3a8
{% extends "base_template.html" %} {% block title %}{{ title }}{% endblock %} {%
block head %}
{{ super() }}
{% endblock %} {% block content %}
<h1
class=
"my-4"
>
Validateur personnalisé
</h1>
{#
{% set cols_my_classes = 'my-md-0 my-4' %}
<div
class=
"row"
>
<div
class=
"col-md-4 {{ cols_my_classes }}"
>
<div
class=
"card bg-faded"
>
<div
class=
"card-body"
>
<h5
class=
"card-title"
>
Schéma {{ val_info.code }}
{% if val_info.version %}
<span
class=
"badge badge-primary"
>
{{ val_info.version }}
</span>
{% else %}
<span
class=
"badge badge-primary"
title=
"Schéma en cours de développement"
>
dev
</span
>
{% endif %}
</h5>
<h6
class=
"card-subtitle mb-2 text-muted"
>
{{ val_info.description }}
</h6>
{% if val_info.author or val_info.contributor %}
<p
class=
"text"
>
{% if val_info.author %} Auteur : {{ val_info.author }}
{% endif %} {% if val_info.contributor %}
<br
/>
Contributeur(s) : {{ val_info.contributor }}
{% endif %}
</p>
{% endif %}
</div>
</div>
</div>
</div>
#}
<div
class=
"row"
>
<div
class=
"col-md-8 {{ cols_my_classes }}"
>
<!-- Tab validator -->
{#
<p
class=
"text"
>
Validez ici le fichier de votre choix
</p>
<ul
class=
"nav nav-tabs"
id=
"myTab"
role=
"tablist"
>
<li
class=
"nav-item"
>
<a
class=
"nav-link active"
id=
"file-tab"
data-toggle=
"tab"
href=
"#file"
role=
"tab"
aria-controls=
"file"
aria-selected=
"true"
>
Fichier
</a
>
</li>
<li
class=
"nav-item"
>
<a
class=
"nav-link"
id=
"url-tab"
data-toggle=
"tab"
href=
"#url"
role=
"tab"
aria-controls=
"url"
aria-selected=
"false"
>
URL
</a
>
</li>
</ul>
{% set padding_class = 'p-3' %}
<div
class=
"tab-content"
id=
"myTabContent"
>
<div
class=
"tab-pane fade show active {{ padding_class }}"
id=
"file"
role=
"tabpanel"
aria-labelledby=
"file-tab"
>
#}
<h2>
Valider un fichier
</h2>
<div>
<form
method=
"POST"
enctype=
"multipart/form-data"
>
<input
type=
"hidden"
name=
"input"
value=
"file"
/>
<div
class=
"form-group"
>
<label
for=
"schema"
>
Indiquez l'URL du schema de validation
</label>
<input
name=
"schema"
type=
"url"
class=
"form-control"
aria-describedby=
"urlHelp"
placeholder=
"https://..."
/>
<div
class=
"form-group"
>
<label
for=
"file"
>
Choisissez un fichier tabulaire à valider (.xlsx, .xls, .ods,
.csv, .tsv, etc.)
</label
>
<input
type=
"file"
class=
"form-control-file"
name=
"file"
id=
"file"
accept=
".csv, .xls, .xlsx, .ods"
/>
</div>
<button
type=
"submit"
class=
"btn btn-primary"
>
Valider
</button>
</form>
</div>
<h2>
Valider une URL
</h2>
<div
class=
"tab-pane {{ padding_class }}"
id=
"url"
role=
"tabpanel"
aria-labelledby=
"url-tab"
>
<form
method=
"GET"
>
<input
type=
"hidden"
name=
"input"
value=
"url"
/>
<div
class=
"form-group"
>
<label
for=
"schema"
>
Indiquez l'URL du schema de validation
</label>
<input
name=
"schema"
type=
"url"
class=
"form-control"
id=
"schema"
aria-describedby=
"urlHelp"
placeholder=
"https://..."
/>
<div
class=
"form-group"
>
<label
for=
"url"
>
Indiquez l'URL de la table à valider
</label>
<input
name=
"url"
type=
"url"
class=
"form-control"
id=
"url"
aria-describedby=
"urlHelp"
placeholder=
"https://..."
/>
</div>
<button
type=
"submit"
class=
"btn btn-primary"
>
Valider
</button>
</form>
</div>
</div>
</div>
</div>
{% endblock %} {% block footer %} {% endblock %}
validata_ui/templates/home.html
View file @
84c0a3a8
{% extends "base_template.html" %}
{% block title %}{{ title }}{% endblock %}
{% block head %}
{% extends "base_template.html" %} {% block title %}{{ title }}{% endblock %} {%
block head %}
{{ super() }}
{% endblock %}
{% block content %}
{% endblock %} {% block content %}
<h1
class=
"my-4"
>
Validez vos jeux de données
</h1>
<h2>
Validateurs
<span
abbr=
"Socle commun des données locales"
>
SCDL
</span></h2>
...
...
@@ -15,7 +13,11 @@
<div
class=
"card-body d-flex flex-column"
>
<h4
class=
"card-title"
>
{{ val.title }}
</h4>
<p
class=
"card-text"
>
{{ val.description }}
</p>
<a
href=
"{{ url_for('scdl_validator', val_code=val.code) }}"
class=
"btn btn-primary mt-auto"
>
Choisir
</a>
<a
href=
"{{ url_for('scdl_validator', val_code=val.code) }}"
class=
"btn btn-primary mt-auto"
>
Choisir
</a
>
</div>
</div>
</div>
...
...
@@ -41,4 +43,11 @@
{% endfor %}
</div>
<h2>
Validateur personnalisé
</h2>
<p>
<a
href=
"{{ url_for('custom_validator') }}"
>
Choisissez le schéma qui vous convient
</a
>
</p>
{% endblock %}
validata_ui/templates/validation_form.html
→
validata_ui/templates/
scdl_
validation_form.html
View file @
84c0a3a8
File moved
validata_ui/validata_util.py
View file @
84c0a3a8
...
...
@@ -3,10 +3,8 @@
import
logging
from
io
import
BytesIO
import
validata_core
from
validata_core.source_helpers
import
build_tabulator_params
from
.validate_helper
import
ValidatorHelper
log
=
logging
.
getLogger
(
__name__
)
...
...
@@ -14,9 +12,9 @@ log = logging.getLogger(__name__)
class
ValidataSource
():
""" Handy class to handle different sort of data source """
def
__init__
(
self
,
type
,
name
,
source
):
def
__init__
(
self
,
type
_
,
name
,
source
):
""" Initialization """
self
.
type
=
type
self
.
type
=
type
_
self
.
name
=
name
self
.
source
=
source
...
...
@@ -25,10 +23,17 @@ class ValidataSource():
self
.
format
=
info
.
get
(
'format'
)
self
.
scheme
=
info
.
get
(
'scheme'
)
def
get_tabulator_params
(
self
):
""" Creates source ready to be ingested by tabulator """
def
is_url
(
self
):
return
self
.
type
==
'url'
return
{
'source'
:
self
.
source
,
'format'
:
self
.
format
,
'scheme'
:
self
.
scheme
}
def
get_url
(
self
):
return
self
.
source
def
get_filename
(
self
):
return
self
.
name
def
get_filecontent
(
self
):
return
self
.
source
def
bytes_data
(
f
):
...
...
validata_ui/validate_helper.py
View file @
84c0a3a8
...
...
@@ -61,9 +61,6 @@ class ValidatorHelper:
return
[
cls
.
schema_info
(
code
)
for
code
in
sorted
(
cls
.
schema_dict
.
keys
())]
@
classmethod
def
validate
(
cls
,
schema_code
,
**
options
):
""" Try to retrieve cached schema from `schema_code`, otherwise pass `schema_code` it as-is """
schema
=
cls
.
schema
(
schema_code
)
if
schema
is
None
:
schema
=
schema_code
return
cls
.
validator
.
validate
(
schema
=
schema
,
**
options
)
def
validate
(
cls
,
schema_url
,
**
options
):
"""Validate"""
return
cls
.
validator
.
validate
(
schema
=
schema_url
,
**
options
)
validata_ui/views.py
View file @
84c0a3a8
...
...
@@ -2,26 +2,26 @@
Routes
"""
import
copy
import
io
import
itertools
import
json
import
logging
import
subprocess
import
tempfile
from
datetime
import
datetime
from
io
import
BytesIO
from
operator
import
itemgetter
from
pathlib
import
Path
from
urllib.parse
import
quote_plus
import
requests
from
backports.datetime_fromisoformat
import
MonkeyPatch
from
flask
import
make_response
,
redirect
,
render_template
,
request
,
url_for
import
tabulator
import
validata_core
from
commonmark
import
commonmark
from
flask
import
make_response
,
redirect
,
render_template
,
request
,
url_for
from
validata_core
import
compute_badge
,
csv_helpers
,
messages
from
validata_core.loaders
import
custom_loaders
import
tabulator
from
.
import
app
,
config
from
.ui_util
import
flash_error
,
flash_warning
from
.validata_util
import
ValidataSource
...
...
@@ -43,6 +43,12 @@ def extract_source_data(source: ValidataSource, preview_rows_nb=5):
rows
=
[]
nb_rows
=
0
# if source.scheme == 'file':
# source.scheme = 'bytes'
import
ipdb
ipdb
.
set_trace
()
options
=
{}
if
source
.
format
==
"csv"
:
options
[
'delimiter'
]
=
csv_helpers
.
detect_dialect
(
source
.
source
,
format
=
source
.
format
,
scheme
=
source
.
scheme
,
...
...
@@ -98,7 +104,7 @@ def improve_errors(errors):
return
list
(
map
(
improve_err
,
errors
))
def
create_validata_ui_report
(
validata_core_report
,
schema
):
def
create_validata_ui_report
(
validata_core_report
,
schema
_dict
):
""" Creates an error report easier to handle and display in templates:
- only one table
- errors are contextualized
...
...
@@ -125,8 +131,8 @@ def create_validata_ui_report(validata_core_report, schema):
report
[
'table'
][
'col_count'
]
=
len
(
headers
)
# Computes column info
schema_fields
=
schema
.
get
(
'fields'
,
[]
)
fields_dict
=
{
f
[
'name'
]:
(
f
.
get
(
'title'
,
'titre non défini'
),
f
.
get
(
'description'
,
''
))
for
f
in
schema_fields
}
fields_dict
=
{
f
[
'name'
]:
(
f
.
get
(
'title'
,
'titre non défini'
),
f
.
get
(
'description'
,
''
)
)
for
f
in
schema_dict
.
get
(
'fields'
,
[])
}
report
[
'table'
][
'headers_title'
]
=
[
fields_dict
[
h
][
0
]
if
h
in
fields_dict
else
'colonne inconnue'
for
h
in
headers
]
report
[
'table'
][
'headers_description'
]
=
[
fields_dict
[
h
][
1
]
if
h
in
fields_dict
else
'Cette colonne n
\'
est pas définie dans le schema'
for
h
in
headers
]
...
...
@@ -160,7 +166,7 @@ def create_validata_ui_report(validata_core_report, schema):
column_comparison_table
=
[]
if
column_comparison_needed
:
column_comparison_table
=
[]
field_names
=
[
f
[
'name'
]
for
f
in
schema_
fields
]
field_names
=
[
f
[
'name'
]
for
f
in
schema_
dict
.
get
(
'fields'
,
[])
]
has_case_errors
=
False
for
t
in
itertools
.
zip_longest
(
headers
,
field_names
,
fillvalue
=
''
):
status
=
'ok'
if
t
[
0
]
==
t
[
1
]
else
'ko'
...
...
@@ -235,28 +241,41 @@ def get_badge_url_and_message(badge):
config
.
SHIELDS_IO_BASE_URL
,
quote_plus
(
msg
),
color
),
msg
)
def
validate
(
schema_
code
,
source
:
ValidataSource
):
def
validate
(
schema_
url
,
source
:
ValidataSource
):
""" Validate source and display report """
if
config
.
API_VALIDATE_ENDPOINT
is
None
:
flash_error
(
"No Validate endpoint defined :-("
)
return
redirect
(
url_for
(
"custom_validator"
))
api_url
=
config
.
API_VALIDATE_ENDPOINT
headers
=
{
"Accept"
:
"application/json"
}
try
:
validata_core_report
=
ValidatorHelper
.
validate
(
schema_code
=
schema_code
,
force_strings
=
True
,
**
source
.
get_tabulator_params
()
)
# Validator.validate() doesn't throw FormatError if source.format is None
# Just do it manually
if
source
.
format
is
None
:
raise
tabulator
.
exceptions
.
FormatError
()
if
source
.
is_url
():
params
=
{
"schema"
:
schema_url
,
"url"
:
source
.
get_url
(),
}
req
=
requests
.
get
(
api_url
,
params
=
params
,
headers
=
headers
)
except
tabulator
.
exceptions
.
FormatError
:
flash_error
(
'Erreur : format de fichier non supporté'
)
return
redirect
(
url_for
(
'scdl_validator'
,
val_code
=
schema_code
))
else
:
files
=
{
'file'
:
(
source
.
name
,
io
.
BytesIO
(
source
.
source
))}
data
=
{
'schema'
:
schema_url
}
req
=
requests
.
post
(
api_url
,
data
=
data
,
files
=
files
,
headers
=
headers
)
except
validata_core
.
MissingHeaderError
:
flash_error
(
"Erreur : impossible d'extraire les données d'entête du fichier source"
)
return
redirect
(
url_for
(
'scdl_validator'
,
val_code
=
schema_code
))
if
not
req
.
ok
:
flash_error
(
"{}: :("
.
format
(
req
.
status_code
))
return
redirect
(
url_for
(
"home"
))
json_response
=
req
.
json
()
validata_core_report
=
json_response
[
'report'
]
schema_dict
=
json_response
[
'schema'
]
except
requests
.
ConnectionError
as
err
:
logging
.
exception
(
err
)
flash_error
(
str
(
err
))
return
redirect
(
url_for
(
'home'
))
# Computes badge from report and badge configuration
badge
=
compute_badge
(
validata_core_report
,
config
.
BADGE_CONFIG
)
...
...
@@ -268,7 +287,7 @@ def validate(schema_code, source: ValidataSource):
msg
=
"l'encodage du fichier est invalide. Veuillez le corriger"
if
'charmap'
in
err
[
'message'
]
else
err
[
'message'
]
flash_error
(
'Erreur de source : {}'
.
format
(
msg
))
return
redirect
(
url_for
(
'
scdl
_validator'
,
val_code
=
schema_code
))
return
redirect
(
url_for
(
'
custom
_validator'
))
source_data
=
extract_source_data
(
source
)
...
...
@@ -276,25 +295,24 @@ def validate(schema_code, source: ValidataSource):
report_datetime
=
datetime
.
fromisoformat
(
validata_core_report
[
'date'
]).
astimezone
()
# Enhance validata_core_report
validata_report
=
create_validata_ui_report
(
validata_core_report
,
ValidatorHelper
.
schema
(
schema_code
).
descriptor
)
validata_report
=
create_validata_ui_report
(
validata_core_report
,
schema_dict
)
# Display report to the user
val_info
=
ValidatorHelper
.
schema_info
(
schema_code
)
return
render_template
(
'validation_report.html'
,
title
=
'Rapport de validation'
,
val_info
=
ValidatorHelper
.
schema_info
(
schema_code
)
,
report
=
validata_report
,
val_info
=
{}
,
report
=
validata_report
,
validation_date
=
report_datetime
.
strftime
(
'le %d/%m/%Y à %Hh%M'
),
source
=
source
,
source_type
=
source
.
type
,
source_data
=
source_data
,
print_mode
=
request
.
args
.
get
(
'print'
,
'false'
)
==
'true'
,
badge_url
=
badge_url
,
badge_msg
=
badge_msg
,
report_str
=
json
.
dumps
(
validata_report
,
sort_keys
=
True
,
indent
=
2
),
breadcrumbs
=
[{
'url'
:
url_for
(
'home'
),
'title'
:
'Accueil'
},
{
'url'
:
url_for
(
'
scdl
_validator'
,
val_code
=
schema_code
),
'title'
:
val_info
[
'title'
]
}])
{
'url'
:
url_for
(
'
custom
_validator'
),
'title'
:
"Rapport de validation"
}])
def
bytes_data
(
f
):
""" Gets bytes data from Werkzeug FileStorage instance """
iob
=
BytesIO
()
iob
=
io
.
BytesIO
()
f
.
save
(
iob
)
iob
.
seek
(
0
)
return
iob
.
getvalue
()
...
...
@@ -376,10 +394,76 @@ def pdf_report(val_code):
return
response
@
app
.
route
(
'/validators/custom'
,
methods
=
[
'GET'
,
'POST'
])
def
custom_validator
():
""" Validator page """
if
config
.
API_VALIDATE_ENDPOINT
is
None
:
flash_error
(
"URL de connexion à l'API non indiquée :-("
)
return
redirect
(
url_for
(
'home'
))
if
request
.
method
==
'GET'
:
input_param
=
request
.
args
.
get
(
'input'
)
url_param
=
request
.
args
.
get
(
"url"
)
schema_param
=
request
.
args
.
get
(
"schema"
)
# First form display
if
input_param
is
None
:
return
render_template
(
'custom_validation_form.html'
,
title
=
"Schéma personnalisé"
,
val_info
=
None
,
breadcrumbs
=
[{
'url'
:
url_for
(
'home'
),
'title'
:
'Accueil'
},
])
# Process URL
else
:
if
url_param
is
None
or
url_param
==
''
:
flash_error
(
"Vous n'avez pas indiqué d'url à valider"
)
return
redirect
(
url_for
(
'custom_validator'
))
if
schema_param
is
None
or
schema_param
==
''
:
flash_error
(
"Vous n'avez pas indiqué d'url de schéma"
)
return
redirect
(
url_for
(
'custom_validator'
))
try
:
return
validate
(
schema_param
,
ValidataSource
(
'url'
,
url_param
,
url_param
))
except
tabulator
.
exceptions
.
FormatError
as
e
:
flash_error
(
'Erreur : Format de ressource non supporté'
)
log
.
info
(
e
)
return
redirect
(
url_for
(
'custom_validator'
))
except
tabulator
.
exceptions
.
HTTPError
as
e
:
flash_error
(
'Erreur : impossible d
\'
accéder au fichier source en ligne'
)
log
.
info
(
e
)
return
redirect
(
url_for
(
'custom_validator'
))
else
:
# POST
input_param
=
request
.
form
.
get
(
'input'
)
if
input_param
is
None
:
flash_error
(
'Aucun fichier à valider'
)
return
redirect
(
url_for
(
'custom_validator'
))
schema_param
=
request
.
form
.
get
(
'schema'
)
if
schema_param
is
None
:
flash_error
(
'Aucun schéma défini'
)
return
redirect
(
url_for
(
'custom_validator'
))
# File validation
if
input_param
==
'file'
:
f
=
request
.
files
.
get
(
'file'
)
if
f
is
None
:
flash_warning
(
"Vous n'avez pas indiqué de fichier à valider"
)
return
redirect
(
url_for
(
'custom_validator'
))
b_content
=
bytes_data
(
f
)
return
validate
(
schema_param
,
ValidataSource
(
'file'
,
f
.
filename
,
b_content
))
return
'Bizarre, vous avez dit bizarre ?'
@
app
.
route
(
'/validators/<val_code>'
,
methods
=
[
'GET'
,
'POST'
])
def
scdl_validator
(
val_code
):
""" Validator page """
# if val_code == "custom":
# return custom_validator()
if
not
ValidatorHelper
.
schema_exist
(
val_code
):
flash_error
(
'Validateur [{}] inconnu'
.
format
(
val_code
))
return
redirect
(
url_for
(
'home'
))
...
...
@@ -391,7 +475,7 @@ def scdl_validator(val_code):
# First form display
if
input_param
is
None
or
input_param
not
in
(
'url'
,
'example'
):
return
render_template
(
'validation_form.html'
,
title
=
val_info
[
'title'
],
return
render_template
(
'
scdl_
validation_form.html'
,
title
=
"
val_info['title']
"
,
val_info
=
val_info
,
breadcrumbs
=
[{
'url'
:
url_for
(
'home'
),
'title'
:
'Accueil'
},
])
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment