Commit 8f981bba authored by Pierre Dittgen's avatar Pierre Dittgen

Website structure is ok. Time to really validate and display CSV data

parent 79dddda6
Pipeline #5 failed with stages
__pycache__/
......@@ -3,4 +3,21 @@
Next generation validata ui, rid of goodtables-ui constraints and using improved goodtables-py.
## Dependencies
- [Flask](http://flask.pocoo.org/)
\ No newline at end of file
- [Flask](http://flask.pocoo.org/)
We recommand using [virtualenv](https://virtualenv.pypa.io/en/stable/) to install dependencies inside application sandbox
Install the project dependencies:
```bash
pip install -e .
```
## Development
Start the web server:
```bash
export FLASK_APP=validata-ui-next
export FLASK_ENV=development
flask run
* Running on http://127.0.0.1:5000/
```
\ No newline at end of file
#!/usr/bin/env python3
"""Run validata ui"""
from setuptools import setup
classifiers = """\
Development Status :: 4 - Beta
Intended Audience :: Developers
Operating System :: OS Independent
Programming Language :: Python
Topic :: Software Development :: Libraries :: Python Modules
"""
setup(
name='validata_ui_next',
version='0.0.1',
author='Pierre Dittgen',
author_email='pierre.dittgen@jailbreak.paris',
classifiers=[classifier for classifier in classifiers.split('\n') if classifier],
description=__doc__,
packages=['validata_ui_next'],
include_package_data=True,
install_requires=[
'flask',
'requests',
'ujson'
]
)
#!/usr/bin/env python3
from pathlib import Path
from flask import Flask
from validata_ui_next.validate import ValidatorHelper
# Schemas settings
schemas_config = {
"scdl-adresses": {
"schema_json_url": "https://git.opendatafrance.net/scdl/adresses/raw/master/schema-scdl-adresses.json",
"custom_checks_json_url": "https://git.opendatafrance.net/scdl/adresses/raw/master/custom-checks.json"
},
"scdl-deliberations": {
"schema_json_url": "https://git.opendatafrance.net/scdl/deliberations/raw/master/schema.json",
"custom_checks_json_url": "https://git.opendatafrance.net/scdl/deliberations/raw/master/custom-checks.json"
},
"scdl-marches-publics": {
"schema_json_url": "https://git.opendatafrance.net/scdl/marches-publics/raw/pdi/schema.json",
"custom_checks_json_url": "https://git.opendatafrance.net/scdl/marches-publics/raw/pdi/custom-checks.json"
},
"scdl-prenoms": {
"schema_json_url": "https://github.com/CharlesNepote/liste-prenoms-nouveaux-nes/raw/v1.1.1/prenom-schema.json",
# "custom_checks_json_url": "https://git.opendatafrance.net/scdl/prenoms/raw/master/custom-checks.json"
},
"scdl-subventions": {
"schema_json_url": "https://git.opendatafrance.net/scdl/subventions/raw/master/schema.json",
"custom_checks_json_url": "https://git.opendatafrance.net/scdl/subventions/raw/master/custom-checks.json"
},
}
ValidatorHelper.init(schemas_config, Path('/tmp'))
# Flask things
app = Flask(__name__)
app.secret_key = 'MyPr3ci0u5$€cr€t'
import validata_ui_next.views
#!/usr/bin/env python3
"""
Validata-ui-next entry-point
"""
import json
import os
from collections import OrderedDict
from pathlib import Path
from flask import Flask, flash, redirect, render_template, request, url_for
app = Flask(__name__)
app.secret_key = 'MyPr3ci0u5$€cr€t'
schemas_config = {
"scdl-prenoms": {
"schema_json_url": "https://git.openqqq.com/",
"custom_checks_json_url": ""}
}
VALIDATOR_JSON_FILE = 'static/data/scdl-validators.json'
class Validators:
validator_map = OrderedDict()
@classmethod
def init(cls, validators_file):
""" Static init """
val_list = json.load(open(validators_file))
cls.validator_map = OrderedDict([(val['code'], val) for val in val_list])
@classmethod
def list(cls):
""" Validator list """
return [val for _, val in cls.validator_map.items()]
@classmethod
def is_valid_code(cls, code):
""" Test if validator code exist """
return code in cls.validator_map
@classmethod
def is_val_enabled(cls, val):
ret = val and val.get('enable', 'true') == 'true'
return ret
@classmethod
def info(cls, code):
""" Return information on validator """
return cls.validator_map.get(code)
# TODO externalize util classes
def validate_url(url):
return "TODO: valider l'url "+url
# return render_template('validation_report.html')
def validate_file(file):
return "TODO: valider le fichier "+file
# return render_template('validation_report.html')
def flash_error(msg):
flash(msg, 'danger')
def flash_warning(msg):
flash(msg, 'warning')
def flash_success(msg):
flash(msg, 'success')
def flash_info(msg):
flash(msg, 'info')
Validators.init(VALIDATOR_JSON_FILE)
@app.route('/')
def home():
validators = [{**val, 'enabled': Validators.is_val_enabled(val)} for val in Validators.list()]
return render_template('home.html', title='Accueil', validators=validators)
@app.route('/about')
def about():
return render_template('about.html', title='À propos',
breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])
@app.route('/validators')
def validators():
return redirect(url_for('home'))
@app.route('/validators/<val_code>', methods=['GET', 'POST'])
def scdl_validator(val_code):
if not Validators.is_valid_code(val_code):
flash_error('Validateur [{}] inconnu'.format(val_code))
return redirect(url_for('home'))
if not Validators.is_val_enabled(Validators.info(val_code)):
flash_error('Pas de validateur pour le schéma [{}]'.format(val_code))
return redirect(url_for('home'))
if request.method == 'GET':
val_info = Validators.info(val_code)
input_param = request.args.get('input')
# First form display
if input_param is None or input_param not in ('url', 'example'):
return render_template('validator.html', title=val_info['name'],
val_info=val_info,
breadcrumbs=[{'url': url_for('home'), 'title': 'Accueil'}, ])
# Process URL
else:
url = request.args.get('url')
if url is None or url == '':
flash_error("Vous n'avez pas indiqué d'url à valider")
return redirect(url_for('scdl_validator', val_code=val_code))
return validate_url(url)
else: # POST
input_param = request.form.get('input')
if input_param is None:
flash_error('Source non définie')
return redirect(url_for('scdl_validator', val_code=val_code))
# File validation
if input_param == 'file':
f = request.files.get('file')
if f is None:
flash_warning("Vous n'avez pas indiqué de fichier à valider")
return redirect(url_for('scdl_validator', val_code=val_code))
f.save(os.path.join('/tmp', f.filename))
return validate_file(f.filename)
return 'Bizarre, vous avez dit bizarre ?'
if __name__ == '__main__':
app.run()
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
/*!
* Bootstrap Reboot v4.1.3 (https://getbootstrap.com/)
* Copyright 2011-2018 The Bootstrap Authors
* Copyright 2011-2018 Twitter, Inc.
* Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
* Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md)
*/
*,
*::before,
*::after {
box-sizing: border-box;
}
html {
font-family: sans-serif;
line-height: 1.15;
-webkit-text-size-adjust: 100%;
-ms-text-size-adjust: 100%;
-ms-overflow-style: scrollbar;
-webkit-tap-highlight-color: rgba(0, 0, 0, 0);
}
@-ms-viewport {
width: device-width;
}
article, aside, figcaption, figure, footer, header, hgroup, main, nav, section {
display: block;
}
body {
margin: 0;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
font-size: 1rem;
font-weight: 400;
line-height: 1.5;
color: #212529;
text-align: left;
background-color: #fff;
}
[tabindex="-1"]:focus {
outline: 0 !important;
}
hr {
box-sizing: content-box;
height: 0;
overflow: visible;
}
h1, h2, h3, h4, h5, h6 {
margin-top: 0;
margin-bottom: 0.5rem;
}
p {
margin-top: 0;
margin-bottom: 1rem;
}
abbr[title],
abbr[data-original-title] {
text-decoration: underline;
-webkit-text-decoration: underline dotted;
text-decoration: underline dotted;
cursor: help;
border-bottom: 0;
}
address {
margin-bottom: 1rem;
font-style: normal;
line-height: inherit;
}
ol,
ul,
dl {
margin-top: 0;
margin-bottom: 1rem;
}
ol ol,
ul ul,
ol ul,
ul ol {
margin-bottom: 0;
}
dt {
font-weight: 700;
}
dd {
margin-bottom: .5rem;
margin-left: 0;
}
blockquote {
margin: 0 0 1rem;
}
dfn {
font-style: italic;
}
b,
strong {
font-weight: bolder;
}
small {
font-size: 80%;
}
sub,
sup {
position: relative;
font-size: 75%;
line-height: 0;
vertical-align: baseline;
}
sub {
bottom: -.25em;
}
sup {
top: -.5em;
}
a {
color: #007bff;
text-decoration: none;
background-color: transparent;
-webkit-text-decoration-skip: objects;
}
a:hover {
color: #0056b3;
text-decoration: underline;
}
a:not([href]):not([tabindex]) {
color: inherit;
text-decoration: none;
}
a:not([href]):not([tabindex]):hover, a:not([href]):not([tabindex]):focus {
color: inherit;
text-decoration: none;
}
a:not([href]):not([tabindex]):focus {
outline: 0;
}
pre,
code,
kbd,
samp {
font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
font-size: 1em;
}
pre {
margin-top: 0;
margin-bottom: 1rem;
overflow: auto;
-ms-overflow-style: scrollbar;
}
figure {
margin: 0 0 1rem;
}
img {
vertical-align: middle;
border-style: none;
}
svg {
overflow: hidden;
vertical-align: middle;
}
table {
border-collapse: collapse;
}
caption {
padding-top: 0.75rem;
padding-bottom: 0.75rem;
color: #6c757d;
text-align: left;
caption-side: bottom;
}
th {
text-align: inherit;
}
label {
display: inline-block;
margin-bottom: 0.5rem;
}
button {
border-radius: 0;
}
button:focus {
outline: 1px dotted;
outline: 5px auto -webkit-focus-ring-color;
}
input,
button,
select,
optgroup,
textarea {
margin: 0;
font-family: inherit;
font-size: inherit;
line-height: inherit;
}
button,
input {
overflow: visible;
}
button,
select {
text-transform: none;
}
button,
html [type="button"],
[type="reset"],
[type="submit"] {
-webkit-appearance: button;
}
button::-moz-focus-inner,
[type="button"]::-moz-focus-inner,
[type="reset"]::-moz-focus-inner,
[type="submit"]::-moz-focus-inner {
padding: 0;
border-style: none;
}
input[type="radio"],
input[type="checkbox"] {
box-sizing: border-box;
padding: 0;
}
input[type="date"],
input[type="time"],
input[type="datetime-local"],
input[type="month"] {
-webkit-appearance: listbox;
}
textarea {
overflow: auto;
resize: vertical;
}
fieldset {
min-width: 0;
padding: 0;
margin: 0;
border: 0;
}
legend {
display: block;
width: 100%;
max-width: 100%;
padding: 0;
margin-bottom: .5rem;
font-size: 1.5rem;
line-height: inherit;
color: inherit;
white-space: normal;
}
progress {
vertical-align: baseline;
}
[type="number"]::-webkit-inner-spin-button,
[type="number"]::-webkit-outer-spin-button {
height: auto;
}
[type="search"] {
outline-offset: -2px;
-webkit-appearance: none;
}
[type="search"]::-webkit-search-cancel-button,
[type="search"]::-webkit-search-decoration {
-webkit-appearance: none;
}
::-webkit-file-upload-button {
font: inherit;
-webkit-appearance: button;
}
output {
display: inline-block;
}
summary {
display: list-item;
cursor: pointer;
}
template {
display: none;
}
[hidden] {
display: none !important;
}
/*# sourceMappingURL=bootstrap-reboot.css.map */
\ No newline at end of file
This diff is collapsed.
/*!
* Bootstrap Reboot v4.1.3 (https://getbootstrap.com/)
* Copyright 2011-2018 The Bootstrap Authors
* Copyright 2011-2018 Twitter, Inc.
* Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
* Forked from Normalize.css, licensed MIT (https://github.com/necolas/normalize.css/blob/master/LICENSE.md)
*/*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-ms-overflow-style:scrollbar;-webkit-tap-highlight-color:transparent}@-ms-viewport{width:device-width}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}dfn{font-style:italic}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent;-webkit-text-decoration-skip:objects}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[type=reset],[type=submit],button,html [type=button]{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}
/*# sourceMappingURL=bootstrap-reboot.min.css.map */
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
{
"scdl-adresses": {
"title": "Adresses locales",
"description": "Liste des adresses locales d'une collectivité",
"specurl": "http://www.opendatafrance.net/SCDL_Adresses_Locales",
"docurl": "https://git.opendatafrance.net/validata/validata-doc/blob/master/static/schemas/scdl-adresses.md",
"examples": [
{
"name": "Adresses fictives invalides",
"url": "https://git.opendatafrance.net/scdl/adresses/raw/v1.1/exemples/exemple_invalide.csv"
},
{
"name": "Adresses de Bayonne avril 2018",
"url": "https://git.opendatafrance.net/scdl/adresses/raw/v1.1/exemples/20180424_bal_216401026.csv"
}
]
},
"scdl-deliberations": {
"title": "Délibérations",
"description": "Liste des délibérations adoptées par une assemblée locale",
"specurl": "http://www.opendatafrance.net/SCDL_Deliberations",
"docurl": "https://git.opendatafrance.net/validata/validata-doc/blob/master/static/schemas/scdl-deliberations.md",
"examples": [
{
"name": "Délibérations fictives valides",
"url": "https://git.opendatafrance.net/scdl/deliberations/raw/v2.0/examples/Deliberations_ok.csv"
},
{
"name": "Délibérations fictives invalides",
"url": "https://git.opendatafrance.net/scdl/deliberations/raw/v2.0/examples/DeliberationsCindoc.csv"
}
]
},
"scdl-marches-publics": {
"title": "Marchés publics",
"description": "Liste des marchés publics attribués par une collectivité",
"specurl": "http://www.opendatafrance.net/SCDL_Marches_Publics",
"docurl": "https://git.opendatafrance.net/validata/validata-doc/blob/master/static/schemas/scdl-marches.md",
"examples": [
{
"name": "Marchés publics fictifs valides",
"url": "https://git.opendatafrance.net/scdl/marches-publics/raw/master/exemples/exemple_marche_public.csv"
},
{
"name": "Marchés publics fictifs invalides",
"url": "https://git.opendatafrance.net/scdl/marches-publics/raw/master/exemples/exemple_marche_public_avec_erreurs.csv"
}
]
},
"scdl-prenoms": {
"title": "Prénoms des nouveaux-nés",
"description": "Liste des prénoms des nouveaux-nés déclarés à l'état-civil",
"specurl": "https://docs.google.com/document/d/1Vk0kpBw3MIocai9JqovLK2HxcUA_3QHnZicqxuOpcQ8/edit?usp=sharing",
"docurl": "https://git.opendatafrance.net/validata/validata-doc/blob/master/static/schemas/scdl-prenoms.md",
"examples": [
{
"name": "Prénoms des nouveaux-nés Digne-les-Bains 2017",
"url": "https://raw.githubusercontent.com/CharlesNepote/liste-prenoms-nouveaux-nes/v1.1.1/DIGNE-PRENOMS-2017.csv"
},
{
"name": "Prénoms des nouveaux-nés fictifs invalides",
"url": "https://raw.githubusercontent.com/CharlesNepote/liste-prenoms-nouveaux-nes/v1.1.1/prenoms-nouveaux-nes.exemple.invalide.1.1.csv"
}
]
},
"scdl-subventions": {
"title": "Subventions",
"description": "Liste des subventions publiques attribuées par une collectivité",
"specurl": "http://www.opendatafrance.net/SCDL_Subventions",
"docurl": "https://git.opendatafrance.net/validata/validata-doc/blob/master/static/schemas/scdl-subventions.md",
"examples": [
{
"name": "Subventions fictives invalides",
"url": "https://git.opendatafrance.net/scdl/subventions/raw/v1.1/exemples/exemple_invalide.csv"
},
{
"name": "Erreur de format (HTML au lieu de CSV)",
"url": "https://git.opendatafrance.net/"
}
]