Commit 280eab1b authored by Christophe Benz's avatar Christophe Benz

Add CI, Dockerfile, rename config key, sort imports

parent 6f9c2d8c
......@@ -10,5 +10,5 @@ SHIELDS_IO_BASE_URL="https://img.shields.io/"
# Validata API endpoint
API_VALIDATE_ENDPOINT=http://127.0.0.1:5600/validate
# UI config file path
UI_CONFIG_FILE=config.json
\ No newline at end of file
# Homepage sections and blocks config file path
# HOMEPAGE_CONFIG_FILE=homepage_config.json
\ No newline at end of file
Run tests:
stage: test
image: python:3.7
script:
- python setup.py test
Build Docker image:
stage: deploy
only:
changes:
- Dockerfile
refs:
- tags
image: docker:stable
services:
- docker:dind
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
script:
- docker build -t $CI_REGISTRY_IMAGE:latest .
- docker push $CI_REGISTRY_IMAGE:latest
tags:
- docker-privileged
Publish on PyPI:
stage: deploy
image: python:3.7
only:
- tags
before_script:
- pip install twine
- python setup.py sdist bdist_wheel
variables:
TWINE_USERNAME: cbenz
# TWINE_PASSWORD: # Secret variable, see project CI settings.
script:
- twine upload dist/*
environment:
name: PyPI
url: https://pypi.org/project/validata-ui/$CI_COMMIT_TAG
## 0.1.0 -> next
## 0.2.0
New features for users:
- validate a tabular file (e.g. CSV) against a schema URL
- allow configuring homepage sections and blocks with a JSON config file (see `HOMEPAGE_CONFIG` environment variable in `.env`)
Non-breaking changes:
- New feature: validate a CSV against a schema URL
- UI now depends on validata-api, no more on validata-core
- UI now requests `validata-api` service to do the validation, and does not depend on `validata-core` anymore
- a Dockerfile has been added
- a Continuous Integration pipeline has been added
- the Docker image is rebuilt for each release
- the Python package is uploaded to [PyPI](https://pypi.org/) for each release
## 0.0.1 -> 0.1.0
## 0.1.0
Non-breaking changes:
......
FROM python:3.7
LABEL maintainer="admin-validata@jailbreak.paris"
EXPOSE 5000
RUN pip install gunicorn
WORKDIR /app
COPY requirements.txt .
RUN pip install --requirement requirements.txt
COPY . .
RUN pip install --editable .
CMD gunicorn --bind 0.0.0.0:5000 validata_ui:app
\ No newline at end of file
......@@ -2,15 +2,22 @@
Validata user interface
## Requirements
## Usage
PDF report uses [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md).
Please install:
```bash
apt install -y chromium
```
You can use the online instance of Validata:
- user interface: https://go.validata.fr/
- API: https://go.validata.fr/api/v1/
- API docs: https://go.validata.fr/api/v1/apidocs
Several software services compose the Validata stack. The recommended way to run it on your computer is to use Docker. Otherwise you can install each component of this stack manually, for example if you want to contribute by developing a new feature or fixing a bug.
## Run with Docker
Read instructions at https://git.opendatafrance.net/validata/validata-docker
## Develop
## Install
### Install
We recommend using [virtualenv](https://virtualenv.pypa.io/en/stable/).
......@@ -20,7 +27,15 @@ Install the project dependencies:
pip install -e .
```
## Configuration
Validata UI depends on [Validata API](https://git.opendatafrance.net/validata/validata-api/), so you must install it also.
PDF report generation uses [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md):
```bash
apt install -y chromium
```
### Configure
```bash
cp .env.example .env
......@@ -30,9 +45,7 @@ Customize the configuration variables in `.env` file.
Do not commit `.env`.
See also: https://github.com/theskumar/python-dotenv
## Development
### Serve
Start the web server...
......
backports-datetime-fromisoformat==1.0.0
commonmark==0.8.1
ezodf==0.3.2
Flask==1.0.2
lxml==4.2.5
python-dotenv==0.10.1
requests==2.22.0
toml==0.10.0
tabulator==1.21.0
opendataschema==0.2.0
validata_core==0.3.4
[isort]
line_length = 120
[pycodestyle]
max_line_length = 120
[pylint]
max_line_length = 120
[aliases]
test=pytest
[tool:pytest]
addopts = --doctest-modules
\ No newline at end of file
#!/usr/bin/env python3
"""Run validata ui"""
from pathlib import Path
from setuptools import setup
classifiers = """\
Development Status :: 4 - Beta
Intended Audience :: Developers
Operating System :: OS Independent
Programming Language :: Python
Topic :: Software Development :: Libraries :: Python Modules
License :: OSI Approved :: GNU Affero General Public License v3
"""
# Gets the long description from the README.md file
readme_filepath = Path(__file__).parent / 'README.md'
with readme_filepath.open('rt', encoding='utf-8') as fd_in:
LONG_DESCRIPTION = fd_in.read()
setup(
name='validata_ui',
version='0.1.0',
version='0.2.0',
description='Validata Web UI',
long_description=LONG_DESCRIPTION,
long_description_content_type="text/markdown",
url='https://git.opendatafrance.net/validata/validata-ui',
author='Validata team',
classifiers=[classifier for classifier in classifiers.split('\n') if classifier],
description=__doc__,
author_email='admin-validata@jailbreak.paris',
license='AGPLv3',
# See https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[
# How mature is this project? Common values are
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 5 - Production/Stable',
# Indicate who your project is intended for
'Intended Audience :: Developers',
'Topic :: Software Development :: Libraries :: Python Modules',
'Operating System :: OS Independent',
# Pick your license as you wish (should match "license" above)
'License :: OSI Approved :: GNU Affero General Public License v3',
# Specify the Python versions you support here. In particular, ensure
# that you indicate whether you support Python 2, Python 3 or both.
'Programming Language :: Python :: 3',
],
packages=['validata_ui'],
include_package_data=True,
zip_safe=True,
install_requires=[
'backports-datetime-fromisoformat',
......@@ -33,9 +61,9 @@ setup(
'requests',
'toml',
'goodtables',
'tabulator',
'validata_core >= 0.2.1, < 0.3',
]
'opendataschema >= 0.2.0, < 0.3',
'validata_core >= 0.3.0, < 0.4',
],
)
......@@ -3,13 +3,13 @@ import os
from pathlib import Path
from urllib.parse import quote_plus
import opendataschema
import flask
import jinja2
import requests
import tableschema
from cachetools.func import ttl_cache
import opendataschema
# Let this import after app initialisation
from . import config
......@@ -25,16 +25,14 @@ def schema_from_url(url):
return tableschema.Schema(url)
# load config.json
ui_config = json.load(config.UI_CONFIG_FILE.open('rt', encoding='utf-8')) if config.UI_CONFIG_FILE else []
# And load schema catalogs which urls are found in config.json
schema_catalog_map = {}
for section in ui_config['sections']:
if isinstance(section['catalog'], str) and section['catalog'].startswith('http'):
code = section['code']
url = section['catalog']
schema_catalog_map[code] = opendataschema.SchemaCatalog(url, download_func=download_with_cache)
if config.HOMEPAGE_CONFIG:
for section in config.HOMEPAGE_CONFIG['sections']:
if isinstance(section['catalog'], str) and section['catalog'].startswith('http'):
code = section['code']
url = section['catalog']
schema_catalog_map[code] = opendataschema.SchemaCatalog(url, download_func=download_with_cache)
# Flask things
app = flask.Flask(__name__)
......@@ -53,4 +51,4 @@ def urlencode(context, value):
# Keep this import after app initialisation (to avoid cyclic imports)
from . import views # isort:skip
from . import views # noqa isort:skip
import json
import logging
import os
from pathlib import Path
......@@ -33,6 +34,9 @@ SHIELDS_IO_BASE_URL = os.environ.get("SHIELDS_IO_BASE_URL") or None
if SHIELDS_IO_BASE_URL and not SHIELDS_IO_BASE_URL.endswith('/'):
SHIELDS_IO_BASE_URL += '/'
UI_CONFIG_FILE = os.environ.get("UI_CONFIG_FILE") or None
if UI_CONFIG_FILE:
UI_CONFIG_FILE = Path(UI_CONFIG_FILE)
HOMEPAGE_CONFIG_FILE = os.environ.get("HOMEPAGE_CONFIG_FILE") or None
HOMEPAGE_CONFIG = None
if HOMEPAGE_CONFIG_FILE:
HOMEPAGE_CONFIG_FILE = Path(HOMEPAGE_CONFIG_FILE)
with HOMEPAGE_CONFIG_FILE.open() as fd:
HOMEPAGE_CONFIG = json.load(fd)
......@@ -15,16 +15,16 @@ from urllib.parse import quote_plus, urlencode
import requests
import tableschema
import tabulator
from backports.datetime_fromisoformat import MonkeyPatch
from commonmark import commonmark
from flask import make_response, redirect, render_template, request, url_for
from validata_core import compute_badge, messages
import tabulator
from validata_core import compute_badge, messages
from . import app, config, ui_config, schema_catalog_map, schema_from_url
from . import app, config, schema_catalog_map, schema_from_url
from .ui_util import flash_error, flash_warning
from .validata_util import ValidataResource, URLValidataResource, UploadedFileValidataResource
from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
MonkeyPatch.patch_fromisoformat()
......@@ -395,7 +395,7 @@ def bytes_data(f):
return iob.getvalue()
def ui_config_with_schema_metadata(ui_config, schema_catalog_map):
def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
"""Replace catalog url within ui_config by schema references
containing schema metadata properties"""
......@@ -424,7 +424,7 @@ def ui_config_with_schema_metadata(ui_config, schema_catalog_map):
def home():
""" Home page """
flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu.')
home_config = ui_config_with_schema_metadata(ui_config, schema_catalog_map)
home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map)
return render_template('home.html', title='Accueil', config=home_config)
......@@ -507,7 +507,7 @@ def compute_validation_form_url(schema_instance: SchemaInstance):
return "{}?{}".format(url, '&'.join(param_list))
@app.route('/table_schema', methods=['GET', 'POST'])
@app.route('/table-schema', methods=['GET', 'POST'])
def custom_validator():
"""Validator form"""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment