diff --git a/.env.example b/.env.example index 3a6863c142ca98f494a3af6394370eeb980d1d98..42c74ba56fd6ecada1291974aa58cb2dd13d848f 100644 --- a/.env.example +++ b/.env.example @@ -10,5 +10,5 @@ SHIELDS_IO_BASE_URL="https://img.shields.io/" # Validata API endpoint API_VALIDATE_ENDPOINT=http://127.0.0.1:5600/validate -# UI config file path -UI_CONFIG_FILE=config.json \ No newline at end of file +# Homepage sections and blocks config file path +# HOMEPAGE_CONFIG_FILE=homepage_config.json \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..9fa8ddcb86f54a8ddcd5d7538f7fe5aa2353cd00 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,43 @@ +Run tests: + stage: test + image: python:3.7 + script: + - python setup.py test + +Build Docker image: + stage: deploy + only: + changes: + - Dockerfile + refs: + - tags + image: docker:stable + services: + - docker:dind + variables: + DOCKER_HOST: tcp://docker:2375 + DOCKER_DRIVER: overlay2 + before_script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + script: + - docker build -t $CI_REGISTRY_IMAGE:latest . + - docker push $CI_REGISTRY_IMAGE:latest + tags: + - docker-privileged + +Publish on PyPI: + stage: deploy + image: python:3.7 + only: + - tags + before_script: + - pip install twine + - python setup.py sdist bdist_wheel + variables: + TWINE_USERNAME: cbenz + # TWINE_PASSWORD: # Secret variable, see project CI settings. + script: + - twine upload dist/* + environment: + name: PyPI + url: https://pypi.org/project/validata-ui/$CI_COMMIT_TAG diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f71821a728c1fd2f1fc40c66282005f24dfc9c5..75626a5002b185045119c2da76d1d12fe2cc62e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,19 @@ -## 0.1.0 -> next +## 0.2.0 + +New features for users: + +- validate a tabular file (e.g. CSV) against a schema URL +- allow configuring homepage sections and blocks with a JSON config file (see `HOMEPAGE_CONFIG` environment variable in `.env`) Non-breaking changes: -- New feature: validate a CSV against a schema URL -- UI now depends on validata-api, no more on validata-core +- UI now requests `validata-api` service to do the validation, and does not depend on `validata-core` anymore +- a Dockerfile has been added +- a Continuous Integration pipeline has been added + - the Docker image is rebuilt for each release + - the Python package is uploaded to [PyPI](https://pypi.org/) for each release -## 0.0.1 -> 0.1.0 +## 0.1.0 Non-breaking changes: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ac235ec1b398c9ffb4aebb51ac48da7d14361aca --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.7 +LABEL maintainer="admin-validata@jailbreak.paris" + +EXPOSE 5000 + +RUN pip install gunicorn + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --requirement requirements.txt + +COPY . . +RUN pip install --editable . + +CMD gunicorn --bind 0.0.0.0:5000 validata_ui:app \ No newline at end of file diff --git a/README.md b/README.md index 182284ba01ba5abffc11dd3961a439cf3e6851d8..cc6bc8b14a1f2d5fe4ae882d0d464df6949045dc 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,22 @@ Validata user interface -## Requirements +## Usage -PDF report uses [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md). -Please install: -```bash -apt install -y chromium -``` +You can use the online instance of Validata: +- user interface: https://go.validata.fr/ +- API: https://go.validata.fr/api/v1/ +- API docs: https://go.validata.fr/api/v1/apidocs + +Several software services compose the Validata stack. The recommended way to run it on your computer is to use Docker. Otherwise you can install each component of this stack manually, for example if you want to contribute by developing a new feature or fixing a bug. + +## Run with Docker + +Read instructions at https://git.opendatafrance.net/validata/validata-docker + +## Develop -## Install +### Install We recommend using [virtualenv](https://virtualenv.pypa.io/en/stable/). @@ -20,7 +27,15 @@ Install the project dependencies: pip install -e . ``` -## Configuration +Validata UI depends on [Validata API](https://git.opendatafrance.net/validata/validata-api/), so you must install it also. + +PDF report generation uses [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md): + +```bash +apt install -y chromium +``` + +### Configure ```bash cp .env.example .env @@ -30,9 +45,7 @@ Customize the configuration variables in `.env` file. Do not commit `.env`. -See also: https://github.com/theskumar/python-dotenv - -## Development +### Serve Start the web server... diff --git a/config.json.example b/homepage_config.json.example similarity index 100% rename from config.json.example rename to homepage_config.json.example diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2c2525af1669ff7316d0065a36f63f842e549628 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +backports-datetime-fromisoformat==1.0.0 +commonmark==0.8.1 +ezodf==0.3.2 +Flask==1.0.2 +lxml==4.2.5 +python-dotenv==0.10.1 +requests==2.22.0 +toml==0.10.0 +tabulator==1.21.0 +opendataschema==0.2.0 +validata_core==0.3.4 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..fc5d43bbc88b28df0d51930e1efdb182ebca0c91 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,14 @@ +[isort] +line_length = 120 + +[pycodestyle] +max_line_length = 120 + +[pylint] +max_line_length = 120 + +[aliases] +test=pytest + +[tool:pytest] +addopts = --doctest-modules \ No newline at end of file diff --git a/setup.py b/setup.py index 1cd21c1a4c2eacc0632fe22bbbf3edfcacc93745..48fe1411738d3ad871a53ce1f9242f7306b04b54 100644 --- a/setup.py +++ b/setup.py @@ -1,27 +1,55 @@ #!/usr/bin/env python3 -"""Run validata ui""" + +from pathlib import Path + from setuptools import setup -classifiers = """\ -Development Status :: 4 - Beta -Intended Audience :: Developers -Operating System :: OS Independent -Programming Language :: Python -Topic :: Software Development :: Libraries :: Python Modules -License :: OSI Approved :: GNU Affero General Public License v3 -""" +# Gets the long description from the README.md file +readme_filepath = Path(__file__).parent / 'README.md' +with readme_filepath.open('rt', encoding='utf-8') as fd_in: + LONG_DESCRIPTION = fd_in.read() setup( name='validata_ui', - version='0.1.0', + version='0.2.0', + + description='Validata Web UI', + long_description=LONG_DESCRIPTION, + long_description_content_type="text/markdown", + + url='https://git.opendatafrance.net/validata/validata-ui', author='Validata team', - classifiers=[classifier for classifier in classifiers.split('\n') if classifier], - description=__doc__, + author_email='admin-validata@jailbreak.paris', + + license='AGPLv3', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 5 - Production/Stable', + + # Indicate who your project is intended for + 'Intended Audience :: Developers', + + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Operating System :: OS Independent', + + # Pick your license as you wish (should match "license" above) + 'License :: OSI Approved :: GNU Affero General Public License v3', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 3', + ], packages=['validata_ui'], - include_package_data=True, + zip_safe=True, + install_requires=[ 'backports-datetime-fromisoformat', @@ -33,9 +61,9 @@ setup( 'requests', 'toml', - 'goodtables', 'tabulator', - 'validata_core >= 0.2.1, < 0.3', - ] + 'opendataschema >= 0.2.0, < 0.3', + 'validata_core >= 0.3.0, < 0.4', + ], ) diff --git a/validata_ui/__init__.py b/validata_ui/__init__.py index 586a18087e6c2d873cd2207a274d38661bfad894..97a2475df89541fa4c76e7cea711293ed45ad673 100644 --- a/validata_ui/__init__.py +++ b/validata_ui/__init__.py @@ -3,13 +3,13 @@ import os from pathlib import Path from urllib.parse import quote_plus -import opendataschema import flask import jinja2 import requests import tableschema from cachetools.func import ttl_cache +import opendataschema # Let this import after app initialisation from . import config @@ -25,16 +25,14 @@ def schema_from_url(url): return tableschema.Schema(url) -# load config.json -ui_config = json.load(config.UI_CONFIG_FILE.open('rt', encoding='utf-8')) if config.UI_CONFIG_FILE else [] - # And load schema catalogs which urls are found in config.json schema_catalog_map = {} -for section in ui_config['sections']: - if isinstance(section['catalog'], str) and section['catalog'].startswith('http'): - code = section['code'] - url = section['catalog'] - schema_catalog_map[code] = opendataschema.SchemaCatalog(url, download_func=download_with_cache) +if config.HOMEPAGE_CONFIG: + for section in config.HOMEPAGE_CONFIG['sections']: + if isinstance(section['catalog'], str) and section['catalog'].startswith('http'): + code = section['code'] + url = section['catalog'] + schema_catalog_map[code] = opendataschema.SchemaCatalog(url, download_func=download_with_cache) # Flask things app = flask.Flask(__name__) @@ -53,4 +51,4 @@ def urlencode(context, value): # Keep this import after app initialisation (to avoid cyclic imports) -from . import views # isort:skip +from . import views # noqa isort:skip diff --git a/validata_ui/config.py b/validata_ui/config.py index 3536050a6810d8d09a56672ac2895ee0951fe54b..4a376581484b9db7522decc3b9d283c8cb52f5c3 100644 --- a/validata_ui/config.py +++ b/validata_ui/config.py @@ -1,3 +1,4 @@ +import json import logging import os from pathlib import Path @@ -33,6 +34,9 @@ SHIELDS_IO_BASE_URL = os.environ.get("SHIELDS_IO_BASE_URL") or None if SHIELDS_IO_BASE_URL and not SHIELDS_IO_BASE_URL.endswith('/'): SHIELDS_IO_BASE_URL += '/' -UI_CONFIG_FILE = os.environ.get("UI_CONFIG_FILE") or None -if UI_CONFIG_FILE: - UI_CONFIG_FILE = Path(UI_CONFIG_FILE) +HOMEPAGE_CONFIG_FILE = os.environ.get("HOMEPAGE_CONFIG_FILE") or None +HOMEPAGE_CONFIG = None +if HOMEPAGE_CONFIG_FILE: + HOMEPAGE_CONFIG_FILE = Path(HOMEPAGE_CONFIG_FILE) + with HOMEPAGE_CONFIG_FILE.open() as fd: + HOMEPAGE_CONFIG = json.load(fd) diff --git a/validata_ui/views.py b/validata_ui/views.py index 538e854a3959e7104c20e53c16a0b0d162e18a5b..d5ddca6d45f04f2da0458ed90a3499603db9a77d 100644 --- a/validata_ui/views.py +++ b/validata_ui/views.py @@ -15,16 +15,16 @@ from urllib.parse import quote_plus, urlencode import requests import tableschema +import tabulator from backports.datetime_fromisoformat import MonkeyPatch from commonmark import commonmark from flask import make_response, redirect, render_template, request, url_for -from validata_core import compute_badge, messages -import tabulator +from validata_core import compute_badge, messages -from . import app, config, ui_config, schema_catalog_map, schema_from_url +from . import app, config, schema_catalog_map, schema_from_url from .ui_util import flash_error, flash_warning -from .validata_util import ValidataResource, URLValidataResource, UploadedFileValidataResource +from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource MonkeyPatch.patch_fromisoformat() @@ -395,7 +395,7 @@ def bytes_data(f): return iob.getvalue() -def ui_config_with_schema_metadata(ui_config, schema_catalog_map): +def homepage_config_with_schema_metadata(ui_config, schema_catalog_map): """Replace catalog url within ui_config by schema references containing schema metadata properties""" @@ -424,7 +424,7 @@ def ui_config_with_schema_metadata(ui_config, schema_catalog_map): def home(): """ Home page """ flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu.') - home_config = ui_config_with_schema_metadata(ui_config, schema_catalog_map) + home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map) return render_template('home.html', title='Accueil', config=home_config) @@ -507,7 +507,7 @@ def compute_validation_form_url(schema_instance: SchemaInstance): return "{}?{}".format(url, '&'.join(param_list)) -@app.route('/table_schema', methods=['GET', 'POST']) +@app.route('/table-schema', methods=['GET', 'POST']) def custom_validator(): """Validator form"""