Skip to content

GitLab

Commit 26d5aed0 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Replace cache_control by requests_cache

parent 5af4e43e
Hide whitespace changes
Inline Side-by-side
.env.example
View file @ 26d5aed0
......@@ -24,3 +24,6 @@ CONFIG=config.example.yaml
#
# BROWSERLESS_API_URL="https://chrome.browserless.io/pdf"
# BROWSERLESS_API_TOKEN="XXX"
# Number of minutes to cache downloaded schemas
# CACHING_EXPIRE_AFTER=0
\ No newline at end of file
.gitignore
View file @ 26d5aed0
__pycache__/
validata_ui.egg-info/
validata_ui_cache.sqlite
.env
.mypy_cache/
.pytest_cache/
......
requirements.in
View file @ 26d5aed0
-e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core
cachecontrol
commonmark
ezodf
flask
......@@ -9,6 +8,7 @@ pydantic
python-dotenv
pyyaml
requests
requests_cache
sentry-sdk[flask] == 0.14.3
toml
opendataschema >= 0.5.5, < 0.6
......
requirements.txt
View file @ 26d5aed0
......@@ -4,65 +4,121 @@
#
# pip-compile
#
-e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core # via -r requirements.in
attrs==20.3.0 # via jsonschema
blinker==1.4 # via sentry-sdk
cachecontrol==0.12.6 # via -r requirements.in
certifi==2020.12.5 # via requests, sentry-sdk
chardet==3.0.4 # via frictionless, requests
click==7.1.2 # via flask, opendataschema, typer
colorama==0.4.4 # via typer
commonmark==0.9.1 # via -r requirements.in
decorator==4.4.2 # via validators
deprecated==1.2.10 # via pygithub
et-xmlfile==1.0.1 # via openpyxl
ezodf==0.3.2 # via -r requirements.in, validata-core
flask==1.1.2 # via -r requirements.in, sentry-sdk
frictionless==3.48.0 # via -r requirements.in, validata-core
idna==2.10 # via requests
importlib-metadata==3.3.0 # via jsonschema
importlib-resources==3.3.0 # via validata-core
isodate==0.6.0 # via frictionless
itsdangerous==1.1.0 # via flask
jdcal==1.4.1 # via openpyxl
jinja2==2.11.2 # via flask
jsonschema==3.2.0 # via frictionless, opendataschema
lxml==4.6.2 # via -r requirements.in, validata-core
markupsafe==1.1.1 # via jinja2
msgpack==1.0.2 # via cachecontrol
numpy==1.19.4 # via pandas
opendataschema==0.5.5 # via -r requirements.in
openpyxl==3.0.5 # via validata-core
pandas==1.1.5 # via tablib
petl==1.6.8 # via frictionless
pydantic==1.7.3 # via -r requirements.in
pygithub==1.54 # via opendataschema
pyjwt==1.7.1 # via pygithub
pyrsistent==0.17.3 # via jsonschema
python-dateutil==2.8.1 # via frictionless, pandas
python-dotenv==0.15.0 # via -r requirements.in
python-gitlab==2.5.0 # via opendataschema
python-slugify==4.0.1 # via frictionless
python-stdnum==1.14 # via validata-core
pytz==2020.4 # via pandas
pyyaml==5.3.1 # via -r requirements.in, frictionless
requests==2.24.0 # via -r requirements.in, cachecontrol, frictionless, opendataschema, pygithub, python-gitlab, validata-core
sentry-sdk[flask]==0.14.3 # via -r requirements.in
shellingham==1.3.2 # via typer
simpleeval==0.9.10 # via frictionless
six==1.15.0 # via isodate, jsonschema, python-dateutil, validators
stringcase==1.2.0 # via frictionless
tablib[pandas]==3.0.0 # via validata-core
text-unidecode==1.3 # via python-slugify
toml==0.10.2 # via -r requirements.in
toolz==0.11.1 # via validata-core
typer[all]==0.3.2 # via frictionless
typing-extensions==3.7.4.3 # via importlib-metadata
urllib3==1.25.11 # via requests, sentry-sdk
validators==0.18.1 # via frictionless
werkzeug==1.0.1 # via flask
wrapt==1.12.1 # via deprecated
zipp==3.4.0 # via importlib-metadata, importlib-resources
-e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core
# via -r requirements.in
attrs==20.3.0
# via jsonschema
blinker==1.4
# via sentry-sdk
certifi==2020.12.5
# via
# requests
# sentry-sdk
chardet==3.0.4
# via
# frictionless
# requests
click==7.1.2
# via
# flask
# opendataschema
# typer
colorama==0.4.4
# via typer
commonmark==0.9.1
# via -r requirements.in
decorator==4.4.2
# via validators
deprecated==1.2.10
# via pygithub
ezodf==0.3.2
# via -r requirements.in
flask==1.1.2
# via
# -r requirements.in
# sentry-sdk
frictionless==3.48.0
# via -r requirements.in
idna==2.10
# via requests
isodate==0.6.0
# via frictionless
itsdangerous==1.1.0
# via flask
jinja2==2.11.2
# via flask
jsonschema==3.2.0
# via
# frictionless
# opendataschema
lxml==4.6.2
# via -r requirements.in
markupsafe==1.1.1
# via jinja2
opendataschema==0.5.5
# via -r requirements.in
petl==1.6.8
# via frictionless
pydantic==1.7.3
# via -r requirements.in
pygithub==1.54
# via opendataschema
pyjwt==1.7.1
# via pygithub
pyrsistent==0.17.3
# via jsonschema
python-dateutil==2.8.1
# via frictionless
python-dotenv==0.15.0
# via -r requirements.in
python-gitlab==2.5.0
# via opendataschema
python-slugify==4.0.1
# via frictionless
pyyaml==5.3.1
# via
# -r requirements.in
# frictionless
requests-cache==0.5.2
# via -r requirements.in
requests==2.24.0
# via
# -r requirements.in
# frictionless
# opendataschema
# pygithub
# python-gitlab
# requests-cache
sentry-sdk[flask]==0.14.3
# via -r requirements.in
shellingham==1.3.2
# via typer
simpleeval==0.9.10
# via frictionless
six==1.15.0
# via
# isodate
# jsonschema
# python-dateutil
# validators
stringcase==1.2.0
# via frictionless
text-unidecode==1.3
# via python-slugify
toml==0.10.2
# via -r requirements.in
typer[all]==0.3.2
# via frictionless
urllib3==1.25.11
# via
# requests
# sentry-sdk
validators==0.18.1
# via frictionless
werkzeug==1.0.1
# via flask
wrapt==1.12.1
# via deprecated
# The following packages are considered to be unsafe in a requirements file:
# setuptools
validata_ui/__init__.py
View file @ 26d5aed0
import logging
from datetime import timedelta
import cachecontrol
import flask
import frictionless
import jinja2
import opendataschema
import pkg_resources
import requests
import requests_cache
from commonmark import commonmark
from pydantic import HttpUrl
......@@ -45,7 +45,10 @@ class SchemaCatalogRegistry:
return opendataschema.SchemaCatalog(ref, session=self.session)
caching_session = cachecontrol.CacheControl(requests.Session())
expire_after = timedelta(minutes=config.CACHING_EXPIRE_AFTER)
caching_session = requests_cache.CachedSession(
backend="sqlite", cache_name="validata_ui_cache", expire_after=expire_after
)
fetch_schema = generate_schema_from_url_func(caching_session)
# And load schema catalogs which URLs are found in 'homepage' key of config.yaml
......
validata_ui/config.py
View file @ 26d5aed0
"""Configuration stuff."""
import yaml
import logging
import os
import sys
......@@ -8,6 +7,7 @@ from pathlib import Path
import requests
import toml
import yaml
from dotenv import load_dotenv
from pydantic.error_wrappers import ValidationError
......@@ -84,3 +84,18 @@ SENTRY_DSN = os.environ.get("SENTRY_DSN")
# PDF generation service
BROWSERLESS_API_URL = os.getenv("BROWSERLESS_API_URL") or None
BROWSERLESS_API_TOKEN = os.getenv("BROWSERLESS_API_TOKEN") or None
# Caching time for schema requests in minutes
CACHING_EXPIRE_AFTER = os.getenv("CACHING_EXPIRE_AFTER") or None
if CACHING_EXPIRE_AFTER is None:
log.info(
"CACHING_EXPIRE_AFTER environment variable not set, using default 0 (no cache)"
)
CACHING_EXPIRE_AFTER = "0"
try:
CACHING_EXPIRE_AFTER = int(CACHING_EXPIRE_AFTER)
log.info("Caching is set to %sm", CACHING_EXPIRE_AFTER)
except ValueError as exc:
raise ValueError(
f"Invalid number of minutes for caching: {CACHING_EXPIRE_AFTER}"
) from exc
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

OpenDataFrance.net