Commit 26d5aed0 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Replace cache_control by requests_cache

parent 5af4e43e
...@@ -24,3 +24,6 @@ CONFIG=config.example.yaml ...@@ -24,3 +24,6 @@ CONFIG=config.example.yaml
# #
# BROWSERLESS_API_URL="https://chrome.browserless.io/pdf" # BROWSERLESS_API_URL="https://chrome.browserless.io/pdf"
# BROWSERLESS_API_TOKEN="XXX" # BROWSERLESS_API_TOKEN="XXX"
# Number of minutes to cache downloaded schemas
# CACHING_EXPIRE_AFTER=0
\ No newline at end of file
__pycache__/ __pycache__/
validata_ui.egg-info/ validata_ui.egg-info/
validata_ui_cache.sqlite
.env .env
.mypy_cache/ .mypy_cache/
.pytest_cache/ .pytest_cache/
......
-e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core -e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core
cachecontrol
commonmark commonmark
ezodf ezodf
flask flask
...@@ -9,6 +8,7 @@ pydantic ...@@ -9,6 +8,7 @@ pydantic
python-dotenv python-dotenv
pyyaml pyyaml
requests requests
requests_cache
sentry-sdk[flask] == 0.14.3 sentry-sdk[flask] == 0.14.3
toml toml
opendataschema >= 0.5.5, < 0.6 opendataschema >= 0.5.5, < 0.6
......
...@@ -4,65 +4,121 @@ ...@@ -4,65 +4,121 @@
# #
# pip-compile # pip-compile
# #
-e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core # via -r requirements.in -e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core
attrs==20.3.0 # via jsonschema # via -r requirements.in
blinker==1.4 # via sentry-sdk attrs==20.3.0
cachecontrol==0.12.6 # via -r requirements.in # via jsonschema
certifi==2020.12.5 # via requests, sentry-sdk blinker==1.4
chardet==3.0.4 # via frictionless, requests # via sentry-sdk
click==7.1.2 # via flask, opendataschema, typer certifi==2020.12.5
colorama==0.4.4 # via typer # via
commonmark==0.9.1 # via -r requirements.in # requests
decorator==4.4.2 # via validators # sentry-sdk
deprecated==1.2.10 # via pygithub chardet==3.0.4
et-xmlfile==1.0.1 # via openpyxl # via
ezodf==0.3.2 # via -r requirements.in, validata-core # frictionless
flask==1.1.2 # via -r requirements.in, sentry-sdk # requests
frictionless==3.48.0 # via -r requirements.in, validata-core click==7.1.2
idna==2.10 # via requests # via
importlib-metadata==3.3.0 # via jsonschema # flask
importlib-resources==3.3.0 # via validata-core # opendataschema
isodate==0.6.0 # via frictionless # typer
itsdangerous==1.1.0 # via flask colorama==0.4.4
jdcal==1.4.1 # via openpyxl # via typer
jinja2==2.11.2 # via flask commonmark==0.9.1
jsonschema==3.2.0 # via frictionless, opendataschema # via -r requirements.in
lxml==4.6.2 # via -r requirements.in, validata-core decorator==4.4.2
markupsafe==1.1.1 # via jinja2 # via validators
msgpack==1.0.2 # via cachecontrol deprecated==1.2.10
numpy==1.19.4 # via pandas # via pygithub
opendataschema==0.5.5 # via -r requirements.in ezodf==0.3.2
openpyxl==3.0.5 # via validata-core # via -r requirements.in
pandas==1.1.5 # via tablib flask==1.1.2
petl==1.6.8 # via frictionless # via
pydantic==1.7.3 # via -r requirements.in # -r requirements.in
pygithub==1.54 # via opendataschema # sentry-sdk
pyjwt==1.7.1 # via pygithub frictionless==3.48.0
pyrsistent==0.17.3 # via jsonschema # via -r requirements.in
python-dateutil==2.8.1 # via frictionless, pandas idna==2.10
python-dotenv==0.15.0 # via -r requirements.in # via requests
python-gitlab==2.5.0 # via opendataschema isodate==0.6.0
python-slugify==4.0.1 # via frictionless # via frictionless
python-stdnum==1.14 # via validata-core itsdangerous==1.1.0
pytz==2020.4 # via pandas # via flask
pyyaml==5.3.1 # via -r requirements.in, frictionless jinja2==2.11.2
requests==2.24.0 # via -r requirements.in, cachecontrol, frictionless, opendataschema, pygithub, python-gitlab, validata-core # via flask
sentry-sdk[flask]==0.14.3 # via -r requirements.in jsonschema==3.2.0
shellingham==1.3.2 # via typer # via
simpleeval==0.9.10 # via frictionless # frictionless
six==1.15.0 # via isodate, jsonschema, python-dateutil, validators # opendataschema
stringcase==1.2.0 # via frictionless lxml==4.6.2
tablib[pandas]==3.0.0 # via validata-core # via -r requirements.in
text-unidecode==1.3 # via python-slugify markupsafe==1.1.1
toml==0.10.2 # via -r requirements.in # via jinja2
toolz==0.11.1 # via validata-core opendataschema==0.5.5
typer[all]==0.3.2 # via frictionless # via -r requirements.in
typing-extensions==3.7.4.3 # via importlib-metadata petl==1.6.8
urllib3==1.25.11 # via requests, sentry-sdk # via frictionless
validators==0.18.1 # via frictionless pydantic==1.7.3
werkzeug==1.0.1 # via flask # via -r requirements.in
wrapt==1.12.1 # via deprecated pygithub==1.54
zipp==3.4.0 # via importlib-metadata, importlib-resources # via opendataschema
pyjwt==1.7.1
# via pygithub
pyrsistent==0.17.3
# via jsonschema
python-dateutil==2.8.1
# via frictionless
python-dotenv==0.15.0
# via -r requirements.in
python-gitlab==2.5.0
# via opendataschema
python-slugify==4.0.1
# via frictionless
pyyaml==5.3.1
# via
# -r requirements.in
# frictionless
requests-cache==0.5.2
# via -r requirements.in
requests==2.24.0
# via
# -r requirements.in
# frictionless
# opendataschema
# pygithub
# python-gitlab
# requests-cache
sentry-sdk[flask]==0.14.3
# via -r requirements.in
shellingham==1.3.2
# via typer
simpleeval==0.9.10
# via frictionless
six==1.15.0
# via
# isodate
# jsonschema
# python-dateutil
# validators
stringcase==1.2.0
# via frictionless
text-unidecode==1.3
# via python-slugify
toml==0.10.2
# via -r requirements.in
typer[all]==0.3.2
# via frictionless
urllib3==1.25.11
# via
# requests
# sentry-sdk
validators==0.18.1
# via frictionless
werkzeug==1.0.1
# via flask
wrapt==1.12.1
# via deprecated
# The following packages are considered to be unsafe in a requirements file: # The following packages are considered to be unsafe in a requirements file:
# setuptools # setuptools
import logging import logging
from datetime import timedelta
import cachecontrol
import flask import flask
import frictionless import frictionless
import jinja2 import jinja2
import opendataschema import opendataschema
import pkg_resources import pkg_resources
import requests import requests_cache
from commonmark import commonmark from commonmark import commonmark
from pydantic import HttpUrl from pydantic import HttpUrl
...@@ -45,7 +45,10 @@ class SchemaCatalogRegistry: ...@@ -45,7 +45,10 @@ class SchemaCatalogRegistry:
return opendataschema.SchemaCatalog(ref, session=self.session) return opendataschema.SchemaCatalog(ref, session=self.session)
caching_session = cachecontrol.CacheControl(requests.Session()) expire_after = timedelta(minutes=config.CACHING_EXPIRE_AFTER)
caching_session = requests_cache.CachedSession(
backend="sqlite", cache_name="validata_ui_cache", expire_after=expire_after
)
fetch_schema = generate_schema_from_url_func(caching_session) fetch_schema = generate_schema_from_url_func(caching_session)
# And load schema catalogs which URLs are found in 'homepage' key of config.yaml # And load schema catalogs which URLs are found in 'homepage' key of config.yaml
......
"""Configuration stuff.""" """Configuration stuff."""
import yaml
import logging import logging
import os import os
import sys import sys
...@@ -8,6 +7,7 @@ from pathlib import Path ...@@ -8,6 +7,7 @@ from pathlib import Path
import requests import requests
import toml import toml
import yaml
from dotenv import load_dotenv from dotenv import load_dotenv
from pydantic.error_wrappers import ValidationError from pydantic.error_wrappers import ValidationError
...@@ -84,3 +84,18 @@ SENTRY_DSN = os.environ.get("SENTRY_DSN") ...@@ -84,3 +84,18 @@ SENTRY_DSN = os.environ.get("SENTRY_DSN")
# PDF generation service # PDF generation service
BROWSERLESS_API_URL = os.getenv("BROWSERLESS_API_URL") or None BROWSERLESS_API_URL = os.getenv("BROWSERLESS_API_URL") or None
BROWSERLESS_API_TOKEN = os.getenv("BROWSERLESS_API_TOKEN") or None BROWSERLESS_API_TOKEN = os.getenv("BROWSERLESS_API_TOKEN") or None
# Caching time for schema requests in minutes
CACHING_EXPIRE_AFTER = os.getenv("CACHING_EXPIRE_AFTER") or None
if CACHING_EXPIRE_AFTER is None:
log.info(
"CACHING_EXPIRE_AFTER environment variable not set, using default 0 (no cache)"
)
CACHING_EXPIRE_AFTER = "0"
try:
CACHING_EXPIRE_AFTER = int(CACHING_EXPIRE_AFTER)
log.info("Caching is set to %sm", CACHING_EXPIRE_AFTER)
except ValueError as exc:
raise ValueError(
f"Invalid number of minutes for caching: {CACHING_EXPIRE_AFTER}"
) from exc
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment