Commit ab56bb02 authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Fix encoding issue

parent bdd2dcb5
......@@ -7,7 +7,8 @@ from io import BytesIO
from pathlib import Path
from urllib.parse import urlparse
from frictionless.helpers import detect_encoding
from frictionless import Control
from validata_core.helpers import detect_encoding
log = logging.getLogger(__name__)
......@@ -18,6 +19,7 @@ class ValidataResource(ABC):
def __init__(self, type_):
self.type = type_
# TODO: merge with validata_core.helpers.prepare_resource!
@abstractmethod
def build_stream_args(self):
"""return (source, option_dict)"""
......@@ -35,7 +37,7 @@ class URLValidataResource(ValidataResource):
def build_stream_args(self):
"""URL implementation"""
return (self.url, {})
return (self.url, {"control": Control(detect_encoding=detect_encoding) })
class UploadedFileValidataResource(ValidataResource):
......@@ -58,15 +60,18 @@ class UploadedFileValidataResource(ValidataResource):
return None
format = detect_format_from_file_extension(self.filename)
options = { "format": format }
if format in {'csv', 'tsv'}:
scheme = 'text'
encoding = detect_encoding(self.content)
source = self.content.decode(encoding)
options["encoding"] = encoding
else:
scheme = "filelike"
source = BytesIO(self.content)
options["scheme"] = scheme
return (source, { "scheme": scheme, "format": format})
return (source, options)
def strip_accents(s):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment