Commit 77a3a8aa authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Use logging module

parent 4a33202a
#!/usr/bin/env python3
#
#
"""
Imports a CSV file into an existing SQLite DB
"""
import argparse
import collections
import csv
import logging
import re
import sqlite3
import sys
......@@ -27,6 +26,8 @@ DELIMITER_NAME_TO_CHAR = collections.OrderedDict(
]
)
log = logging.getLogger(__name__)
class SqliteTableHelper:
"""Ease SQLite table handling"""
......@@ -56,6 +57,7 @@ class SqliteTableHelper:
sql_tpl = "CREATE TABLE {} ({})".format(
self.table_name, ", ".join(table_constraints)
)
log.debug(sql_tpl)
self.cur.execute(sql_tpl)
def flush_insert(self, data_buffer, doit):
......@@ -65,9 +67,9 @@ class SqliteTableHelper:
try:
self.cur.executemany(self.sql_insert, data_buffer)
except sqlite3.ProgrammingError:
print("PROBLEM !!!!")
log.exception()
for i, row in enumerate(data_buffer):
print("#{} ({}) {}".format(i, len(row), row))
log.debug("#%d (%d) %r", i, len(row), row)
self._inserted_rows_nb += len(data_buffer)
data_buffer.clear()
......@@ -119,10 +121,11 @@ def import_csv(csv_filepath: Path, table_name, conn, csv_delimiter, col_type):
try:
val = int(val) if val != "" else 0
except ValueError:
print(
"Line {}: {}: Incorrect integer value [{}]".format(
row_id, headers[i], val
)
log.warning(
"Line %d: %s: Incorrect integer value %r",
row_id,
headers[i],
val,
)
val = 0
......@@ -130,17 +133,18 @@ def import_csv(csv_filepath: Path, table_name, conn, csv_delimiter, col_type):
try:
val = float(val) if val != "" else 0.0
except ValueError:
print(
"Line {}: {}: Incorrect float value [{}]".format(
row_id, headers[i], val
)
log.error(
"Line %d: %d: Incorrect float value %r",
row_id,
headers[i],
val,
)
val = 0.0
typed_row.append(val)
return typed_row
print("Importing {} into table {}...".format(str(csv_filepath), table_name))
log.info("Importing %s into table %s...", str(csv_filepath), table_name)
cur = conn.cursor()
cur.execute("BEGIN TRANSACTION")
......@@ -169,17 +173,15 @@ def import_csv(csv_filepath: Path, table_name, conn, csv_delimiter, col_type):
if col_type:
for index, sqltype in col_type.items():
if index >= len(header):
print(
" Unknown column #{} to treat as {}, stopping here.".format(
index, sqltype
)
log.error(
" Unknown column #%d to treat as %s, stopping here.",
index,
sqltype,
)
sys.exit(1)
else:
print(
" Column {} is treated as {}".format(
header[index], sqltype
)
log.debug(
" Column %s is treated as %s", header[index], sqltype
)
# Creates table
......@@ -191,10 +193,7 @@ def import_csv(csv_filepath: Path, table_name, conn, csv_delimiter, col_type):
if len(row) == len(header):
row_buffer.append(type_row_values(row, col_type, cp, header))
else:
print(
"WARN: bad number of columns on row #{}, ignoring data".format(cp),
file=sys.stderr,
)
log.warning("Bad number of columns on row #%d, ignoring data", cp)
# Flush?
sth.flush_insert(row_buffer, cp != 0 and cp % sql_batch_rows_nb == 0)
......@@ -233,8 +232,19 @@ def main():
"--float_cols",
help='Indexes (starting at 0 and separated by ",") of columns containing float data',
)
parser.add_argument("--log", default="INFO", help="level of logging messages")
args = parser.parse_args()
# Logging settings
numeric_level = getattr(logging, args.log.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError("Invalid log level: {}".format(args.log))
logging.basicConfig(
format="%(levelname)s:%(name)s:%(asctime)s:%(message)s",
level=numeric_level,
stream=sys.stdout, # Use stderr if script outputs data to stdout.
)
# To avoid "field larger than field limit (131072)" error
csv.field_size_limit(sys.maxsize)
......@@ -243,7 +253,7 @@ def main():
# DB connection
db_file_name = str(args.db_file)
print("Populating {} database".format(db_file_name))
log.info("Populating %s database", db_file_name)
conn = sqlite3.connect(db_file_name)
# Table name overloading
......@@ -278,7 +288,7 @@ def main():
# Close
conn.close()
print("Done ({} rows imported in {:.2f}s).".format(rows_nb, end - start))
log.info("Done (%d rows imported in %.2fs).", rows_nb, end - start)
if __name__ == "__main__":
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment