Implement first working database init script
This commit is contained in:
parent
e0ddbaa24b
commit
7ad0df77cd
2
projects/project-3/openapi/.gitignore
vendored
Normal file
2
projects/project-3/openapi/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
tmp/
|
||||||
|
bike-data.db
|
||||||
146
projects/project-3/openapi/db_init.py
Normal file
146
projects/project-3/openapi/db_init.py
Normal file
@ -0,0 +1,146 @@
|
|||||||
|
import os.path
|
||||||
|
import shutil
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import logging
|
||||||
|
|
||||||
|
|
||||||
|
WORKING_DIR = os.getcwd()
|
||||||
|
TMP_DIR = os.path.join(WORKING_DIR, "tmp")
|
||||||
|
|
||||||
|
logging.basicConfig(format="%(asctime)-15s [%(levelname)8s] - %(message)s", level=logging.ERROR)
|
||||||
|
LOG = logging.getLogger("Importer")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ApiExportFile:
|
||||||
|
path: str
|
||||||
|
download_url: str
|
||||||
|
etag: str
|
||||||
|
|
||||||
|
|
||||||
|
def get_online_files_list(subdir_filter=None, file_extension_filter=None):
|
||||||
|
import urllib.parse
|
||||||
|
import xml.etree.ElementTree
|
||||||
|
import requests
|
||||||
|
|
||||||
|
base_uri = "https://s3-eu-west-1.amazonaws.com/cycling.data.tfl.gov.uk/"
|
||||||
|
xml_data = xml.etree.ElementTree.fromstringlist(requests.get(base_uri).text)
|
||||||
|
entries = []
|
||||||
|
|
||||||
|
for child in xml_data.findall('{http://s3.amazonaws.com/doc/2006-03-01/}Contents'):
|
||||||
|
key = child.find('{http://s3.amazonaws.com/doc/2006-03-01/}Key').text
|
||||||
|
etag = child.find('{http://s3.amazonaws.com/doc/2006-03-01/}ETag').text
|
||||||
|
if key.endswith('/'):
|
||||||
|
continue
|
||||||
|
|
||||||
|
download_url = base_uri + urllib.parse.quote_plus(key, safe="/")
|
||||||
|
entries.append(
|
||||||
|
ApiExportFile(key, download_url, etag)
|
||||||
|
)
|
||||||
|
|
||||||
|
if subdir_filter:
|
||||||
|
entries = list(filter(lambda el: el.path.startswith(subdir_filter), entries))
|
||||||
|
|
||||||
|
if file_extension_filter:
|
||||||
|
entries = list(filter(lambda el: el.path.endswith(file_extension_filter), entries))
|
||||||
|
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def download_file(url, save_path):
|
||||||
|
import os.path
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
save_path = os.path.join(TMP_DIR, save_path)
|
||||||
|
|
||||||
|
if os.path.exists(save_path):
|
||||||
|
LOG.warning(f"Skipping exists: {save_path}")
|
||||||
|
return
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(save_path), exist_ok=True)
|
||||||
|
|
||||||
|
LOG.info(f"DOWNLOADING... {url} to {save_path}")
|
||||||
|
urllib.request.urlretrieve(url, save_path)
|
||||||
|
|
||||||
|
|
||||||
|
class BikeDatabase:
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
import sqlite3
|
||||||
|
self.conn = sqlite3.connect("bike-data.db")
|
||||||
|
self.cursor = self.conn.cursor()
|
||||||
|
|
||||||
|
def init_database(self):
|
||||||
|
self.conn.execute("""CREATE TABLE IF NOT EXISTS usage_stats(
|
||||||
|
rental_id INTEGER PRIMARY KEY,
|
||||||
|
duration INTEGER,
|
||||||
|
bike_id INTEGER,
|
||||||
|
end_date INTEGER,
|
||||||
|
end_station_id INTEGER,
|
||||||
|
end_station_name TEXT,
|
||||||
|
start_date INTEGER,
|
||||||
|
start_station_id INTEGER,
|
||||||
|
start_station_name TEXT
|
||||||
|
)""")
|
||||||
|
self.conn.execute("CREATE TABLE IF NOT EXISTS read_files(file_path TEXT, etag TEXT PRIMARY KEY)")
|
||||||
|
|
||||||
|
def is_file_already_imported(self, etag):
|
||||||
|
rows = self.conn.execute("SELECT * FROM read_files WHERE etag LIKE ?", (etag,)).fetchall()
|
||||||
|
return len(rows) != 0
|
||||||
|
|
||||||
|
def import_usage_stats_file(self, export_file: ApiExportFile):
|
||||||
|
from datetime import datetime
|
||||||
|
import csv
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.chdir(TMP_DIR)
|
||||||
|
|
||||||
|
LOG.info(f"Importing {export_file.path}")
|
||||||
|
with open(export_file.path, "r", newline='') as file:
|
||||||
|
LOG.info(f"Reading file {export_file.path}")
|
||||||
|
entries = list(csv.DictReader(file))
|
||||||
|
mapped = []
|
||||||
|
for entry in entries:
|
||||||
|
try:
|
||||||
|
mapped.append((
|
||||||
|
int(entry['Rental Id']),
|
||||||
|
int(entry['Duration'] or "-1"),
|
||||||
|
int(entry['Bike Id'] or "-1"),
|
||||||
|
int(datetime.strptime(entry['End Date'][:16], "%d/%m/%Y %H:%M").timestamp()) if entry['End Date'] else -1,
|
||||||
|
int(entry['EndStation Id'] or "-1"),
|
||||||
|
entry['EndStation Name'],
|
||||||
|
int(datetime.strptime(entry['Start Date'][:16], "%d/%m/%Y %H:%M").timestamp()) if entry['Start Date'] else -1,
|
||||||
|
int(entry['StartStation Id']),
|
||||||
|
entry['StartStation Name']
|
||||||
|
))
|
||||||
|
except ValueError as e:
|
||||||
|
LOG.error(f"Value Error {e} on line {entry}")
|
||||||
|
return
|
||||||
|
except KeyError as e:
|
||||||
|
LOG.error(f"Key Error {e} on line {entry}")
|
||||||
|
return
|
||||||
|
LOG.info(f"Writing {len(mapped)} entries to DB")
|
||||||
|
self.cursor.executemany("INSERT INTO usage_stats VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", mapped)
|
||||||
|
self.cursor.execute("INSERT INTO read_files VALUES (?, ?)", (export_file.path, export_file.etag))
|
||||||
|
self.conn.commit()
|
||||||
|
LOG.info(f"Finished import of {export_file.path}")
|
||||||
|
os.chdir(WORKING_DIR)
|
||||||
|
shutil.rmtree(TMP_DIR)
|
||||||
|
LOG.info("Deleted temp dir")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
all_files = get_online_files_list(subdir_filter="usage-stats", file_extension_filter=".csv")
|
||||||
|
db = BikeDatabase()
|
||||||
|
db.init_database()
|
||||||
|
|
||||||
|
for file in all_files:
|
||||||
|
if not db.is_file_already_imported(file.etag):
|
||||||
|
download_file(file.download_url, file.path)
|
||||||
|
db.import_usage_stats_file(file)
|
||||||
|
else:
|
||||||
|
LOG.warning(f"Skipping import of {file.path}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -1,34 +0,0 @@
|
|||||||
import urllib.parse
|
|
||||||
import urllib.request
|
|
||||||
import xml.etree.ElementTree
|
|
||||||
import os.path
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
BASE_URI = "https://s3-eu-west-1.amazonaws.com/cycling.data.tfl.gov.uk/"
|
|
||||||
BASE_DIR = "data/"
|
|
||||||
|
|
||||||
xml_data = xml.etree.ElementTree.fromstringlist(requests.get(BASE_URI).text)
|
|
||||||
|
|
||||||
for child in xml_data.findall('{http://s3.amazonaws.com/doc/2006-03-01/}Contents'):
|
|
||||||
key = child.find('{http://s3.amazonaws.com/doc/2006-03-01/}Key').text
|
|
||||||
if key.endswith('/'):
|
|
||||||
continue
|
|
||||||
|
|
||||||
parts = key.rsplit('/')
|
|
||||||
# create download url
|
|
||||||
parts.append(urllib.parse.quote_plus(parts.pop()))
|
|
||||||
download_url = BASE_URI + "/".join(parts)
|
|
||||||
|
|
||||||
# create folders and files
|
|
||||||
parts.append(urllib.parse.unquote_plus(parts.pop()))
|
|
||||||
save_path = BASE_DIR + "/".join(parts)
|
|
||||||
os.makedirs(os.path.dirname(save_path), exist_ok=True)
|
|
||||||
|
|
||||||
# skip already downloaded files
|
|
||||||
if os.path.exists(save_path):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# do the download
|
|
||||||
urllib.request.urlretrieve(download_url, save_path)
|
|
||||||
print(f"DOWNLOADING... {download_url} to {save_path}")
|
|
||||||
@ -1,57 +0,0 @@
|
|||||||
import csv
|
|
||||||
import os
|
|
||||||
|
|
||||||
|
|
||||||
def read_files(base_dir, max_num=2):
|
|
||||||
data = []
|
|
||||||
for file in os.listdir(base_dir):
|
|
||||||
if max_num == 0:
|
|
||||||
break
|
|
||||||
print(f'reading file \'{file}\'')
|
|
||||||
with open(os.path.join(base_dir, file), 'r', newline='') as input_file:
|
|
||||||
data.extend(list(csv.DictReader(input_file)))
|
|
||||||
max_num -= 1
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
def get_key_counts(data):
|
|
||||||
all_key_sets = [e.keys() for e in data]
|
|
||||||
key_counts = {}
|
|
||||||
for key_set in all_key_sets:
|
|
||||||
for key in key_set:
|
|
||||||
try:
|
|
||||||
key_counts[key] += 1
|
|
||||||
except KeyError:
|
|
||||||
key_counts[key] = 1
|
|
||||||
return key_counts
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
data = read_files('data/usage-stats', max_num=10)
|
|
||||||
print("Sorting")
|
|
||||||
data = sorted(data, key=lambda entry: int(entry['Rental Id']))
|
|
||||||
print(f'final length of data {len(data)}')
|
|
||||||
|
|
||||||
# counts = get_key_counts(data)
|
|
||||||
# for count, val in counts.items():
|
|
||||||
# if val != len(data):
|
|
||||||
# print(count, val)
|
|
||||||
# print(counts)
|
|
||||||
with open('test.csv', 'w', newline='') as out_file:
|
|
||||||
writer = csv.DictWriter(out_file, data[0].keys(), extrasaction='ignore')
|
|
||||||
writer.writeheader()
|
|
||||||
writer.writerows(data)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
|
|
||||||
|
|
||||||
#
|
|
||||||
# with open('test.csv') as again_in:
|
|
||||||
# reader2 = csv.DictReader(again_in)
|
|
||||||
# data2 = list(reader2)
|
|
||||||
#
|
|
||||||
# print(data2[0])
|
|
||||||
#
|
|
||||||
# print(f"Is same? {data[0] == data2[0]}")
|
|
||||||
Loading…
Reference in New Issue
Block a user