diff --git a/src/db.py b/src/db.py index 7c6e1c9..c03e55e 100644 --- a/src/db.py +++ b/src/db.py @@ -60,6 +60,43 @@ def get_profile_game_ids(user_id, conn=None): return [row[0] for row in data] +@connect_by_default +def insert_player_stats(stats, conn=None): + cols_api, cols_db = get_cols("player_stats") + if type(cols_api) != list or type(cols_db) != list: + raise Exception( + "could not get columns for player_stats:", cols_api, cols_db + ) + + vals = map(lambda game: vals_in_order(game, cols_api), stats) + + with conn.cursor() as curs: + sql = f""" + INSERT into data.player_stats ({', '.join(cols_db)}) + VALUES ({', '.join(['%s'] * len(cols_db))}); """ + curs.executemany(sql, vals) + conn.commit() + + +@connect_by_default +def insert_profile_meta(meta, conn=None): + cols_api, cols_db = get_cols("profile_meta") + if type(cols_api) != list or type(cols_db) != list: + raise Exception( + "could not get columns for profile_meta:", cols_api, cols_db + ) + + vals = vals_in_order(meta, cols_api) + sql = f""" + INSERT INTO data.profile_meta ({', '.join(cols_db)}) + VALUES ({', '.join(['%s'] * len(cols_db))}); + """ + + with conn.cursor() as curs: + curs.execute(sql, vals) + conn.commit() + + @connect_by_default def insert_profile_games(games, conn=None): cols_api, cols_db = get_cols("profile_game") @@ -69,11 +106,11 @@ def insert_profile_games(games, conn=None): ) vals = map(lambda game: vals_in_order(game, cols_api), games) + sql = f""" + INSERT into data.profile_game ({', '.join(cols_db)}) + VALUES ({', '.join(['%s'] * len(cols_db))}); + """ with conn.cursor() as curs: - sql = f""" - INSERT into data.profile_game ({', '.join(cols_db)}) - VALUES ({', '.join(['%s'] * len(cols_db))}); - """ curs.executemany(sql, vals) conn.commit() diff --git a/src/extract.py b/src/extract.py index 5bbf61a..cb66bfa 100644 --- a/src/extract.py +++ b/src/extract.py @@ -4,14 +4,10 @@ from transform import ( meta_from_profile, games_from_profile, player_stats_from_game, - vals_in_order, ) import db import logging -## TODO seperate out loading from extraction -## this currently handles getting data from api and loading into db - @db.connect_by_default def extract_profile_meta(id, api=Leetify(), conn=None): @@ -23,15 +19,8 @@ def extract_profile_meta(id, api=Leetify(), conn=None): profile = api.get_profile(id) meta = meta_from_profile(profile) - vals = vals_in_order(meta, cols_api) - with conn.cursor() as curs: - sql = f""" - INSERT INTO data.profile_meta ({', '.join(cols_db)}) - VALUES ({', '.join(['%s'] * len(cols_db))}); - """ - curs.execute(sql, vals) - conn.commit() + db.insert_profile_meta(meta, conn=conn) @db.connect_by_default @@ -43,36 +32,15 @@ def extract_profile_games(profile, conn=None): ) games = games_from_profile(profile) - games = map(lambda game: vals_in_order(game, cols_api), games) - with conn.cursor() as curs: - curs.executemany( - f""" - INSERT into data.profile_game ({', '.join(cols_db)}) - VALUES ({', '.join(['%s'] * len(cols_db))}); - """, - games, - ) - conn.commit() + db.insert_profile_games(games, conn) @db.connect_by_default def extract_player_stats(game, conn=None): - cols_api, cols_db = get_cols("player_stats") - if type(cols_api) != list or type(cols_db) != list: - raise Exception( - "could not get columns for player_stats:", cols_api, cols_db - ) - stats = player_stats_from_game(game) - vals = map(lambda game: vals_in_order(game, cols_api), stats) - with conn.cursor() as curs: - sql = f""" - INSERT into data.player_stats ({', '.join(cols_db)}) - VALUES ({', '.join(['%s'] * len(cols_db))}); """ - curs.executemany(sql, vals) - conn.commit() + db.insert_player_stats(stats, conn=conn) @db.connect_by_default