diff --git a/src/db.py b/src/db.py index 32afa6c..e335f2f 100644 --- a/src/db.py +++ b/src/db.py @@ -1,10 +1,10 @@ import psycopg2 as pg from os import environ as env - -from extract import vals_in_order +from transform import vals_in_order def connect(): + print("connecting") return pg.connect( dbname=env.get("POSTGRES_DB"), user=env.get("POSTGRES_USER"), @@ -14,10 +14,17 @@ def connect(): ) -def get_all_profile_names(conn=None): - if not conn: - conn = connect() +def connect_by_default(func, conn_func=connect): + def wrapper(*args, **kargs): + if not kargs.get("conn"): + kargs["conn"] = conn_func() + return func(*args, **kargs) + return wrapper + + +@connect_by_default +def get_all_profile_names(conn=None): with conn.cursor() as curs: curs.execute("""SELECT name, leetify_user_id FROM data.profile_meta;""") data = curs.fetchall() @@ -25,10 +32,8 @@ def get_all_profile_names(conn=None): return data +@connect_by_default def get_profile_game_count(user_id, conn=None): - if not conn: - conn = connect() - with conn.cursor() as curs: curs.execute( """SELECT count(*) FROM data.profile_game @@ -42,10 +47,8 @@ def get_profile_game_count(user_id, conn=None): return data[0] +@connect_by_default def get_profile_game_ids(user_id, conn=None): - if not conn: - conn = connect() - with conn.cursor() as curs: curs.execute( """SELECT game_id FROM data.profile_game @@ -57,10 +60,8 @@ def get_profile_game_ids(user_id, conn=None): return [row[0] for row in data] +@connect_by_default def insert_games(games, conn=None): - if not conn: - conn = connect() - cols_api = [ "leetifyUserId", "ctLeetifyRating", diff --git a/src/extract.py b/src/extract.py index c3c9cfe..057f42b 100644 --- a/src/extract.py +++ b/src/extract.py @@ -3,45 +3,16 @@ from transform import ( meta_from_profile, games_from_profile, player_stats_from_game, + vals_in_order, ) import db -from typing import Tuple import logging -from itertools import chain ## TODO seperate out loading from extraction ## this currently handles getting data from api and loading into db - -def vals_in_order(data: dict, keys: list): - vals = [data[key] for key in keys] - return vals - - -def dict_to_col_val(data: dict) -> Tuple[list, list]: - cols = [] - vals = [] - for k, v in data.items(): - cols.append(k) - vals.append(v) - - return cols, vals - - -def dict_to_sql_str(data: dict) -> Tuple[str, str]: - cols, vals = dict_to_col_val(data) - vals = map(str, vals) - - cols = ", ".join(cols) - vals = ", ".join(vals) - - return cols, vals - - +@db.connect_by_default def extract_profile_meta(id, api=Leetify(), conn=None): - if not conn: - conn = db.connect() - cols_api = [ "name", "steam64Id", @@ -74,10 +45,8 @@ def extract_profile_meta(id, api=Leetify(), conn=None): conn.commit() +@db.connect_by_default def extract_profile_games(profile, conn=None): - if not conn: - conn = db.connect() - cols_api = [ "leetifyUserId", "ctLeetifyRating", @@ -134,10 +103,8 @@ def extract_profile_games(profile, conn=None): conn.commit() +@db.connect_by_default def extract_player_stats(game, conn=None): - if not conn: - conn = db.connect() - cols_api = [ "id", "gameId", @@ -318,58 +285,56 @@ def extract_player_stats(game, conn=None): conn.commit() +@db.connect_by_default def insert_new_profile_games(games, user_id, conn=None): - if not conn: - conn = db.connect() - - game_ids = db.get_profile_game_ids(user_id, conn) + game_ids = db.get_profile_game_ids(user_id, conn=conn) game_ids = set(game_ids) new_games = [game for game in games if game.get("gameId") not in game_ids] logging.info(f"Inserting {len(new_games)} for: {user_id}") - db.insert_games(new_games) + db.insert_games(new_games, conn=conn) return new_games -def insert_player_stats(game_ids, conn=None, api=Leetify()): - if not conn: - conn = db.connect() - - player_stats = map(api.get_match, game_ids) - player_stats = map(player_stats_from_game, player_stats) - - player_stats = chain.from_iterable(player_stats) - print(player_stats) +@db.connect_by_default +def insert_player_stats(game_ids, api=Leetify(), conn=None): + logging.info(f"Inserting player_stats from {len(game_ids)} games") + print_every = int(len(game_ids) / 10) + for i, id in enumerate(game_ids): + if i % print_every == 0: + logging.info(f"Inserted player_stats from {i}/{len(game_ids)} games") + game = api.get_match(id) + extract_player_stats(game, conn=conn) def get_all(api=Leetify()): conn = db.connect() - data = db.get_all_profile_names(conn) - games = [] + data = db.get_all_profile_names(conn=conn) + game_ids = [] for name, id in data: logging.info(f"Getting data for {name}: {id}") profile = api.get_profile(id) - games = games_from_profile(profile) + profile_games = games_from_profile(profile) - api_games = len(games) - db_games = db.get_profile_game_count(id, conn) + api_games = len(profile_games) + db_games = db.get_profile_game_count(id, conn=conn) if api_games > db_games: logging.info(f"Getting new games for {name}: {id}") - new_games = insert_new_profile_games(games, id, conn) - games.extend(new_games) + new_games = insert_new_profile_games(profile_games, id, conn=conn) + new_game_ids = [game.get("gameId") for game in new_games] + game_ids.extend(new_game_ids) elif api_games < db_games: logging.error(f"API returned less games then in DB for {name}: {id}") logging.info(f"Games synced with Leetify for {name}: {id}") - game_ids = [game.get("game_id") for game in games] - insert_player_stats(game_ids, conn) - - # TODO get sync player_stats and profile_games + logging.info(f"Updating player_stats with {len(game_ids)} new games") + insert_player_stats(game_ids, conn=conn) + logging.info("DONE Updating player_stats with new games") if __name__ == "__main__": diff --git a/src/transform.py b/src/transform.py index d8d3a66..693fe13 100644 --- a/src/transform.py +++ b/src/transform.py @@ -4,6 +4,11 @@ T = TypeVar("T") U = TypeVar("U") +def vals_in_order(data: dict, keys: list): + vals = [data[key] for key in keys] + return vals + + def extract_cols(data: dict, cols: list[str]) -> dict: return {key: data.get(key) for key in cols} @@ -183,6 +188,5 @@ def player_stats_from_game(game: dict) -> list[dict]: if not stats: raise Exception("Could not get stats from game", game) - - stats = map(cols_from_player_stats,stats) + stats = map(cols_from_player_stats, stats) return list(stats)