alpha runner

src/db.py

@@ -1,10 +1,10 @@
 import psycopg2 as pg
 from os import environ as env
-
+from transform import vals_in_order
-from extract import vals_in_order
 
 
 def connect():
+    print("connecting")
     return pg.connect(
         dbname=env.get("POSTGRES_DB"),
         user=env.get("POSTGRES_USER"),
@@ -14,10 +14,17 @@ def connect():
     )
 
 
-def get_all_profile_names(conn=None):
+def connect_by_default(func, conn_func=connect):
-    if not conn:
+    def wrapper(*args, **kargs):
-        conn = connect()
+        if not kargs.get("conn"):
+            kargs["conn"] = conn_func()
+        return func(*args, **kargs)
 
+    return wrapper
+
+
+@connect_by_default
+def get_all_profile_names(conn=None):
     with conn.cursor() as curs:
         curs.execute("""SELECT name, leetify_user_id FROM data.profile_meta;""")
         data = curs.fetchall()
@@ -25,10 +32,8 @@ def get_all_profile_names(conn=None):
     return data
 
 
+@connect_by_default
 def get_profile_game_count(user_id, conn=None):
-    if not conn:
-        conn = connect()
-
     with conn.cursor() as curs:
         curs.execute(
             """SELECT count(*) FROM data.profile_game
@@ -42,10 +47,8 @@ def get_profile_game_count(user_id, conn=None):
     return data[0]
 
 
+@connect_by_default
 def get_profile_game_ids(user_id, conn=None):
-    if not conn:
-        conn = connect()
-
     with conn.cursor() as curs:
         curs.execute(
             """SELECT game_id FROM data.profile_game
@@ -57,10 +60,8 @@ def get_profile_game_ids(user_id, conn=None):
     return [row[0] for row in data]
 
 
+@connect_by_default
 def insert_games(games, conn=None):
-    if not conn:
-        conn = connect()
-
     cols_api = [
         "leetifyUserId",
         "ctLeetifyRating",

src/extract.py

@@ -3,45 +3,16 @@ from transform import (
     meta_from_profile,
     games_from_profile,
     player_stats_from_game,
+    vals_in_order,
 )
 import db
-from typing import Tuple
 import logging
-from itertools import chain
 
 ## TODO seperate out loading from extraction
 ## this currently handles getting data from api and loading into db
 
-
+@db.connect_by_default
-def vals_in_order(data: dict, keys: list):
-    vals = [data[key] for key in keys]
-    return vals
-
-
-def dict_to_col_val(data: dict) -> Tuple[list, list]:
-    cols = []
-    vals = []
-    for k, v in data.items():
-        cols.append(k)
-        vals.append(v)
-
-    return cols, vals
-
-
-def dict_to_sql_str(data: dict) -> Tuple[str, str]:
-    cols, vals = dict_to_col_val(data)
-    vals = map(str, vals)
-
-    cols = ", ".join(cols)
-    vals = ", ".join(vals)
-
-    return cols, vals
-
-
 def extract_profile_meta(id, api=Leetify(), conn=None):
-    if not conn:
-        conn = db.connect()
-
     cols_api = [
         "name",
         "steam64Id",
@@ -74,10 +45,8 @@ def extract_profile_meta(id, api=Leetify(), conn=None):
     conn.commit()
 
 
+@db.connect_by_default
 def extract_profile_games(profile, conn=None):
-    if not conn:
-        conn = db.connect()
-
     cols_api = [
         "leetifyUserId",
         "ctLeetifyRating",
@@ -134,10 +103,8 @@ def extract_profile_games(profile, conn=None):
     conn.commit()
 
 
+@db.connect_by_default
 def extract_player_stats(game, conn=None):
-    if not conn:
-        conn = db.connect()
-
     cols_api = [
         "id",
         "gameId",
@@ -318,58 +285,56 @@ def extract_player_stats(game, conn=None):
     conn.commit()
 
 
+@db.connect_by_default
 def insert_new_profile_games(games, user_id, conn=None):
-    if not conn:
+    game_ids = db.get_profile_game_ids(user_id, conn=conn)
-        conn = db.connect()
-
-    game_ids = db.get_profile_game_ids(user_id, conn)
 
     game_ids = set(game_ids)
     new_games = [game for game in games if game.get("gameId") not in game_ids]
 
     logging.info(f"Inserting {len(new_games)} for: {user_id}")
-    db.insert_games(new_games)
+    db.insert_games(new_games, conn=conn)
 
     return new_games
 
 
-def insert_player_stats(game_ids, conn=None, api=Leetify()):
+@db.connect_by_default
-    if not conn:
+def insert_player_stats(game_ids, api=Leetify(), conn=None):
-        conn = db.connect()
+    logging.info(f"Inserting player_stats from {len(game_ids)} games")
-
+    print_every = int(len(game_ids) / 10)
-    player_stats = map(api.get_match, game_ids)
+    for i, id in enumerate(game_ids):
-    player_stats = map(player_stats_from_game, player_stats)
+        if i % print_every == 0:
-
+            logging.info(f"Inserted player_stats from {i}/{len(game_ids)} games")
-    player_stats = chain.from_iterable(player_stats)
+        game = api.get_match(id)
-    print(player_stats)
+        extract_player_stats(game, conn=conn)
 
 
 def get_all(api=Leetify()):
     conn = db.connect()
 
-    data = db.get_all_profile_names(conn)
+    data = db.get_all_profile_names(conn=conn)
-    games = []
+    game_ids = []
     for name, id in data:
         logging.info(f"Getting data for {name}: {id}")
         profile = api.get_profile(id)
-        games = games_from_profile(profile)
+        profile_games = games_from_profile(profile)
 
-        api_games = len(games)
+        api_games = len(profile_games)
-        db_games = db.get_profile_game_count(id, conn)
+        db_games = db.get_profile_game_count(id, conn=conn)
 
         if api_games > db_games:
             logging.info(f"Getting new games for {name}: {id}")
-            new_games = insert_new_profile_games(games, id, conn)
+            new_games = insert_new_profile_games(profile_games, id, conn=conn)
-            games.extend(new_games)
+            new_game_ids = [game.get("gameId") for game in new_games]
+            game_ids.extend(new_game_ids)
         elif api_games < db_games:
             logging.error(f"API returned less games then in DB for {name}: {id}")
 
         logging.info(f"Games synced with Leetify for {name}: {id}")
 
-    game_ids = [game.get("game_id") for game in games]
+    logging.info(f"Updating player_stats with {len(game_ids)} new games")
-    insert_player_stats(game_ids, conn)
+    insert_player_stats(game_ids, conn=conn)
-
+    logging.info("DONE Updating player_stats with new games")
-    # TODO get  sync player_stats and profile_games
 
 
 if __name__ == "__main__":

src/transform.py

@@ -4,6 +4,11 @@ T = TypeVar("T")
 U = TypeVar("U")
 
 
+def vals_in_order(data: dict, keys: list):
+    vals = [data[key] for key in keys]
+    return vals
+
+
 def extract_cols(data: dict, cols: list[str]) -> dict:
     return {key: data.get(key) for key in cols}
 
@@ -183,6 +188,5 @@ def player_stats_from_game(game: dict) -> list[dict]:
     if not stats:
         raise Exception("Could not get stats from game", game)
 
-
+    stats = map(cols_from_player_stats, stats)
-    stats = map(cols_from_player_stats,stats)
     return list(stats)