requirments.txt

@@ -3,5 +3,3 @@ click
 psycopg2-binary
 ratelimit
 backoff
-fastapi
-python-multipart

src/api.py

@@ -1,55 +0,0 @@
-from fastapi import FastAPI, status
-from transform import get_db_cols
-from datetime import datetime
-import db
-
-
-app = FastAPI()
-
-
-@app.get("/players")
-async def get_players():
-    cols = get_db_cols("profile_meta")
-    profiles = db.get_all_profiles()
-    profiles = map(
-        lambda profile: {cols[i]: profile[i] for i, _ in enumerate(profile)},
-        profiles,
-    )
-
-    return list(profiles)
-
-
-@app.get("/players/{player_id}/games")
-async def get_player_games(
-    player_id,
-    begin: datetime | None = None,
-    end: datetime | None = None,
-):
-    print(begin)
-    print(end)
-
-    game_ids = db.get_profile_game_ids(player_id, begin=begin, end=end)
-    print(len(game_ids))
-    if len(game_ids) == 0:
-        return status.HTTP_404_NOT_FOUND
-
-    return {"game_ids": game_ids}
-
-
-@app.get("/players/{player_id}/player_stats/")
-async def get_player_stats(player_id: str, game_ids_param: str | None = None):
-    if not game_ids_param or len(game_ids_param) == 0:
-        game_ids = db.get_profile_game_ids(player_id)
-    else:
-        game_ids = [id.strip('" ') for id in game_ids_param.split(",")]
-
-    print(len(game_ids))
-    cols = get_db_cols("player_stats")
-    stats = db.get_player_stats(game_ids)
-    print(len(stats))
-    stats = map(
-        lambda stat: {cols[i]: stat[i] for i, _ in enumerate(stat)},
-        stats,
-    )
-
-    return {"player_id": player_id, "player_stats": list(stats)}

src/db.py

@@ -32,15 +32,6 @@ def get_all_profile_names(conn=None):
     return data
 
 
-@connect_by_default
-def get_all_profiles(conn=None):
-    with conn.cursor() as curs:
-        curs.execute("SELECT * from data.profile_meta;")
-        data = curs.fetchall()
-
-    return data
-
-
 @connect_by_default
 def get_profile_game_count(user_id, conn=None):
     with conn.cursor() as curs:
@@ -57,79 +48,18 @@ def get_profile_game_count(user_id, conn=None):
 
 
 @connect_by_default
-def get_profile_game_ids(user_id, begin=None, end=None, conn=None):
+def get_profile_game_ids(user_id, conn=None):
-    sql = """
-            SELECT game_id FROM data.profile_game
-            WHERE leetify_user_id = %s"""
-    vals = [user_id]
-    if begin:
-        sql += """
-            AND game_finished_at >= (%s)
-        """
-        vals.append(begin)
-    if end:
-        sql += """
-            AND game_finished_at < (%s)
-        """
-        vals.append(end)
-    sql += ";"
     with conn.cursor() as curs:
-        curs.execute(sql, (*vals,))
+        curs.execute(
+            """SELECT game_id FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
         data = curs.fetchall()
 
     return [row[0] for row in data]
 
 
-@connect_by_default
-def get_player_stats(game_ids, conn=None):
-    sql = """
-            SELECT * FROM data.player_stats
-            WHERE game_id = ANY(%s);
-            """
-    with conn.cursor() as curs:
-        curs.execute(sql, (game_ids,))
-        data = curs.fetchall()
-
-    return data
-
-
-@connect_by_default
-def insert_player_stats(stats, conn=None):
-    cols_api, cols_db = get_cols("player_stats")
-    if type(cols_api) != list or type(cols_db) != list:
-        raise Exception(
-            "could not get columns for player_stats:", cols_api, cols_db
-        )
-
-    vals = map(lambda game: vals_in_order(game, cols_api), stats)
-
-    with conn.cursor() as curs:
-        sql = f"""
-              INSERT into data.player_stats ({', '.join(cols_db)})
-              VALUES ({', '.join(['%s'] * len(cols_db))}); """
-        curs.executemany(sql, vals)
-    conn.commit()
-
-
-@connect_by_default
-def insert_profile_meta(meta, conn=None):
-    cols_api, cols_db = get_cols("profile_meta")
-    if type(cols_api) != list or type(cols_db) != list:
-        raise Exception(
-            "could not get columns for profile_meta:", cols_api, cols_db
-        )
-
-    vals = vals_in_order(meta, cols_api)
-    sql = f"""
-    INSERT INTO data.profile_meta ({', '.join(cols_db)})
-    VALUES ({', '.join(['%s'] * len(cols_db))});
-    """
-
-    with conn.cursor() as curs:
-        curs.execute(sql, vals)
-    conn.commit()
-
-
 @connect_by_default
 def insert_profile_games(games, conn=None):
     cols_api, cols_db = get_cols("profile_game")
@@ -139,11 +69,11 @@ def insert_profile_games(games, conn=None):
         )
 
     vals = map(lambda game: vals_in_order(game, cols_api), games)
+
+    with conn.cursor() as curs:
         sql = f"""
             INSERT into data.profile_game ({', '.join(cols_db)})
             VALUES ({', '.join(['%s'] * len(cols_db))});
         """
-
-    with conn.cursor() as curs:
         curs.executemany(sql, vals)
     conn.commit()

src/extract.py

@@ -4,10 +4,14 @@ from transform import (
     meta_from_profile,
     games_from_profile,
     player_stats_from_game,
+    vals_in_order,
 )
 import db
 import logging
 
+## TODO seperate out loading from extraction
+## this currently handles getting data from api and loading into db
+
 
 @db.connect_by_default
 def extract_profile_meta(id, api=Leetify(), conn=None):
@@ -19,8 +23,15 @@ def extract_profile_meta(id, api=Leetify(), conn=None):
 
     profile = api.get_profile(id)
     meta = meta_from_profile(profile)
+    vals = vals_in_order(meta, cols_api)
 
-    db.insert_profile_meta(meta, conn=conn)
+    with conn.cursor() as curs:
+        sql = f"""
+        INSERT INTO data.profile_meta ({', '.join(cols_db)})
+        VALUES ({', '.join(['%s'] * len(cols_db))});
+        """
+        curs.execute(sql, vals)
+    conn.commit()
 
 
 @db.connect_by_default
@@ -32,15 +43,36 @@ def extract_profile_games(profile, conn=None):
         )
 
     games = games_from_profile(profile)
+    games = map(lambda game: vals_in_order(game, cols_api), games)
 
-    db.insert_profile_games(games, conn)
+    with conn.cursor() as curs:
+        curs.executemany(
+            f"""
+            INSERT into data.profile_game ({', '.join(cols_db)})
+            VALUES ({', '.join(['%s'] * len(cols_db))});
+            """,
+            games,
+        )
+    conn.commit()
 
 
 @db.connect_by_default
 def extract_player_stats(game, conn=None):
-    stats = player_stats_from_game(game)
+    cols_api, cols_db = get_cols("player_stats")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for player_stats:", cols_api, cols_db
+        )
 
-    db.insert_player_stats(stats, conn=conn)
+    stats = player_stats_from_game(game)
+    vals = map(lambda game: vals_in_order(game, cols_api), stats)
+
+    with conn.cursor() as curs:
+        sql = f"""
+              INSERT into data.player_stats ({', '.join(cols_db)})
+              VALUES ({', '.join(['%s'] * len(cols_db))}); """
+        curs.executemany(sql, vals)
+    conn.commit()
 
 
 @db.connect_by_default