refactor db interactions

src/db.py

@@ -60,6 +60,43 @@ def get_profile_game_ids(user_id, conn=None):
     return [row[0] for row in data]
 
 
+@connect_by_default
+def insert_player_stats(stats, conn=None):
+    cols_api, cols_db = get_cols("player_stats")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for player_stats:", cols_api, cols_db
+        )
+
+    vals = map(lambda game: vals_in_order(game, cols_api), stats)
+
+    with conn.cursor() as curs:
+        sql = f"""
+              INSERT into data.player_stats ({', '.join(cols_db)})
+              VALUES ({', '.join(['%s'] * len(cols_db))}); """
+        curs.executemany(sql, vals)
+    conn.commit()
+
+
+@connect_by_default
+def insert_profile_meta(meta, conn=None):
+    cols_api, cols_db = get_cols("profile_meta")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_meta:", cols_api, cols_db
+        )
+
+    vals = vals_in_order(meta, cols_api)
+    sql = f"""
+    INSERT INTO data.profile_meta ({', '.join(cols_db)})
+    VALUES ({', '.join(['%s'] * len(cols_db))});
+    """
+
+    with conn.cursor() as curs:
+        curs.execute(sql, vals)
+    conn.commit()
+
+
 @connect_by_default
 def insert_profile_games(games, conn=None):
     cols_api, cols_db = get_cols("profile_game")
@@ -69,11 +106,11 @@ def insert_profile_games(games, conn=None):
         )
 
     vals = map(lambda game: vals_in_order(game, cols_api), games)
-
-    with conn.cursor() as curs:
     sql = f"""
         INSERT into data.profile_game ({', '.join(cols_db)})
         VALUES ({', '.join(['%s'] * len(cols_db))});
     """
+
+    with conn.cursor() as curs:
         curs.executemany(sql, vals)
     conn.commit()

src/extract.py

@@ -4,14 +4,10 @@ from transform import (
     meta_from_profile,
     games_from_profile,
     player_stats_from_game,
-    vals_in_order,
 )
 import db
 import logging
 
-## TODO seperate out loading from extraction
-## this currently handles getting data from api and loading into db
-
 
 @db.connect_by_default
 def extract_profile_meta(id, api=Leetify(), conn=None):
@@ -23,15 +19,8 @@ def extract_profile_meta(id, api=Leetify(), conn=None):
 
     profile = api.get_profile(id)
     meta = meta_from_profile(profile)
-    vals = vals_in_order(meta, cols_api)
 
-    with conn.cursor() as curs:
+    db.insert_profile_meta(meta, conn=conn)
-        sql = f"""
-        INSERT INTO data.profile_meta ({', '.join(cols_db)})
-        VALUES ({', '.join(['%s'] * len(cols_db))});
-        """
-        curs.execute(sql, vals)
-    conn.commit()
 
 
 @db.connect_by_default
@@ -43,36 +32,15 @@ def extract_profile_games(profile, conn=None):
         )
 
     games = games_from_profile(profile)
-    games = map(lambda game: vals_in_order(game, cols_api), games)
 
-    with conn.cursor() as curs:
+    db.insert_profile_games(games, conn)
-        curs.executemany(
-            f"""
-            INSERT into data.profile_game ({', '.join(cols_db)})
-            VALUES ({', '.join(['%s'] * len(cols_db))});
-            """,
-            games,
-        )
-    conn.commit()
 
 
 @db.connect_by_default
 def extract_player_stats(game, conn=None):
-    cols_api, cols_db = get_cols("player_stats")
-    if type(cols_api) != list or type(cols_db) != list:
-        raise Exception(
-            "could not get columns for player_stats:", cols_api, cols_db
-        )
-
     stats = player_stats_from_game(game)
-    vals = map(lambda game: vals_in_order(game, cols_api), stats)
 
-    with conn.cursor() as curs:
+    db.insert_player_stats(stats, conn=conn)
-        sql = f"""
-              INSERT into data.player_stats ({', '.join(cols_db)})
-              VALUES ({', '.join(['%s'] * len(cols_db))}); """
-        curs.executemany(sql, vals)
-    conn.commit()
 
 
 @db.connect_by_default