wip runner for meta and profile games

src/extract.py

@@ -1,25 +1,17 @@
-from .leetify import Leetify
+from leetify import Leetify
-from os import environ as env
+from transform import (
-from .transform import (
     meta_from_profile,
     games_from_profile,
     player_stats_from_game,
 )
-import psycopg2 as pg
+import db
 from typing import Tuple
+import logging
+from itertools import chain
 
 ## TODO seperate out loading from extraction
 ## this currently handles getting data from api and loading into db
 
-def connect():
-    return pg.connect(
-        dbname=env.get("POSTGRES_DB"),
-        user=env.get("POSTGRES_USER"),
-        password=env.get("POSTGRES_PASSWORD"),
-        host=env.get("POSTGRES_HOST"),
-        port=env.get("POSTGRES_PORT"),
-    )
-
 
 def vals_in_order(data: dict, keys: list):
     vals = [data[key] for key in keys]
@@ -48,7 +40,7 @@ def dict_to_sql_str(data: dict) -> Tuple[str, str]:
 
 def extract_profile_meta(id, api=Leetify(), conn=None):
     if not conn:
-        conn = connect()
+        conn = db.connect()
 
     cols_api = [
         "name",
@@ -84,7 +76,7 @@ def extract_profile_meta(id, api=Leetify(), conn=None):
 
 def extract_profile_games(profile, conn=None):
     if not conn:
-        conn = connect()
+        conn = db.connect()
 
     cols_api = [
         "leetifyUserId",
@@ -144,7 +136,7 @@ def extract_profile_games(profile, conn=None):
 
 def extract_player_stats(game, conn=None):
     if not conn:
-        conn = connect()
+        conn = db.connect()
 
     cols_api = [
         "id",
@@ -324,3 +316,62 @@ def extract_player_stats(game, conn=None):
               VALUES ({', '.join(['%s'] * len(cols_db))}); """
         curs.executemany(sql, vals)
     conn.commit()
+
+
+def insert_new_profile_games(games, user_id, conn=None):
+    if not conn:
+        conn = db.connect()
+
+    game_ids = db.get_profile_game_ids(user_id, conn)
+
+    game_ids = set(game_ids)
+    new_games = [game for game in games if game.get("gameId") not in game_ids]
+
+    logging.info(f"Inserting {len(new_games)} for: {user_id}")
+    db.insert_games(new_games)
+
+    return new_games
+
+
+def insert_player_stats(game_ids, conn=None, api=Leetify()):
+    if not conn:
+        conn = db.connect()
+
+    player_stats = map(api.get_match, game_ids)
+    player_stats = map(player_stats_from_game, player_stats)
+
+    player_stats = chain.from_iterable(player_stats)
+    print(player_stats)
+
+
+def get_all(api=Leetify()):
+    conn = db.connect()
+
+    data = db.get_all_profile_names(conn)
+    games = []
+    for name, id in data:
+        logging.info(f"Getting data for {name}: {id}")
+        profile = api.get_profile(id)
+        games = games_from_profile(profile)
+
+        api_games = len(games)
+        db_games = db.get_profile_game_count(id, conn)
+
+        if api_games > db_games:
+            logging.info(f"Getting new games for {name}: {id}")
+            new_games = insert_new_profile_games(games, id, conn)
+            games.extend(new_games)
+        elif api_games < db_games:
+            logging.error(f"API returned less games then in DB for {name}: {id}")
+
+        logging.info(f"Games synced with Leetify for {name}: {id}")
+
+    game_ids = [game.get("game_id") for game in games]
+    insert_player_stats(game_ids, conn)
+
+    # TODO get  sync player_stats and profile_games
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    get_all()