refactored functions to use centeralized column name definition

src/db.py

@@ -0,0 +1,79 @@
+import psycopg2 as pg
+from os import environ as env
+from transform import vals_in_order, get_cols
+
+
+def connect():
+    print("connecting")
+    return pg.connect(
+        dbname=env.get("POSTGRES_DB"),
+        user=env.get("POSTGRES_USER"),
+        password=env.get("POSTGRES_PASSWORD"),
+        host=env.get("POSTGRES_HOST"),
+        port=env.get("POSTGRES_PORT"),
+    )
+
+
+def connect_by_default(func, conn_func=connect):
+    def wrapper(*args, **kargs):
+        if not kargs.get("conn"):
+            kargs["conn"] = conn_func()
+        return func(*args, **kargs)
+
+    return wrapper
+
+
+@connect_by_default
+def get_all_profile_names(conn=None):
+    with conn.cursor() as curs:
+        curs.execute("""SELECT name, leetify_user_id FROM data.profile_meta;""")
+        data = curs.fetchall()
+
+    return data
+
+
+@connect_by_default
+def get_profile_game_count(user_id, conn=None):
+    with conn.cursor() as curs:
+        curs.execute(
+            """SELECT count(*) FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
+        data = curs.fetchone()
+
+    if not data:
+        data = [0]
+    return data[0]
+
+
+@connect_by_default
+def get_profile_game_ids(user_id, conn=None):
+    with conn.cursor() as curs:
+        curs.execute(
+            """SELECT game_id FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
+        data = curs.fetchall()
+
+    return [row[0] for row in data]
+
+
+@connect_by_default
+def insert_profile_games(games, conn=None):
+    cols_api, cols_db = get_cols("profile_game")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
+
+    vals = map(lambda game: vals_in_order(game, cols_api), games)
+
+    with conn.cursor() as curs:
+        sql = f"""
+            INSERT into data.profile_game ({', '.join(cols_db)})
+            VALUES ({', '.join(['%s'] * len(cols_db))});
+        """
+        curs.executemany(sql, vals)
+    conn.commit()

src/extract.py

@@ -1,73 +1,25 @@
-from .leetify import Leetify
-from os import environ as env
-from .transform import (
+from leetify import Leetify
+from transform import (
+    get_cols,
     meta_from_profile,
     games_from_profile,
     player_stats_from_game,
+    vals_in_order,
 )
-import psycopg2 as pg
-from typing import Tuple
+import db
+import logging
 
 ## TODO seperate out loading from extraction
 ## this currently handles getting data from api and loading into db
 
-def connect():
-    return pg.connect(
-        dbname=env.get("POSTGRES_DB"),
-        user=env.get("POSTGRES_USER"),
-        password=env.get("POSTGRES_PASSWORD"),
-        host=env.get("POSTGRES_HOST"),
-        port=env.get("POSTGRES_PORT"),
-    )
-
-
-def vals_in_order(data: dict, keys: list):
-    vals = [data[key] for key in keys]
-    return vals
-
-
-def dict_to_col_val(data: dict) -> Tuple[list, list]:
-    cols = []
-    vals = []
-    for k, v in data.items():
-        cols.append(k)
-        vals.append(v)
-
-    return cols, vals
-
-
-def dict_to_sql_str(data: dict) -> Tuple[str, str]:
-    cols, vals = dict_to_col_val(data)
-    vals = map(str, vals)
-
-    cols = ", ".join(cols)
-    vals = ", ".join(vals)
-
-    return cols, vals
-
 
+@db.connect_by_default
 def extract_profile_meta(id, api=Leetify(), conn=None):
-    if not conn:
-        conn = connect()
-
-    cols_api = [
-        "name",
-        "steam64Id",
-        "isCollector",
-        "isLeetifyStaff",
-        "isProPlan",
-        "leetifyUserId",
-        "faceitNickname",
-    ]
-    cols_db = [
-        "name",
-        "steam64_id",
-        "is_collector",
-        "is_leetify_staff",
-        "is_pro_plan",
-        "leetify_user_id",
-        "faceit_nickname",
-    ]
+    cols_api, cols_db = get_cols("profile_meta")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_meta:", cols_api, cols_db
+        )
 
     profile = api.get_profile(id)
     meta = meta_from_profile(profile)
@@ -82,51 +34,13 @@ def extract_profile_meta(id, api=Leetify(), conn=None):
     conn.commit()
 
 
+@db.connect_by_default
 def extract_profile_games(profile, conn=None):
-    if not conn:
-        conn = connect()
-
-    cols_api = [
-        "leetifyUserId",
-        "ctLeetifyRating",
-        "ctLeetifyRatingRounds",
-        "dataSource",
-        "elo",
-        "gameFinishedAt",
-        "gameId",
-        "isCs2",
-        "mapName",
-        "matchResult",
-        "scores",
-        "skillLevel",
-        "tLeetifyRating",
-        "tLeetifyRatingRounds",
-        "deaths",
-        "hasBannedPlayer",
-        "kills",
-        "partySize",
-    ]
-
-    cols_db = [
-        "leetify_user_id",
-        "ct_Leetify_rating",
-        "ct_Leetify_rating_rounds",
-        "data_source",
-        "elo",
-        "game_finished_at",
-        "game_id",
-        "is_cs2",
-        "map_name",
-        "match_result",
-        "scores",
-        "skill_level",
-        "t_leetify_rating",
-        "t_leetify_rating_rounds",
-        "deaths",
-        "has_banned_player",
-        "kills",
-        "party_size",
-    ]
+    cols_api, cols_db = get_cols("profile_games")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
 
     games = games_from_profile(profile)
     games = map(lambda game: vals_in_order(game, cols_api), games)
@@ -142,180 +56,15 @@ def extract_profile_games(profile, conn=None):
     conn.commit()
 
 
+@db.connect_by_default
 def extract_player_stats(game, conn=None):
-    if not conn:
-        conn = connect()
+    cols_api, cols_db = get_cols("player_stats")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for player_stats:", cols_api, cols_db
+        )
 
-    cols_api = [
-        "id",
-        "gameId",
-        "gameFinishedAt",
-        "steam64Id",
-        "name",
-        "preaim",
-        "reactionTime",
-        "accuracy",
-        "accuracyEnemySpotted",
-        "accuracyHead",
-        "shotsFiredEnemySpotted",
-        "shotsFired",
-        "shotsHitEnemySpotted",
-        "shotsHitFriend",
-        "shotsHitFriendHead",
-        "shotsHitFoe",
-        "shotsHitFoeHead",
-        "utilityOnDeathAvg",
-        "heFoesDamageAvg",
-        "heFriendsDamageAvg",
-        "heThrown",
-        "molotovThrown",
-        "smokeThrown",
-        "smokeThrownCT",
-        "smokeThrownCTGood",
-        "smokeThrownCTGoodRatio",
-        "smokeThrownCTFoes",
-        "counterStrafingShotsAll",
-        "counterStrafingShotsBad",
-        "counterStrafingShotsGood",
-        "counterStrafingShotsGoodRatio",
-        "flashbangHitFoe",
-        "flashbangLeadingToKill",
-        "flashbangHitFoeAvgDuration",
-        "flashbangHitFriend",
-        "flashbangThrown",
-        "flashAssist",
-        "score",
-        "initialTeamNumber",
-        "mvps",
-        "ctRoundsWon",
-        "ctRoundsLost",
-        "tRoundsWon",
-        "tRoundsLost",
-        "sprayAccuracy",
-        "molotovFoesDamageAvg",
-        "molotovFriendsDamageAvg",
-        "color",
-        "totalKills",
-        "totalDeaths",
-        "kdRatio",
-        "multi2k",
-        "multi3k",
-        "multi4k",
-        "multi5k",
-        "hltvRating",
-        "hsp",
-        "roundsSurvived",
-        "roundsSurvivedPercentage",
-        "dpr",
-        "totalAssists",
-        "totalDamage",
-        "tradeKillOpportunities",
-        "tradeKillAttempts",
-        "tradeKillsSucceeded",
-        "tradeKillAttemptsPercentage",
-        "tradeKillsSuccessPercentage",
-        "tradeKillOpportunitiesPerRound",
-        "tradedDeathOpportunities",
-        "tradedDeathAttempts",
-        "tradedDeathAttemptsPercentage",
-        "tradedDeathsSucceeded",
-        "tradedDeathsSuccessPercentage",
-        "tradedDeathsOpportunitiesPerRound",
-        "leetifyRating",
-        "personalPerformanceRating",
-        "ctLeetifyRating",
-        "tLeetifyRating",
-        "leetifyUserId",
-        "isCollector",
-        "isProPlan",
-        "isLeetifyStaff",
-    ]
-    cols_db = [
-        "id",
-        "game_id",
-        "game_finished_at",
-        "steam64_id",
-        "name",
-        "preaim",
-        "reaction_time",
-        "accuracy",
-        "accuracy_enemy_spotted",
-        "accuracy_head",
-        "shots_fired_enemy_spotted",
-        "shots_fired",
-        "shots_hit_enemy_spotted",
-        "shots_hit_friend",
-        "shots_Hit_Friend_Head",
-        "shots_Hit_Foe",
-        "shots_Hit_Foe_Head",
-        "utility_On_Death_Avg",
-        "he_foes_damage_avg",
-        "he_friends_damage_avg",
-        "he_thrown",
-        "molotov_thrown",
-        "smoke_thrown",
-        "smoke_thrown_ct",
-        "smoke_thrown_ct_good",
-        "smoke_thrown_ct_good_ratio",
-        "smoke_thrown_ct_foes",
-        "counter_strafing_shots_all",
-        "counter_strafing_shots_bad",
-        "counter_strafing_shots_good",
-        "counter_strafing_shots_good_ratio",
-        "flashbang_hit_foe",
-        "flashbang_leading_to_kill",
-        "flashbang_hit_foe_avg_duration",
-        "flashbang_hit_friend",
-        "flashbang_thrown",
-        "flash_assist",
-        "score",
-        "initial_Team_Number",
-        "mvps",
-        "ct_rounds_won",
-        "ct_rounds_lost",
-        "t_rounds_won",
-        "t_rounds_lost",
-        "spray_accuracy",
-        "molotov_foes_damage_avg",
-        "molotov_friends_damage_avg",
-        "color",
-        "total_kills",
-        "total_deaths",
-        "kd_ratio",
-        "multi2k",
-        "multi3k",
-        "multi4k",
-        "multi5k",
-        "hltv_rating",
-        "hsp",
-        "rounds_survived",
-        "rounds_survived_percentage",
-        "dpr",
-        "total_assists",
-        "total_damage",
-        "trade_kill_opportunities",
-        "trade_kill_attempts",
-        "trade_kills_succeeded",
-        "trade_kill_attempts_percentage",
-        "trade_kills_success_percentage",
-        "trade_kill_opportunities_per_round",
-        "traded_death_opportunities",
-        "traded_death_attempts",
-        "traded_death_attempts_percentage",
-        "traded_deaths_succeeded",
-        "traded_deaths_success_percentage",
-        "traded_deaths_opportunities_per_round",
-        "leetify_rating",
-        "personal_performance_rating",
-        "ct_leetify_rating",
-        "t_leetify_rating",
-        "leetify_user_id",
-        "is_collector",
-        "is_pro_plan",
-        "is_leetify_staff",
-    ]
     stats = player_stats_from_game(game)
-
     vals = map(lambda game: vals_in_order(game, cols_api), stats)
 
     with conn.cursor() as curs:
@@ -324,3 +73,60 @@ def extract_player_stats(game, conn=None):
               VALUES ({', '.join(['%s'] * len(cols_db))}); """
         curs.executemany(sql, vals)
     conn.commit()
+
+
+@db.connect_by_default
+def insert_new_profile_games(games, user_id, conn=None):
+    game_ids = db.get_profile_game_ids(user_id, conn=conn)
+
+    game_ids = set(game_ids)
+    new_games = [game for game in games if game.get("gameId") not in game_ids]
+
+    logging.info(f"Inserting {len(new_games)} for: {user_id}")
+    db.insert_profile_games(new_games, conn=conn)
+
+    return new_games
+
+
+@db.connect_by_default
+def insert_player_stats(game_ids, api=Leetify(), conn=None):
+    logging.info(f"Inserting player_stats from {len(game_ids)} games")
+    print_every = int(len(game_ids) / 10)
+    for i, id in enumerate(game_ids):
+        if i % print_every == 0:
+            logging.info(f"Inserted player_stats from {i}/{len(game_ids)} games")
+        game = api.get_match(id)
+        extract_player_stats(game, conn=conn)
+
+
+def get_all(api=Leetify()):
+    conn = db.connect()
+
+    data = db.get_all_profile_names(conn=conn)
+    game_ids = []
+    for name, id in data:
+        logging.info(f"Getting data for {name}: {id}")
+        profile = api.get_profile(id)
+        profile_games = games_from_profile(profile)
+
+        api_games = len(profile_games)
+        db_games = db.get_profile_game_count(id, conn=conn)
+
+        if api_games > db_games:
+            logging.info(f"Getting new games for {name}: {id}")
+            new_games = insert_new_profile_games(profile_games, id, conn=conn)
+            new_game_ids = [game.get("gameId") for game in new_games]
+            game_ids.extend(new_game_ids)
+        elif api_games < db_games:
+            logging.error(f"API returned less games then in DB for {name}: {id}")
+
+        logging.info(f"Games synced with Leetify for {name}: {id}")
+
+    logging.info(f"Updating player_stats with {len(game_ids)} new games")
+    insert_player_stats(game_ids, conn=conn)
+    logging.info("DONE Updating player_stats with new games")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    get_all()

src/leetify/core.py

@@ -9,6 +9,7 @@ class Leetify:
     profile_base_url = f"{api_base_url}/profile"
     match_base_url = f"{api_base_url}/games"
 
+    @on_exception(expo, Exception, max_tries=3)
     @sleep_and_retry
     @limits(1, 5)
     def __get_page(self, url: str) -> dict:

src/transform.py

@@ -1,9 +1,15 @@
+from types import NoneType
 from typing import TypeVar
 
 T = TypeVar("T")
 U = TypeVar("U")
 
 
+def vals_in_order(data: dict, keys: list):
+    vals = [data[key] for key in keys]
+    return vals
+
+
 def extract_cols(data: dict, cols: list[str]) -> dict:
     return {key: data.get(key) for key in cols}
 
@@ -12,9 +18,54 @@ def score_to_text(score: list[int]) -> str:
     return "-".join(map(str, score))
 
 
-# maybe could get columns form db
-def cols_from_player_stats(player_stats: dict) -> dict:
-    cols = [
+## returns column names if the form:
+## {"<table_name>": ([<names from api>]}, [<names from db>])}
+def get_cols(table_name=None):
+    cols = {
+        "profile_game": (
+            [
+                "leetifyUserId",
+                "ctLeetifyRating",
+                "ctLeetifyRatingRounds",
+                "dataSource",
+                "elo",
+                "gameFinishedAt",
+                "gameId",
+                "isCs2",
+                "mapName",
+                "matchResult",
+                "scores",
+                "skillLevel",
+                "tLeetifyRating",
+                "tLeetifyRatingRounds",
+                "deaths",
+                "hasBannedPlayer",
+                "kills",
+                "partySize",
+            ],
+            [
+                "leetify_user_id",
+                "ct_Leetify_rating",
+                "ct_Leetify_rating_rounds",
+                "data_source",
+                "elo",
+                "game_finished_at",
+                "game_id",
+                "is_cs2",
+                "map_name",
+                "match_result",
+                "scores",
+                "skill_level",
+                "t_leetify_rating",
+                "t_leetify_rating_rounds",
+                "deaths",
+                "has_banned_player",
+                "kills",
+                "party_size",
+            ],
+        ),
+        "player_stats": (
+            [
                 "id",
                 "gameId",
                 "gameFinishedAt",
@@ -97,31 +148,144 @@ def cols_from_player_stats(player_stats: dict) -> dict:
                 "isCollector",
                 "isProPlan",
                 "isLeetifyStaff",
-    ]
+            ],
+            [
+                "id",
+                "game_id",
+                "game_finished_at",
+                "steam64_id",
+                "name",
+                "preaim",
+                "reaction_time",
+                "accuracy",
+                "accuracy_enemy_spotted",
+                "accuracy_head",
+                "shots_fired_enemy_spotted",
+                "shots_fired",
+                "shots_hit_enemy_spotted",
+                "shots_hit_friend",
+                "shots_Hit_Friend_Head",
+                "shots_Hit_Foe",
+                "shots_Hit_Foe_Head",
+                "utility_On_Death_Avg",
+                "he_foes_damage_avg",
+                "he_friends_damage_avg",
+                "he_thrown",
+                "molotov_thrown",
+                "smoke_thrown",
+                "smoke_thrown_ct",
+                "smoke_thrown_ct_good",
+                "smoke_thrown_ct_good_ratio",
+                "smoke_thrown_ct_foes",
+                "counter_strafing_shots_all",
+                "counter_strafing_shots_bad",
+                "counter_strafing_shots_good",
+                "counter_strafing_shots_good_ratio",
+                "flashbang_hit_foe",
+                "flashbang_leading_to_kill",
+                "flashbang_hit_foe_avg_duration",
+                "flashbang_hit_friend",
+                "flashbang_thrown",
+                "flash_assist",
+                "score",
+                "initial_Team_Number",
+                "mvps",
+                "ct_rounds_won",
+                "ct_rounds_lost",
+                "t_rounds_won",
+                "t_rounds_lost",
+                "spray_accuracy",
+                "molotov_foes_damage_avg",
+                "molotov_friends_damage_avg",
+                "color",
+                "total_kills",
+                "total_deaths",
+                "kd_ratio",
+                "multi2k",
+                "multi3k",
+                "multi4k",
+                "multi5k",
+                "hltv_rating",
+                "hsp",
+                "rounds_survived",
+                "rounds_survived_percentage",
+                "dpr",
+                "total_assists",
+                "total_damage",
+                "trade_kill_opportunities",
+                "trade_kill_attempts",
+                "trade_kills_succeeded",
+                "trade_kill_attempts_percentage",
+                "trade_kills_success_percentage",
+                "trade_kill_opportunities_per_round",
+                "traded_death_opportunities",
+                "traded_death_attempts",
+                "traded_death_attempts_percentage",
+                "traded_deaths_succeeded",
+                "traded_deaths_success_percentage",
+                "traded_deaths_opportunities_per_round",
+                "leetify_rating",
+                "personal_performance_rating",
+                "ct_leetify_rating",
+                "t_leetify_rating",
+                "leetify_user_id",
+                "is_collector",
+                "is_pro_plan",
+                "is_leetify_staff",
+            ],
+        ),
+        "profile_meta": (
+            [
+                "name",
+                "steam64Id",
+                "isCollector",
+                "isLeetifyStaff",
+                "isProPlan",
+                "leetifyUserId",
+                "faceitNickname",
+            ],
+            [
+                "name",
+                "steam64_id",
+                "is_collector",
+                "is_leetify_staff",
+                "is_pro_plan",
+                "leetify_user_id",
+                "faceit_nickname",
+            ],
+        ),
+    }
+    if table_name:
+        return cols[table_name]
+    return cols
 
+
+def get_api_cols(table_name: str | NoneType = None):
+    col_pos = 0
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
+def get_db_cols(table_name=None):
+    col_pos = 1
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
+# maybe could get columns form db
+def cols_from_player_stats(player_stats: dict) -> dict:
+    cols = get_api_cols("player_stats")
+    if type(cols) != list:
+        raise Exception("could not get api columns for player_stats")
     return extract_cols(player_stats, cols)
 
 
 def cols_from_profile_game(game: dict) -> dict:
-    cols = [
-        "ctLeetifyRating",
-        "ctLeetifyRatingRounds",
-        "dataSource",
-        "elo",
-        "gameFinishedAt",
-        "gameId",
-        "isCs2",
-        "mapName",
-        "matchResult",
-        "scores",
-        "skillLevel",
-        "tLeetifyRating",
-        "tLeetifyRatingRounds",
-        "deaths",
-        "hasBannedPlayer",
-        "kills",
-        "partySize",
-    ]
+    cols = get_api_cols("profile_game")
+    if type(cols) != list:
+        raise Exception("could not get api columns for profile_game")
     return extract_cols(game, cols)
 
 
@@ -130,15 +294,9 @@ def meta_from_profile(profile: dict) -> dict:
     if not meta:
         raise Exception("Could not get profile metadata", profile)
 
-    cols = [
-        "name",
-        "steam64Id",
-        "isCollector",
-        "isLeetifyStaff",
-        "isProPlan",
-        "leetifyUserId",
-        "faceitNickname",
-    ]
+    cols = get_api_cols("profile_meta")
+    if type(cols) != list:
+        raise Exception("could not get api columns for profile_meta")
 
     return extract_cols(meta, cols)
 
@@ -151,7 +309,7 @@ def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]:
 def convert_game_scores(game: dict) -> dict:
     score = game.get("scores")
     if not score:
-        raise Exception("Could not get score from prfile game", game)
+        raise Exception("Could not get score from profile game", game)
 
     score = score_to_text(score)
     game["scores"] = score
@@ -183,6 +341,5 @@ def player_stats_from_game(game: dict) -> list[dict]:
     if not stats:
         raise Exception("Could not get stats from game", game)
 
-
     stats = map(cols_from_player_stats, stats)
     return list(stats)