From d4bbfce4f17084b760a8be8b43f44831b3d70f23 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Thu, 1 Feb 2024 23:05:18 -0500 Subject: [PATCH] refactored functions to use centeralized column name definition --- src/db.py | 49 ++-------- src/extract.py | 248 ++++------------------------------------------- src/transform.py | 14 ++- 3 files changed, 38 insertions(+), 273 deletions(-) diff --git a/src/db.py b/src/db.py index e335f2f..7c6e1c9 100644 --- a/src/db.py +++ b/src/db.py @@ -1,6 +1,6 @@ import psycopg2 as pg from os import environ as env -from transform import vals_in_order +from transform import vals_in_order, get_cols def connect(): @@ -61,47 +61,12 @@ def get_profile_game_ids(user_id, conn=None): @connect_by_default -def insert_games(games, conn=None): - cols_api = [ - "leetifyUserId", - "ctLeetifyRating", - "ctLeetifyRatingRounds", - "dataSource", - "elo", - "gameFinishedAt", - "gameId", - "isCs2", - "mapName", - "matchResult", - "scores", - "skillLevel", - "tLeetifyRating", - "tLeetifyRatingRounds", - "deaths", - "hasBannedPlayer", - "kills", - "partySize", - ] - cols_db = [ - "leetify_user_id", - "ct_Leetify_rating", - "ct_Leetify_rating_rounds", - "data_source", - "elo", - "game_finished_at", - "game_id", - "is_cs2", - "map_name", - "match_result", - "scores", - "skill_level", - "t_leetify_rating", - "t_leetify_rating_rounds", - "deaths", - "has_banned_player", - "kills", - "party_size", - ] +def insert_profile_games(games, conn=None): + cols_api, cols_db = get_cols("profile_game") + if type(cols_api) != list or type(cols_db) != list: + raise Exception( + "could not get columns for profile_game:", cols_api, cols_db + ) vals = map(lambda game: vals_in_order(game, cols_api), games) diff --git a/src/extract.py b/src/extract.py index 057f42b..5bbf61a 100644 --- a/src/extract.py +++ b/src/extract.py @@ -1,5 +1,6 @@ from leetify import Leetify from transform import ( + get_cols, meta_from_profile, games_from_profile, player_stats_from_game, @@ -11,26 +12,14 @@ import logging ## TODO seperate out loading from extraction ## this currently handles getting data from api and loading into db + @db.connect_by_default def extract_profile_meta(id, api=Leetify(), conn=None): - cols_api = [ - "name", - "steam64Id", - "isCollector", - "isLeetifyStaff", - "isProPlan", - "leetifyUserId", - "faceitNickname", - ] - cols_db = [ - "name", - "steam64_id", - "is_collector", - "is_leetify_staff", - "is_pro_plan", - "leetify_user_id", - "faceit_nickname", - ] + cols_api, cols_db = get_cols("profile_meta") + if type(cols_api) != list or type(cols_db) != list: + raise Exception( + "could not get columns for profile_meta:", cols_api, cols_db + ) profile = api.get_profile(id) meta = meta_from_profile(profile) @@ -47,47 +36,11 @@ def extract_profile_meta(id, api=Leetify(), conn=None): @db.connect_by_default def extract_profile_games(profile, conn=None): - cols_api = [ - "leetifyUserId", - "ctLeetifyRating", - "ctLeetifyRatingRounds", - "dataSource", - "elo", - "gameFinishedAt", - "gameId", - "isCs2", - "mapName", - "matchResult", - "scores", - "skillLevel", - "tLeetifyRating", - "tLeetifyRatingRounds", - "deaths", - "hasBannedPlayer", - "kills", - "partySize", - ] - - cols_db = [ - "leetify_user_id", - "ct_Leetify_rating", - "ct_Leetify_rating_rounds", - "data_source", - "elo", - "game_finished_at", - "game_id", - "is_cs2", - "map_name", - "match_result", - "scores", - "skill_level", - "t_leetify_rating", - "t_leetify_rating_rounds", - "deaths", - "has_banned_player", - "kills", - "party_size", - ] + cols_api, cols_db = get_cols("profile_games") + if type(cols_api) != list or type(cols_db) != list: + raise Exception( + "could not get columns for profile_game:", cols_api, cols_db + ) games = games_from_profile(profile) games = map(lambda game: vals_in_order(game, cols_api), games) @@ -105,176 +58,13 @@ def extract_profile_games(profile, conn=None): @db.connect_by_default def extract_player_stats(game, conn=None): - cols_api = [ - "id", - "gameId", - "gameFinishedAt", - "steam64Id", - "name", - "preaim", - "reactionTime", - "accuracy", - "accuracyEnemySpotted", - "accuracyHead", - "shotsFiredEnemySpotted", - "shotsFired", - "shotsHitEnemySpotted", - "shotsHitFriend", - "shotsHitFriendHead", - "shotsHitFoe", - "shotsHitFoeHead", - "utilityOnDeathAvg", - "heFoesDamageAvg", - "heFriendsDamageAvg", - "heThrown", - "molotovThrown", - "smokeThrown", - "smokeThrownCT", - "smokeThrownCTGood", - "smokeThrownCTGoodRatio", - "smokeThrownCTFoes", - "counterStrafingShotsAll", - "counterStrafingShotsBad", - "counterStrafingShotsGood", - "counterStrafingShotsGoodRatio", - "flashbangHitFoe", - "flashbangLeadingToKill", - "flashbangHitFoeAvgDuration", - "flashbangHitFriend", - "flashbangThrown", - "flashAssist", - "score", - "initialTeamNumber", - "mvps", - "ctRoundsWon", - "ctRoundsLost", - "tRoundsWon", - "tRoundsLost", - "sprayAccuracy", - "molotovFoesDamageAvg", - "molotovFriendsDamageAvg", - "color", - "totalKills", - "totalDeaths", - "kdRatio", - "multi2k", - "multi3k", - "multi4k", - "multi5k", - "hltvRating", - "hsp", - "roundsSurvived", - "roundsSurvivedPercentage", - "dpr", - "totalAssists", - "totalDamage", - "tradeKillOpportunities", - "tradeKillAttempts", - "tradeKillsSucceeded", - "tradeKillAttemptsPercentage", - "tradeKillsSuccessPercentage", - "tradeKillOpportunitiesPerRound", - "tradedDeathOpportunities", - "tradedDeathAttempts", - "tradedDeathAttemptsPercentage", - "tradedDeathsSucceeded", - "tradedDeathsSuccessPercentage", - "tradedDeathsOpportunitiesPerRound", - "leetifyRating", - "personalPerformanceRating", - "ctLeetifyRating", - "tLeetifyRating", - "leetifyUserId", - "isCollector", - "isProPlan", - "isLeetifyStaff", - ] - cols_db = [ - "id", - "game_id", - "game_finished_at", - "steam64_id", - "name", - "preaim", - "reaction_time", - "accuracy", - "accuracy_enemy_spotted", - "accuracy_head", - "shots_fired_enemy_spotted", - "shots_fired", - "shots_hit_enemy_spotted", - "shots_hit_friend", - "shots_Hit_Friend_Head", - "shots_Hit_Foe", - "shots_Hit_Foe_Head", - "utility_On_Death_Avg", - "he_foes_damage_avg", - "he_friends_damage_avg", - "he_thrown", - "molotov_thrown", - "smoke_thrown", - "smoke_thrown_ct", - "smoke_thrown_ct_good", - "smoke_thrown_ct_good_ratio", - "smoke_thrown_ct_foes", - "counter_strafing_shots_all", - "counter_strafing_shots_bad", - "counter_strafing_shots_good", - "counter_strafing_shots_good_ratio", - "flashbang_hit_foe", - "flashbang_leading_to_kill", - "flashbang_hit_foe_avg_duration", - "flashbang_hit_friend", - "flashbang_thrown", - "flash_assist", - "score", - "initial_Team_Number", - "mvps", - "ct_rounds_won", - "ct_rounds_lost", - "t_rounds_won", - "t_rounds_lost", - "spray_accuracy", - "molotov_foes_damage_avg", - "molotov_friends_damage_avg", - "color", - "total_kills", - "total_deaths", - "kd_ratio", - "multi2k", - "multi3k", - "multi4k", - "multi5k", - "hltv_rating", - "hsp", - "rounds_survived", - "rounds_survived_percentage", - "dpr", - "total_assists", - "total_damage", - "trade_kill_opportunities", - "trade_kill_attempts", - "trade_kills_succeeded", - "trade_kill_attempts_percentage", - "trade_kills_success_percentage", - "trade_kill_opportunities_per_round", - "traded_death_opportunities", - "traded_death_attempts", - "traded_death_attempts_percentage", - "traded_deaths_succeeded", - "traded_deaths_success_percentage", - "traded_deaths_opportunities_per_round", - "leetify_rating", - "personal_performance_rating", - "ct_leetify_rating", - "t_leetify_rating", - "leetify_user_id", - "is_collector", - "is_pro_plan", - "is_leetify_staff", - ] - stats = player_stats_from_game(game) + cols_api, cols_db = get_cols("player_stats") + if type(cols_api) != list or type(cols_db) != list: + raise Exception( + "could not get columns for player_stats:", cols_api, cols_db + ) + stats = player_stats_from_game(game) vals = map(lambda game: vals_in_order(game, cols_api), stats) with conn.cursor() as curs: @@ -293,7 +83,7 @@ def insert_new_profile_games(games, user_id, conn=None): new_games = [game for game in games if game.get("gameId") not in game_ids] logging.info(f"Inserting {len(new_games)} for: {user_id}") - db.insert_games(new_games, conn=conn) + db.insert_profile_games(new_games, conn=conn) return new_games diff --git a/src/transform.py b/src/transform.py index 04d6caa..bd2586e 100644 --- a/src/transform.py +++ b/src/transform.py @@ -259,26 +259,33 @@ def get_cols(table_name=None): return cols[table_name] return cols + def get_api_cols(table_name: str | NoneType = None): col_pos = 0 if table_name: return get_cols(table_name)[col_pos] return {key: val[col_pos] for key, val in get_cols().items()} -def get_db_cols(table_name = None): + +def get_db_cols(table_name=None): col_pos = 1 if table_name: return get_cols(table_name)[col_pos] return {key: val[col_pos] for key, val in get_cols().items()} + # maybe could get columns form db def cols_from_player_stats(player_stats: dict) -> dict: cols = get_api_cols("player_stats") + if type(cols) != list: + raise Exception("could not get api columns for player_stats") return extract_cols(player_stats, cols) def cols_from_profile_game(game: dict) -> dict: cols = get_api_cols("profile_game") + if type(cols) != list: + raise Exception("could not get api columns for profile_game") return extract_cols(game, cols) @@ -288,6 +295,9 @@ def meta_from_profile(profile: dict) -> dict: raise Exception("Could not get profile metadata", profile) cols = get_api_cols("profile_meta") + if type(cols) != list: + raise Exception("could not get api columns for profile_meta") + return extract_cols(meta, cols) @@ -299,7 +309,7 @@ def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]: def convert_game_scores(game: dict) -> dict: score = game.get("scores") if not score: - raise Exception("Could not get score from prfile game", game) + raise Exception("Could not get score from profile game", game) score = score_to_text(score) game["scores"] = score