diff --git a/src/extract.py b/src/extract.py index 831ccca..495a44a 100644 --- a/src/extract.py +++ b/src/extract.py @@ -8,6 +8,8 @@ from .transform import ( import psycopg2 as pg from typing import Tuple +## TODO seperate out loading from extraction +## this currently handles getting data from api and loading into db def connect(): return pg.connect( @@ -143,18 +145,182 @@ def extract_profile_games(profile, conn=None): def extract_player_stats(game, conn=None): if not conn: conn = connect() + + cols_api = [ + "id", + "gameId", + "gameFinishedAt", + "steam64Id", + "name", + "preaim", + "reactionTime", + "accuracy", + "accuracyEnemySpotted", + "accuracyHead", + "shotsFiredEnemySpotted", + "shotsFired", + "shotsHitEnemySpotted", + "shotsHitFriend", + "shotsHitFriendHead", + "shotsHitFoe", + "shotsHitFoeHead", + "utilityOnDeathAvg", + "heFoesDamageAvg", + "heFriendsDamageAvg", + "heThrown", + "molotovThrown", + "smokeThrown", + "smokeThrownCT", + "smokeThrownCTGood", + "smokeThrownCTGoodRatio", + "smokeThrownCTFoes", + "counterStrafingShotsAll", + "counterStrafingShotsBad", + "counterStrafingShotsGood", + "counterStrafingShotsGoodRatio", + "flashbangHitFoe", + "flashbangLeadingToKill", + "flashbangHitFoeAvgDuration", + "flashbangHitFriend", + "flashbangThrown", + "flashAssist", + "score", + "initialTeamNumber", + "mvps", + "ctRoundsWon", + "ctRoundsLost", + "tRoundsWon", + "tRoundsLost", + "sprayAccuracy", + "molotovFoesDamageAvg", + "molotovFriendsDamageAvg", + "color", + "totalKills", + "totalDeaths", + "kdRatio", + "multi2k", + "multi3k", + "multi4k", + "multi5k", + "hltvRating", + "hsp", + "roundsSurvived", + "roundsSurvivedPercentage", + "dpr", + "totalAssists", + "totalDamage", + "tradeKillOpportunities", + "tradeKillAttempts", + "tradeKillsSucceeded", + "tradeKillAttemptsPercentage", + "tradeKillsSuccessPercentage", + "tradeKillOpportunitiesPerRound", + "tradedDeathOpportunities", + "tradedDeathAttempts", + "tradedDeathAttemptsPercentage", + "tradedDeathsSucceeded", + "tradedDeathsSuccessPercentage", + "tradedDeathsOpportunitiesPerRound", + "leetifyRating", + "personalPerformanceRating", + "ctLeetifyRating", + "tLeetifyRating", + "leetifyUserId", + "isCollector", + "isProPlan", + "isLeetifyStaff", + ] + cols_db = [ + "id", + "game_id", + "game_finished_at", + "steam64_id", + "name", + "preaim", + "reaction_time", + "accuracy", + "accuracy_enemy_spotted", + "accuracy_head", + "shots_fired_enemy_spotted", + "shots_fired", + "shots_hit_enemy_spotted", + "shots_hit_friend", + "shots_Hit_Friend_Head", + "shots_Hit_Foe", + "shots_Hit_Foe_Head", + "utility_On_Death_Avg", + "he_foes_damage_avg", + "he_friends_damage_avg", + "he_thrown", + "molotov_thrown", + "smoke_thrown", + "smoke_thrown_ct", + "smoke_thrown_ct_good", + "smoke_thrown_ct_good_ratio", + "smoke_thrown_ct_foes", + "counter_strafing_shots_all", + "counter_strafing_shots_bad", + "counter_strafing_shots_good", + "counter_strafing_shots_good_ratio", + "flashbang_hit_foe", + "flashbang_leading_to_kill", + "flashbang_hit_foe_avg_duration", + "flashbang_hit_friend", + "flashbang_thrown", + "flash_assist", + "score", + "initial_Team_Number", + "mvps", + "ct_rounds_won", + "ct_rounds_lost", + "t_rounds_won", + "t_rounds_lost", + "spray_accuracy", + "molotov_foes_damage_avg", + "molotov_friends_damage_avg", + "color", + "total_kills", + "total_deaths", + "kd_ratio", + "multi2k", + "multi3k", + "multi4k", + "multi5k", + "hltv_rating", + "hsp", + "rounds_survived", + "rounds_survived_percentage", + "dpr", + "total_assists", + "total_damage", + "trade_kill_opportunities", + "trade_kill_attempts", + "trade_kills_succeeded", + "trade_kill_attempts_percentage", + "trade_kills_success_percentage", + "trade_kill_opportunities_per_round", + "traded_death_opportunities", + "traded_death_attempts", + "traded_death_attempts_percentage", + "traded_deaths_succeeded", + "traded_deaths_success_percentage", + "traded_deaths_opportunities_per_round", + "leetify_rating", + "personal_performance_rating", + "ct_leetify_rating", + "t_leetify_rating", + "leetify_user_id", + "is_collector", + "is_pro_plan", + "is_leetify_staff", + ] stats = player_stats_from_game(game) - games = map(dict_to_sql_str, stats) + vals = map(lambda game: vals_in_order(game, cols_api), stats) with conn.cursor() as curs: - for ( - cols, - vals, - ) in games: - curs.execute( - """INSERT INTO data.player_stats (%s) - VALUES (%s);""", - (cols, vals), - ) + sql = f""" + INSERT into data.player_stats ({', '.join(cols_db)}) + VALUES ({', '.join(['%s'] * len(cols_db))}); """ + curs.executemany(sql, vals) conn.commit()