diff --git a/requirments.txt b/requirments.txt index a93ea82..224f248 100644 --- a/requirments.txt +++ b/requirments.txt @@ -1,3 +1,5 @@ pytest click -psycopg2 +psycopg2-binary +ratelimit +backoff diff --git a/sqitch/deploy/player-stats.sql b/sqitch/deploy/player-stats.sql index c124591..4d56f85 100644 --- a/sqitch/deploy/player-stats.sql +++ b/sqitch/deploy/player-stats.sql @@ -5,87 +5,87 @@ BEGIN; CREATE TABLE IF NOT EXISTS data.player_stats ( id TEXT, - gameId TEXT, - gameFinishedAt TIMESTAMP, - steam64Id TEXT, + game_id TEXT, + game_finished_at TIMESTAMP, + steam64_id TEXT, name TEXT, preaim NUMERIC, - reactionTime NUMERIC, + reaction_time NUMERIC, accuracy NUMERIC, - accuracyEnemySpotted NUMERIC, - accuracyHead NUMERIC, - shotsFiredEnemySpotted NUMERIC, - shotsFired NUMERIC, - shotsHitEnemySpotted NUMERIC, - shotsHitFriend NUMERIC, - shotsHitFriendHead NUMERIC, - shotsHitFoe NUMERIC, - shotsHitFoeHead NUMERIC, - utilityOnDeathAvg NUMERIC, - heFoesDamageAvg NUMERIC, - heFriendsDamageAvg NUMERIC, - heThrown NUMERIC, - molotovThrown NUMERIC, - smokeThrown NUMERIC, - smokeThrownCT NUMERIC, - smokeThrownCTGood NUMERIC, - smokeThrownCTGoodRatio NUMERIC, - smokeThrownCTFoes NUMERIC, - counterStrafingShotsAll NUMERIC, - counterStrafingShotsBad NUMERIC, - counterStrafingShotsGood NUMERIC, - counterStrafingShotsGoodRatio NUMERIC, - flashbangHitFoe NUMERIC, - flashbangLeadingToKill NUMERIC, - flashbangHitFoeAvgDuration NUMERIC, - flashbangHitFriend NUMERIC, - flashbangThrown NUMERIC, - flashAssist NUMERIC, + accuracy_enemy_spotted NUMERIC, + accuracy_head NUMERIC, + shots_fired_enemy_spotted NUMERIC, + shots_fired NUMERIC, + shots_hit_enemy_spotted NUMERIC, + shots_hit_friend NUMERIC, + shots_Hit_Friend_Head NUMERIC, + shots_Hit_Foe NUMERIC, + shots_Hit_Foe_Head NUMERIC, + utility_On_Death_Avg NUMERIC, + he_foes_damage_avg NUMERIC, + he_friends_damage_avg NUMERIC, + he_thrown NUMERIC, + molotov_thrown NUMERIC, + smoke_thrown NUMERIC, + smoke_thrown_ct NUMERIC, + smoke_thrown_ct_good NUMERIC, + smoke_thrown_ct_good_ratio NUMERIC, + smoke_thrown_ct_foes NUMERIC, + counter_strafing_shots_all NUMERIC, + counter_strafing_shots_bad NUMERIC, + counter_strafing_shots_good NUMERIC, + counter_strafing_shots_good_ratio NUMERIC, + flashbang_hit_foe NUMERIC, + flashbang_leading_to_kill NUMERIC, + flashbang_hit_foe_avg_duration NUMERIC, + flashbang_hit_friend NUMERIC, + flashbang_thrown NUMERIC, + flash_assist NUMERIC, score NUMERIC, - initialTeamNumber NUMERIC, + initial_Team_Number NUMERIC, mvps NUMERIC, - ctRoundsWon NUMERIC, - ctRoundsLost NUMERIC, - tRoundsWon NUMERIC, - tRoundsLost NUMERIC, - sprayAccuracy NUMERIC, - molotovFoesDamageAvg NUMERIC, - molotovFriendsDamageAvg NUMERIC, + ct_rounds_won NUMERIC, + ct_rounds_lost NUMERIC, + t_rounds_won NUMERIC, + t_rounds_lost NUMERIC, + spray_accuracy NUMERIC, + molotov_foes_damage_avg NUMERIC, + molotov_friends_damage_avg NUMERIC, color NUMERIC, - totalKills NUMERIC, - totalDeaths NUMERIC, - kdRatio NUMERIC, + total_kills NUMERIC, + total_deaths NUMERIC, + kd_ratio NUMERIC, multi2k NUMERIC, multi3k NUMERIC, multi4k NUMERIC, multi5k NUMERIC, - hltvRating NUMERIC, + hltv_rating NUMERIC, hsp NUMERIC, - roundsSurvived NUMERIC, - roundsSurvivedPercentage NUMERIC, + rounds_survived NUMERIC, + rounds_survived_percentage NUMERIC, dpr NUMERIC, - totalAssists NUMERIC, - totalDamage NUMERIC, - tradeKillOpportunities NUMERIC, - tradeKillAttempts NUMERIC, - tradeKillsSucceeded NUMERIC, - tradeKillAttemptsPercentage NUMERIC, - tradeKillsSuccessPercentage NUMERIC, - tradeKillOpportunitiesPerRound NUMERIC, - tradedDeathOpportunities NUMERIC, - tradedDeathAttempts NUMERIC, - tradedDeathAttemptsPercentage NUMERIC, - tradedDeathsSucceeded NUMERIC, - tradedDeathsSuccessPercentage NUMERIC, - tradedDeathsOpportunitiesPerRound NUMERIC, - leetifyRating NUMERIC, - personalPerformanceRating NUMERIC, - ctLeetifyRating NUMERIC, - tLeetifyRating NUMERIC, - leetifyUserId TEXT, - isCollector BOOL, - isProPlan BOOL, - isLeetifyStaff BOOL + total_assists NUMERIC, + total_damage NUMERIC, + trade_kill_opportunities NUMERIC, + trade_kill_attempts NUMERIC, + trade_kills_succeeded NUMERIC, + trade_kill_attempts_percentage NUMERIC, + trade_kills_success_percentage NUMERIC, + trade_kill_opportunities_per_round NUMERIC, + traded_death_opportunities NUMERIC, + traded_death_attempts NUMERIC, + traded_death_attempts_percentage NUMERIC, + traded_deaths_succeeded NUMERIC, + traded_deaths_success_percentage NUMERIC, + traded_deaths_opportunities_per_round NUMERIC, + leetify_rating NUMERIC, + personal_performance_rating NUMERIC, + ct_leetify_rating NUMERIC, + t_leetify_rating NUMERIC, + leetify_user_id TEXT, + is_collector BOOL, + is_pro_plan BOOL, + is_leetify_staff BOOL ); COMMIT; diff --git a/sqitch/deploy/profile-game.sql b/sqitch/deploy/profile-game.sql index 712a651..00aedac 100644 --- a/sqitch/deploy/profile-game.sql +++ b/sqitch/deploy/profile-game.sql @@ -4,24 +4,24 @@ BEGIN; CREATE TABLE IF NOT EXISTS data.profile_game ( - leetifyUserId TEXT, - ctLeetifyRating NUMERIC, - ctLeetifyRatingRounds NUMERIC, - dataSource NUMERIC, + leetify_user_id TEXT, + ct_Leetify_rating NUMERIC, + ct_Leetify_rating_rounds NUMERIC, + data_source TEXT, elo NUMERIC, - gameFinishedAt TIMESTAMP, - gameId TEXT, - isCs2 BOOL, - mapName TEXT, - matchResult TEXT, + game_finished_at TIMESTAMP, + game_id TEXT, + is_cs2 BOOL, + map_name TEXT, + match_result TEXT, scores TEXT, -- has to be extracted from array - skillLevel NUMERIC, - tLeetifyRating NUMERIC, - tLeetifyRatingRounds NUMERIC, + skill_level NUMERIC, + t_leetify_rating NUMERIC, + t_leetify_rating_rounds NUMERIC, deaths NUMERIC, - hasBannedPlayer BOOL, + has_banned_player BOOL, kills NUMERIC, - partySize NUMERIC + party_size NUMERIC ); diff --git a/sqitch/deploy/profile-meta.sql b/sqitch/deploy/profile-meta.sql index 91583ff..99ce581 100644 --- a/sqitch/deploy/profile-meta.sql +++ b/sqitch/deploy/profile-meta.sql @@ -4,12 +4,13 @@ BEGIN; CREATE TABLE IF NOT EXISTS data.profile_meta ( - steam64Id TEXT, - isCollector BOOL, - isLeetifyStaff BOOL, - isProPlan BOOL, - leetifyUserId TEXT, - faceitNickname TEXT + name TEXT, + steam64_id TEXT, + is_collector BOOL, + is_leetify_staff BOOL, + is_pro_plan BOOL, + leetify_user_id TEXT, + faceit_nickname TEXT ); COMMIT; diff --git a/sqitch/verify/player-stats.sql b/sqitch/verify/player-stats.sql index e276b31..982872f 100644 --- a/sqitch/verify/player-stats.sql +++ b/sqitch/verify/player-stats.sql @@ -3,26 +3,31 @@ BEGIN; -- XXX Add verifications here. -SELECT id, gameId, gameFinishedAt, steam64Id, name, preaim, reactionTime, - accuracy, accuracyEnemySpotted, accuracyHead, shotsFiredEnemySpotted, - shotsFired, shotsHitEnemySpotted, shotsHitFriend, shotsHitFriendHead, - shotsHitFoe, shotsHitFoeHead, utilityOnDeathAvg, heFoesDamageAvg, - heFriendsDamageAvg, heThrown, molotovThrown, smokeThrown, - smokeThrownCT, smokeThrownCTGood, smokeThrownCTGoodRatio, smokeThrownCTFoes, - counterStrafingShotsAll, counterStrafingShotsBad, counterStrafingShotsGood, - counterStrafingShotsGoodRatio, flashbangHitFoe, flashbangLeadingToKill, - flashbangHitFoeAvgDuration, flashbangHitFriend, flashbangThrown, flashAssist, - score, initialTeamNumber, mvps, ctRoundsWon, ctRoundsLost, tRoundsWon, - tRoundsLost, sprayAccuracy, molotovFoesDamageAvg, molotovFriendsDamageAvg, - color, totalKills, totalDeaths, kdRatio, multi2k, multi3k, multi4k, multi5k, - hltvRating, hsp, roundsSurvived, roundsSurvivedPercentage, dpr, totalAssists, - totalDamage, tradeKillOpportunities, tradeKillAttempts, tradeKillsSucceeded, - tradeKillAttemptsPercentage, tradeKillsSuccessPercentage, - tradeKillOpportunitiesPerRound, tradedDeathOpportunities, - tradedDeathAttempts, tradedDeathAttemptsPercentage, tradedDeathsSucceeded, - tradedDeathsSuccessPercentage, leetifyRating, personalPerformanceRating, - ctLeetifyRating, tLeetifyRating, leetifyUserId, isCollector, isProPlan, - isLeetifyStaff +SELECT id, game_id, game_finished_at, steam64_id, name, preaim, reaction_time, + accuracy, accuracy_enemy_spotted, accuracy_head, shots_fired_enemy_spotted, + shots_fired, shots_hit_enemy_spotted, shots_hit_friend, + shots_Hit_Friend_Head, shots_Hit_Foe, shots_Hit_Foe_Head, + utility_On_Death_Avg, he_foes_damage_avg, he_friends_damage_avg, he_thrown, + molotov_thrown, smoke_thrown, smoke_thrown_ct, smoke_thrown_ct_good, + smoke_thrown_ct_good_ratio, smoke_thrown_ct_foes, + counter_strafing_shots_all, counter_strafing_shots_bad, + counter_strafing_shots_good, counter_strafing_shots_good_ratio, + flashbang_hit_foe, flashbang_leading_to_kill, + flashbang_hit_foe_avg_duration, flashbang_hit_friend, flashbang_thrown, + flash_assist, score, initial_Team_Number, mvps, ct_rounds_won, + ct_rounds_lost, t_rounds_won, t_rounds_lost, spray_accuracy, + molotov_foes_damage_avg, molotov_friends_damage_avg, color, + total_kills, total_deaths, kd_ratio, multi2k, multi3k, multi4k, multi5k, + hltv_rating, hsp, rounds_survived, rounds_survived_percentage, dpr, + total_assists, total_damage, trade_kill_opportunities, trade_kill_attempts, + trade_kills_succeeded, trade_kill_attempts_percentage, + trade_kills_success_percentage, trade_kill_opportunities_per_round, + traded_death_opportunities, traded_death_attempts, + traded_death_attempts_percentage, traded_deaths_succeeded, + traded_deaths_success_percentage, traded_deaths_opportunities_per_round, + leetify_rating, personal_performance_rating, ct_leetify_rating, + t_leetify_rating, leetify_user_id, + is_collector, is_pro_plan, is_leetify_staff FROM data.player_stats; ROLLBACK; diff --git a/sqitch/verify/profile-game.sql b/sqitch/verify/profile-game.sql index cfc4f29..2416333 100644 --- a/sqitch/verify/profile-game.sql +++ b/sqitch/verify/profile-game.sql @@ -3,10 +3,10 @@ BEGIN; -- XXX Add verifications here. -SELECT leetifyUserId, ctLeetifyRating, ctLeetifyRatingRounds, dataSource, elo, - gameFinishedAt, gameId, isCs2, mapName, matchResult, scores, skillLevel, - tLeetifyRating, tLeetifyRatingRounds, deaths, hasBannedPlayer, - kills, partySize +SELECT leetify_user_id, ct_Leetify_rating, ct_Leetify_rating_rounds, + data_source, elo, game_finished_at, game_id, is_cs2, map_name, match_result, + scores, skill_level, t_leetify_rating, t_leetify_rating_rounds, deaths, + has_banned_player, kills, party_size FROM data.profile_game WHERE FALSE; diff --git a/sqitch/verify/profile-meta.sql b/sqitch/verify/profile-meta.sql index 36484f7..29bf0a4 100644 --- a/sqitch/verify/profile-meta.sql +++ b/sqitch/verify/profile-meta.sql @@ -2,8 +2,8 @@ BEGIN; -SELECT steam64Id, isCollector, isLeetifyStaff, - isProPlan, leetifyUserId, faceitNickname +SELECT name, steam64_id, is_collector, is_leetify_staff, is_pro_plan, + leetify_user_id, faceit_nickname FROM data.profile_meta WHERE FALSE; diff --git a/src/extract.py b/src/extract.py index f265370..495a44a 100644 --- a/src/extract.py +++ b/src/extract.py @@ -1,6 +1,15 @@ -import psycopg2 as pg +from .leetify import Leetify from os import environ as env +from .transform import ( + meta_from_profile, + games_from_profile, + player_stats_from_game, +) +import psycopg2 as pg +from typing import Tuple +## TODO seperate out loading from extraction +## this currently handles getting data from api and loading into db def connect(): return pg.connect( @@ -10,3 +19,308 @@ def connect(): host=env.get("POSTGRES_HOST"), port=env.get("POSTGRES_PORT"), ) + + +def vals_in_order(data: dict, keys: list): + vals = [data[key] for key in keys] + return vals + + +def dict_to_col_val(data: dict) -> Tuple[list, list]: + cols = [] + vals = [] + for k, v in data.items(): + cols.append(k) + vals.append(v) + + return cols, vals + + +def dict_to_sql_str(data: dict) -> Tuple[str, str]: + cols, vals = dict_to_col_val(data) + vals = map(str, vals) + + cols = ", ".join(cols) + vals = ", ".join(vals) + + return cols, vals + + +def extract_profile_meta(id, api=Leetify(), conn=None): + if not conn: + conn = connect() + + cols_api = [ + "name", + "steam64Id", + "isCollector", + "isLeetifyStaff", + "isProPlan", + "leetifyUserId", + "faceitNickname", + ] + cols_db = [ + "name", + "steam64_id", + "is_collector", + "is_leetify_staff", + "is_pro_plan", + "leetify_user_id", + "faceit_nickname", + ] + + profile = api.get_profile(id) + meta = meta_from_profile(profile) + vals = vals_in_order(meta, cols_api) + + with conn.cursor() as curs: + sql = f""" + INSERT INTO data.profile_meta ({', '.join(cols_db)}) + VALUES ({', '.join(['%s'] * len(cols_db))}); + """ + curs.execute(sql, vals) + conn.commit() + + +def extract_profile_games(profile, conn=None): + if not conn: + conn = connect() + + cols_api = [ + "leetifyUserId", + "ctLeetifyRating", + "ctLeetifyRatingRounds", + "dataSource", + "elo", + "gameFinishedAt", + "gameId", + "isCs2", + "mapName", + "matchResult", + "scores", + "skillLevel", + "tLeetifyRating", + "tLeetifyRatingRounds", + "deaths", + "hasBannedPlayer", + "kills", + "partySize", + ] + + cols_db = [ + "leetify_user_id", + "ct_Leetify_rating", + "ct_Leetify_rating_rounds", + "data_source", + "elo", + "game_finished_at", + "game_id", + "is_cs2", + "map_name", + "match_result", + "scores", + "skill_level", + "t_leetify_rating", + "t_leetify_rating_rounds", + "deaths", + "has_banned_player", + "kills", + "party_size", + ] + + games = games_from_profile(profile) + games = map(lambda game: vals_in_order(game, cols_api), games) + + with conn.cursor() as curs: + curs.executemany( + f""" + INSERT into data.profile_game ({', '.join(cols_db)}) + VALUES ({', '.join(['%s'] * len(cols_db))}); + """, + games, + ) + conn.commit() + + +def extract_player_stats(game, conn=None): + if not conn: + conn = connect() + + cols_api = [ + "id", + "gameId", + "gameFinishedAt", + "steam64Id", + "name", + "preaim", + "reactionTime", + "accuracy", + "accuracyEnemySpotted", + "accuracyHead", + "shotsFiredEnemySpotted", + "shotsFired", + "shotsHitEnemySpotted", + "shotsHitFriend", + "shotsHitFriendHead", + "shotsHitFoe", + "shotsHitFoeHead", + "utilityOnDeathAvg", + "heFoesDamageAvg", + "heFriendsDamageAvg", + "heThrown", + "molotovThrown", + "smokeThrown", + "smokeThrownCT", + "smokeThrownCTGood", + "smokeThrownCTGoodRatio", + "smokeThrownCTFoes", + "counterStrafingShotsAll", + "counterStrafingShotsBad", + "counterStrafingShotsGood", + "counterStrafingShotsGoodRatio", + "flashbangHitFoe", + "flashbangLeadingToKill", + "flashbangHitFoeAvgDuration", + "flashbangHitFriend", + "flashbangThrown", + "flashAssist", + "score", + "initialTeamNumber", + "mvps", + "ctRoundsWon", + "ctRoundsLost", + "tRoundsWon", + "tRoundsLost", + "sprayAccuracy", + "molotovFoesDamageAvg", + "molotovFriendsDamageAvg", + "color", + "totalKills", + "totalDeaths", + "kdRatio", + "multi2k", + "multi3k", + "multi4k", + "multi5k", + "hltvRating", + "hsp", + "roundsSurvived", + "roundsSurvivedPercentage", + "dpr", + "totalAssists", + "totalDamage", + "tradeKillOpportunities", + "tradeKillAttempts", + "tradeKillsSucceeded", + "tradeKillAttemptsPercentage", + "tradeKillsSuccessPercentage", + "tradeKillOpportunitiesPerRound", + "tradedDeathOpportunities", + "tradedDeathAttempts", + "tradedDeathAttemptsPercentage", + "tradedDeathsSucceeded", + "tradedDeathsSuccessPercentage", + "tradedDeathsOpportunitiesPerRound", + "leetifyRating", + "personalPerformanceRating", + "ctLeetifyRating", + "tLeetifyRating", + "leetifyUserId", + "isCollector", + "isProPlan", + "isLeetifyStaff", + ] + cols_db = [ + "id", + "game_id", + "game_finished_at", + "steam64_id", + "name", + "preaim", + "reaction_time", + "accuracy", + "accuracy_enemy_spotted", + "accuracy_head", + "shots_fired_enemy_spotted", + "shots_fired", + "shots_hit_enemy_spotted", + "shots_hit_friend", + "shots_Hit_Friend_Head", + "shots_Hit_Foe", + "shots_Hit_Foe_Head", + "utility_On_Death_Avg", + "he_foes_damage_avg", + "he_friends_damage_avg", + "he_thrown", + "molotov_thrown", + "smoke_thrown", + "smoke_thrown_ct", + "smoke_thrown_ct_good", + "smoke_thrown_ct_good_ratio", + "smoke_thrown_ct_foes", + "counter_strafing_shots_all", + "counter_strafing_shots_bad", + "counter_strafing_shots_good", + "counter_strafing_shots_good_ratio", + "flashbang_hit_foe", + "flashbang_leading_to_kill", + "flashbang_hit_foe_avg_duration", + "flashbang_hit_friend", + "flashbang_thrown", + "flash_assist", + "score", + "initial_Team_Number", + "mvps", + "ct_rounds_won", + "ct_rounds_lost", + "t_rounds_won", + "t_rounds_lost", + "spray_accuracy", + "molotov_foes_damage_avg", + "molotov_friends_damage_avg", + "color", + "total_kills", + "total_deaths", + "kd_ratio", + "multi2k", + "multi3k", + "multi4k", + "multi5k", + "hltv_rating", + "hsp", + "rounds_survived", + "rounds_survived_percentage", + "dpr", + "total_assists", + "total_damage", + "trade_kill_opportunities", + "trade_kill_attempts", + "trade_kills_succeeded", + "trade_kill_attempts_percentage", + "trade_kills_success_percentage", + "trade_kill_opportunities_per_round", + "traded_death_opportunities", + "traded_death_attempts", + "traded_death_attempts_percentage", + "traded_deaths_succeeded", + "traded_deaths_success_percentage", + "traded_deaths_opportunities_per_round", + "leetify_rating", + "personal_performance_rating", + "ct_leetify_rating", + "t_leetify_rating", + "leetify_user_id", + "is_collector", + "is_pro_plan", + "is_leetify_staff", + ] + stats = player_stats_from_game(game) + + vals = map(lambda game: vals_in_order(game, cols_api), stats) + + with conn.cursor() as curs: + sql = f""" + INSERT into data.player_stats ({', '.join(cols_db)}) + VALUES ({', '.join(['%s'] * len(cols_db))}); """ + curs.executemany(sql, vals) + conn.commit() diff --git a/src/leetify/core.py b/src/leetify/core.py index 29420de..9f69ecb 100644 --- a/src/leetify/core.py +++ b/src/leetify/core.py @@ -1,4 +1,7 @@ +from ratelimit.decorators import sleep_and_retry import requests +from ratelimit import limits, RateLimitException +from backoff import expo, on_exception class Leetify: @@ -6,6 +9,8 @@ class Leetify: profile_base_url = f"{api_base_url}/profile" match_base_url = f"{api_base_url}/games" + @sleep_and_retry + @limits(1, 5) def __get_page(self, url: str) -> dict: resp = requests.get(url) return resp.json()