andrei-stoica
/
leetify-data
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactored functions to use centeralized column name definition

This commit is contained in:
Andrei Stoica 2024-02-01 23:05:18 -05:00
parent 2536280844
commit d4bbfce4f1
3 changed files with 38 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/db.py
View File

@ -1,6 +1,6 @@
import psycopg2 as pg import psycopg2 as pg
from os import environ as env from os import environ as env
from transform import vals_in_order from transform import vals_in_order, get_cols
def connect(): def connect():
@ -61,47 +61,12 @@ def get_profile_game_ids(user_id, conn=None):
@connect_by_default @connect_by_default
def insert_games(games, conn=None): def insert_profile_games(games, conn=None):
cols_api = [ cols_api, cols_db = get_cols("profile_game")
"leetifyUserId", if type(cols_api) != list or type(cols_db) != list:
"ctLeetifyRating", raise Exception(
"ctLeetifyRatingRounds", "could not get columns for profile_game:", cols_api, cols_db
"dataSource", )
"elo",
"gameFinishedAt",
"gameId",
"isCs2",
"mapName",
"matchResult",
"scores",
"skillLevel",
"tLeetifyRating",
"tLeetifyRatingRounds",
"deaths",
"hasBannedPlayer",
"kills",
"partySize",
]
cols_db = [
"leetify_user_id",
"ct_Leetify_rating",
"ct_Leetify_rating_rounds",
"data_source",
"elo",
"game_finished_at",
"game_id",
"is_cs2",
"map_name",
"match_result",
"scores",
"skill_level",
"t_leetify_rating",
"t_leetify_rating_rounds",
"deaths",
"has_banned_player",
"kills",
"party_size",
]
vals = map(lambda game: vals_in_order(game, cols_api), games) vals = map(lambda game: vals_in_order(game, cols_api), games)

248
src/extract.py
View File

@ -1,5 +1,6 @@
from leetify import Leetify from leetify import Leetify
from transform import ( from transform import (
get_cols,
meta_from_profile, meta_from_profile,
games_from_profile, games_from_profile,
player_stats_from_game, player_stats_from_game,
@ -11,26 +12,14 @@ import logging
## TODO seperate out loading from extraction ## TODO seperate out loading from extraction
## this currently handles getting data from api and loading into db ## this currently handles getting data from api and loading into db
@db.connect_by_default @db.connect_by_default
def extract_profile_meta(id, api=Leetify(), conn=None): def extract_profile_meta(id, api=Leetify(), conn=None):
cols_api = [ cols_api, cols_db = get_cols("profile_meta")
"name", if type(cols_api) != list or type(cols_db) != list:
"steam64Id", raise Exception(
"isCollector", "could not get columns for profile_meta:", cols_api, cols_db
"isLeetifyStaff", )
"isProPlan",
"leetifyUserId",
"faceitNickname",
]
cols_db = [
"name",
"steam64_id",
"is_collector",
"is_leetify_staff",
"is_pro_plan",
"leetify_user_id",
"faceit_nickname",
]
profile = api.get_profile(id) profile = api.get_profile(id)
meta = meta_from_profile(profile) meta = meta_from_profile(profile)
@ -47,47 +36,11 @@ def extract_profile_meta(id, api=Leetify(), conn=None):
@db.connect_by_default @db.connect_by_default
def extract_profile_games(profile, conn=None): def extract_profile_games(profile, conn=None):
cols_api = [ cols_api, cols_db = get_cols("profile_games")
"leetifyUserId", if type(cols_api) != list or type(cols_db) != list:
"ctLeetifyRating", raise Exception(
"ctLeetifyRatingRounds", "could not get columns for profile_game:", cols_api, cols_db
"dataSource", )
"elo",
"gameFinishedAt",
"gameId",
"isCs2",
"mapName",
"matchResult",
"scores",
"skillLevel",
"tLeetifyRating",
"tLeetifyRatingRounds",
"deaths",
"hasBannedPlayer",
"kills",
"partySize",
]
cols_db = [
"leetify_user_id",
"ct_Leetify_rating",
"ct_Leetify_rating_rounds",
"data_source",
"elo",
"game_finished_at",
"game_id",
"is_cs2",
"map_name",
"match_result",
"scores",
"skill_level",
"t_leetify_rating",
"t_leetify_rating_rounds",
"deaths",
"has_banned_player",
"kills",
"party_size",
]
games = games_from_profile(profile) games = games_from_profile(profile)
games = map(lambda game: vals_in_order(game, cols_api), games) games = map(lambda game: vals_in_order(game, cols_api), games)
@ -105,176 +58,13 @@ def extract_profile_games(profile, conn=None):
@db.connect_by_default @db.connect_by_default
def extract_player_stats(game, conn=None): def extract_player_stats(game, conn=None):
cols_api = [ cols_api, cols_db = get_cols("player_stats")
"id", if type(cols_api) != list or type(cols_db) != list:
"gameId", raise Exception(
"gameFinishedAt", "could not get columns for player_stats:", cols_api, cols_db
"steam64Id", )
"name",
"preaim",
"reactionTime",
"accuracy",
"accuracyEnemySpotted",
"accuracyHead",
"shotsFiredEnemySpotted",
"shotsFired",
"shotsHitEnemySpotted",
"shotsHitFriend",
"shotsHitFriendHead",
"shotsHitFoe",
"shotsHitFoeHead",
"utilityOnDeathAvg",
"heFoesDamageAvg",
"heFriendsDamageAvg",
"heThrown",
"molotovThrown",
"smokeThrown",
"smokeThrownCT",
"smokeThrownCTGood",
"smokeThrownCTGoodRatio",
"smokeThrownCTFoes",
"counterStrafingShotsAll",
"counterStrafingShotsBad",
"counterStrafingShotsGood",
"counterStrafingShotsGoodRatio",
"flashbangHitFoe",
"flashbangLeadingToKill",
"flashbangHitFoeAvgDuration",
"flashbangHitFriend",
"flashbangThrown",
"flashAssist",
"score",
"initialTeamNumber",
"mvps",
"ctRoundsWon",
"ctRoundsLost",
"tRoundsWon",
"tRoundsLost",
"sprayAccuracy",
"molotovFoesDamageAvg",
"molotovFriendsDamageAvg",
"color",
"totalKills",
"totalDeaths",
"kdRatio",
"multi2k",
"multi3k",
"multi4k",
"multi5k",
"hltvRating",
"hsp",
"roundsSurvived",
"roundsSurvivedPercentage",
"dpr",
"totalAssists",
"totalDamage",
"tradeKillOpportunities",
"tradeKillAttempts",
"tradeKillsSucceeded",
"tradeKillAttemptsPercentage",
"tradeKillsSuccessPercentage",
"tradeKillOpportunitiesPerRound",
"tradedDeathOpportunities",
"tradedDeathAttempts",
"tradedDeathAttemptsPercentage",
"tradedDeathsSucceeded",
"tradedDeathsSuccessPercentage",
"tradedDeathsOpportunitiesPerRound",
"leetifyRating",
"personalPerformanceRating",
"ctLeetifyRating",
"tLeetifyRating",
"leetifyUserId",
"isCollector",
"isProPlan",
"isLeetifyStaff",
]
cols_db = [
"id",
"game_id",
"game_finished_at",
"steam64_id",
"name",
"preaim",
"reaction_time",
"accuracy",
"accuracy_enemy_spotted",
"accuracy_head",
"shots_fired_enemy_spotted",
"shots_fired",
"shots_hit_enemy_spotted",
"shots_hit_friend",
"shots_Hit_Friend_Head",
"shots_Hit_Foe",
"shots_Hit_Foe_Head",
"utility_On_Death_Avg",
"he_foes_damage_avg",
"he_friends_damage_avg",
"he_thrown",
"molotov_thrown",
"smoke_thrown",
"smoke_thrown_ct",
"smoke_thrown_ct_good",
"smoke_thrown_ct_good_ratio",
"smoke_thrown_ct_foes",
"counter_strafing_shots_all",
"counter_strafing_shots_bad",
"counter_strafing_shots_good",
"counter_strafing_shots_good_ratio",
"flashbang_hit_foe",
"flashbang_leading_to_kill",
"flashbang_hit_foe_avg_duration",
"flashbang_hit_friend",
"flashbang_thrown",
"flash_assist",
"score",
"initial_Team_Number",
"mvps",
"ct_rounds_won",
"ct_rounds_lost",
"t_rounds_won",
"t_rounds_lost",
"spray_accuracy",
"molotov_foes_damage_avg",
"molotov_friends_damage_avg",
"color",
"total_kills",
"total_deaths",
"kd_ratio",
"multi2k",
"multi3k",
"multi4k",
"multi5k",
"hltv_rating",
"hsp",
"rounds_survived",
"rounds_survived_percentage",
"dpr",
"total_assists",
"total_damage",
"trade_kill_opportunities",
"trade_kill_attempts",
"trade_kills_succeeded",
"trade_kill_attempts_percentage",
"trade_kills_success_percentage",
"trade_kill_opportunities_per_round",
"traded_death_opportunities",
"traded_death_attempts",
"traded_death_attempts_percentage",
"traded_deaths_succeeded",
"traded_deaths_success_percentage",
"traded_deaths_opportunities_per_round",
"leetify_rating",
"personal_performance_rating",
"ct_leetify_rating",
"t_leetify_rating",
"leetify_user_id",
"is_collector",
"is_pro_plan",
"is_leetify_staff",
]
stats = player_stats_from_game(game)
stats = player_stats_from_game(game)
vals = map(lambda game: vals_in_order(game, cols_api), stats) vals = map(lambda game: vals_in_order(game, cols_api), stats)
with conn.cursor() as curs: with conn.cursor() as curs:
@ -293,7 +83,7 @@ def insert_new_profile_games(games, user_id, conn=None):
new_games = [game for game in games if game.get("gameId") not in game_ids] new_games = [game for game in games if game.get("gameId") not in game_ids]
logging.info(f"Inserting {len(new_games)} for: {user_id}") logging.info(f"Inserting {len(new_games)} for: {user_id}")
db.insert_games(new_games, conn=conn) db.insert_profile_games(new_games, conn=conn)
return new_games return new_games

14
src/transform.py
View File

@ -259,26 +259,33 @@ def get_cols(table_name=None):
return cols[table_name] return cols[table_name]
return cols return cols
def get_api_cols(table_name: str | NoneType = None): def get_api_cols(table_name: str | NoneType = None):
col_pos = 0 col_pos = 0
if table_name: if table_name:
return get_cols(table_name)[col_pos] return get_cols(table_name)[col_pos]
return {key: val[col_pos] for key, val in get_cols().items()} return {key: val[col_pos] for key, val in get_cols().items()}
def get_db_cols(table_name = None):
def get_db_cols(table_name=None):
col_pos = 1 col_pos = 1
if table_name: if table_name:
return get_cols(table_name)[col_pos] return get_cols(table_name)[col_pos]
return {key: val[col_pos] for key, val in get_cols().items()} return {key: val[col_pos] for key, val in get_cols().items()}
# maybe could get columns form db # maybe could get columns form db
def cols_from_player_stats(player_stats: dict) -> dict: def cols_from_player_stats(player_stats: dict) -> dict:
cols = get_api_cols("player_stats") cols = get_api_cols("player_stats")
if type(cols) != list:
raise Exception("could not get api columns for player_stats")
return extract_cols(player_stats, cols) return extract_cols(player_stats, cols)
def cols_from_profile_game(game: dict) -> dict: def cols_from_profile_game(game: dict) -> dict:
cols = get_api_cols("profile_game") cols = get_api_cols("profile_game")
if type(cols) != list:
raise Exception("could not get api columns for profile_game")
return extract_cols(game, cols) return extract_cols(game, cols)
@ -288,6 +295,9 @@ def meta_from_profile(profile: dict) -> dict:
raise Exception("Could not get profile metadata", profile) raise Exception("Could not get profile metadata", profile)
cols = get_api_cols("profile_meta") cols = get_api_cols("profile_meta")
if type(cols) != list:
raise Exception("could not get api columns for profile_meta")
return extract_cols(meta, cols) return extract_cols(meta, cols)
@ -299,7 +309,7 @@ def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]:
def convert_game_scores(game: dict) -> dict: def convert_game_scores(game: dict) -> dict:
score = game.get("scores") score = game.get("scores")
if not score: if not score:
raise Exception("Could not get score from prfile game", game) raise Exception("Could not get score from profile game", game)
score = score_to_text(score) score = score_to_text(score)
game["scores"] = score game["scores"] = score