refactored out transofmations to own file
This commit is contained in:
parent
1b157f2202
commit
be24c1a639
176
src/extract.py
176
src/extract.py
|
|
@ -1,9 +1,5 @@
|
||||||
import psycopg2 as pg
|
import psycopg2 as pg
|
||||||
from os import environ as env
|
from os import environ as env
|
||||||
from typing import TypeVar
|
|
||||||
|
|
||||||
T = TypeVar("T")
|
|
||||||
U = TypeVar("U")
|
|
||||||
|
|
||||||
|
|
||||||
def connect():
|
def connect():
|
||||||
|
|
@ -14,175 +10,3 @@ def connect():
|
||||||
host=env.get("POSTGRES_HOST"),
|
host=env.get("POSTGRES_HOST"),
|
||||||
port=env.get("POSTGRES_PORT"),
|
port=env.get("POSTGRES_PORT"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def extract_cols(data: dict, cols: list[str]) -> dict: return {key: data.get(key) for key in cols}
|
|
||||||
|
|
||||||
|
|
||||||
def score_to_text(score: list[int]) -> str:
|
|
||||||
return "-".join(map(str, score))
|
|
||||||
|
|
||||||
|
|
||||||
# maybe could get columns form db
|
|
||||||
def cols_from_player_stats(player_stats: dict) -> dict:
|
|
||||||
cols = [
|
|
||||||
"id",
|
|
||||||
"gameId",
|
|
||||||
"gameFinishedAt",
|
|
||||||
"steam64Id",
|
|
||||||
"name",
|
|
||||||
"preaim",
|
|
||||||
"reactionTime",
|
|
||||||
"accuracy",
|
|
||||||
"accuracyEnemySpotted",
|
|
||||||
"accuracyHead",
|
|
||||||
"shotsFiredEnemySpotted",
|
|
||||||
"shotsFired",
|
|
||||||
"shotsHitEnemySpotted",
|
|
||||||
"shotsHitFriend",
|
|
||||||
"shotsHitFriendHead",
|
|
||||||
"shotsHitFoe",
|
|
||||||
"shotsHitFoeHead",
|
|
||||||
"utilityOnDeathAvg",
|
|
||||||
"heFoesDamageAvg",
|
|
||||||
"heFriendsDamageAvg",
|
|
||||||
"heThrown",
|
|
||||||
"molotovThrown",
|
|
||||||
"smokeThrown",
|
|
||||||
"smokeThrownCT",
|
|
||||||
"smokeThrownCTGood",
|
|
||||||
"smokeThrownCTGoodRatio",
|
|
||||||
"smokeThrownCTFoes",
|
|
||||||
"counterStrafingShotsAll",
|
|
||||||
"counterStrafingShotsBad",
|
|
||||||
"counterStrafingShotsGood",
|
|
||||||
"counterStrafingShotsGoodRatio",
|
|
||||||
"flashbangHitFoe",
|
|
||||||
"flashbangLeadingToKill",
|
|
||||||
"flashbangHitFoeAvgDuration",
|
|
||||||
"flashbangHitFriend",
|
|
||||||
"flashbangThrown",
|
|
||||||
"flashAssist",
|
|
||||||
"score",
|
|
||||||
"initialTeamNumber",
|
|
||||||
"mvps",
|
|
||||||
"ctRoundsWon",
|
|
||||||
"ctRoundsLost",
|
|
||||||
"tRoundsWon",
|
|
||||||
"tRoundsLost",
|
|
||||||
"sprayAccuracy",
|
|
||||||
"molotovFoesDamageAvg",
|
|
||||||
"molotovFriendsDamageAvg",
|
|
||||||
"color",
|
|
||||||
"totalKills",
|
|
||||||
"totalDeaths",
|
|
||||||
"kdRatio",
|
|
||||||
"multi2k",
|
|
||||||
"multi3k",
|
|
||||||
"multi4k",
|
|
||||||
"multi5k",
|
|
||||||
"hltvRating",
|
|
||||||
"hsp",
|
|
||||||
"roundsSurvived",
|
|
||||||
"roundsSurvivedPercentage",
|
|
||||||
"dpr",
|
|
||||||
"totalAssists",
|
|
||||||
"totalDamage",
|
|
||||||
"tradeKillOpportunities",
|
|
||||||
"tradeKillAttempts",
|
|
||||||
"tradeKillsSucceeded",
|
|
||||||
"tradeKillAttemptsPercentage",
|
|
||||||
"tradeKillsSuccessPercentage",
|
|
||||||
"tradeKillOpportunitiesPerRound",
|
|
||||||
"tradedDeathOpportunities",
|
|
||||||
"tradedDeathAttempts",
|
|
||||||
"tradedDeathAttemptsPercentage",
|
|
||||||
"tradedDeathsSucceeded",
|
|
||||||
"tradedDeathsSuccessPercentage",
|
|
||||||
"tradedDeathsOpportunitiesPerRound",
|
|
||||||
"leetifyRating",
|
|
||||||
"personalPerformanceRating",
|
|
||||||
"ctLeetifyRating",
|
|
||||||
"tLeetifyRating",
|
|
||||||
"leetifyUserId",
|
|
||||||
"isCollector",
|
|
||||||
"isProPlan",
|
|
||||||
"isLeetifyStaff",
|
|
||||||
]
|
|
||||||
|
|
||||||
return extract_cols(player_stats, cols)
|
|
||||||
|
|
||||||
|
|
||||||
def cols_from_profile_game(game: dict) -> dict:
|
|
||||||
cols = [
|
|
||||||
"ctLeetifyRating",
|
|
||||||
"ctLeetifyRatingRounds",
|
|
||||||
"dataSource",
|
|
||||||
"elo",
|
|
||||||
"gameFinishedAt",
|
|
||||||
"gameId",
|
|
||||||
"isCs2",
|
|
||||||
"mapName",
|
|
||||||
"matchResult",
|
|
||||||
"scores",
|
|
||||||
"skillLevel",
|
|
||||||
"tLeetifyRating",
|
|
||||||
"tLeetifyRatingRounds",
|
|
||||||
"deaths",
|
|
||||||
"hasBannedPlayer",
|
|
||||||
"kills",
|
|
||||||
"partySize",
|
|
||||||
]
|
|
||||||
return extract_cols(game, cols)
|
|
||||||
|
|
||||||
|
|
||||||
def meta_from_profile(profile: dict) -> dict:
|
|
||||||
meta = profile.get("meta")
|
|
||||||
if not meta:
|
|
||||||
raise Exception("Could not get profile metadata", profile)
|
|
||||||
|
|
||||||
cols = [
|
|
||||||
"name",
|
|
||||||
"steam64Id",
|
|
||||||
"isCollector",
|
|
||||||
"isLeetifyStaff",
|
|
||||||
"isProPlan",
|
|
||||||
"leetifyUserId",
|
|
||||||
"faceitNickname",
|
|
||||||
]
|
|
||||||
|
|
||||||
return extract_cols(meta, cols)
|
|
||||||
|
|
||||||
|
|
||||||
def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]:
|
|
||||||
data[key] = value
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
def convert_game_scores(game: dict) -> dict:
|
|
||||||
score = game.get("scores")
|
|
||||||
if not score:
|
|
||||||
raise Exception("Could not get score from prfile game", game)
|
|
||||||
|
|
||||||
score = score_to_text(score)
|
|
||||||
game["scores"] = score
|
|
||||||
return game
|
|
||||||
|
|
||||||
|
|
||||||
def games_from_profile(profile: dict) -> list:
|
|
||||||
games = profile.get("games")
|
|
||||||
if not games and not type(games) == list:
|
|
||||||
raise Exception("Could not get games from profile", profile)
|
|
||||||
|
|
||||||
meta = profile.get("meta")
|
|
||||||
if not meta:
|
|
||||||
raise Exception("Could not get profile metadata", profile)
|
|
||||||
|
|
||||||
id = meta.get("leetifyUserId")
|
|
||||||
if not id:
|
|
||||||
raise Exception("Could not get id from profile metadata", meta, profile)
|
|
||||||
|
|
||||||
games = map(cols_from_profile_game, games)
|
|
||||||
games = map(lambda game: insert_value(game, "leetifyUserId", id), games)
|
|
||||||
games = map(convert_game_scores, games)
|
|
||||||
return list(games)
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,188 @@
|
||||||
|
from typing import TypeVar
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
U = TypeVar("U")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_cols(data: dict, cols: list[str]) -> dict:
|
||||||
|
return {key: data.get(key) for key in cols}
|
||||||
|
|
||||||
|
|
||||||
|
def score_to_text(score: list[int]) -> str:
|
||||||
|
return "-".join(map(str, score))
|
||||||
|
|
||||||
|
|
||||||
|
# maybe could get columns form db
|
||||||
|
def cols_from_player_stats(player_stats: dict) -> dict:
|
||||||
|
cols = [
|
||||||
|
"id",
|
||||||
|
"gameId",
|
||||||
|
"gameFinishedAt",
|
||||||
|
"steam64Id",
|
||||||
|
"name",
|
||||||
|
"preaim",
|
||||||
|
"reactionTime",
|
||||||
|
"accuracy",
|
||||||
|
"accuracyEnemySpotted",
|
||||||
|
"accuracyHead",
|
||||||
|
"shotsFiredEnemySpotted",
|
||||||
|
"shotsFired",
|
||||||
|
"shotsHitEnemySpotted",
|
||||||
|
"shotsHitFriend",
|
||||||
|
"shotsHitFriendHead",
|
||||||
|
"shotsHitFoe",
|
||||||
|
"shotsHitFoeHead",
|
||||||
|
"utilityOnDeathAvg",
|
||||||
|
"heFoesDamageAvg",
|
||||||
|
"heFriendsDamageAvg",
|
||||||
|
"heThrown",
|
||||||
|
"molotovThrown",
|
||||||
|
"smokeThrown",
|
||||||
|
"smokeThrownCT",
|
||||||
|
"smokeThrownCTGood",
|
||||||
|
"smokeThrownCTGoodRatio",
|
||||||
|
"smokeThrownCTFoes",
|
||||||
|
"counterStrafingShotsAll",
|
||||||
|
"counterStrafingShotsBad",
|
||||||
|
"counterStrafingShotsGood",
|
||||||
|
"counterStrafingShotsGoodRatio",
|
||||||
|
"flashbangHitFoe",
|
||||||
|
"flashbangLeadingToKill",
|
||||||
|
"flashbangHitFoeAvgDuration",
|
||||||
|
"flashbangHitFriend",
|
||||||
|
"flashbangThrown",
|
||||||
|
"flashAssist",
|
||||||
|
"score",
|
||||||
|
"initialTeamNumber",
|
||||||
|
"mvps",
|
||||||
|
"ctRoundsWon",
|
||||||
|
"ctRoundsLost",
|
||||||
|
"tRoundsWon",
|
||||||
|
"tRoundsLost",
|
||||||
|
"sprayAccuracy",
|
||||||
|
"molotovFoesDamageAvg",
|
||||||
|
"molotovFriendsDamageAvg",
|
||||||
|
"color",
|
||||||
|
"totalKills",
|
||||||
|
"totalDeaths",
|
||||||
|
"kdRatio",
|
||||||
|
"multi2k",
|
||||||
|
"multi3k",
|
||||||
|
"multi4k",
|
||||||
|
"multi5k",
|
||||||
|
"hltvRating",
|
||||||
|
"hsp",
|
||||||
|
"roundsSurvived",
|
||||||
|
"roundsSurvivedPercentage",
|
||||||
|
"dpr",
|
||||||
|
"totalAssists",
|
||||||
|
"totalDamage",
|
||||||
|
"tradeKillOpportunities",
|
||||||
|
"tradeKillAttempts",
|
||||||
|
"tradeKillsSucceeded",
|
||||||
|
"tradeKillAttemptsPercentage",
|
||||||
|
"tradeKillsSuccessPercentage",
|
||||||
|
"tradeKillOpportunitiesPerRound",
|
||||||
|
"tradedDeathOpportunities",
|
||||||
|
"tradedDeathAttempts",
|
||||||
|
"tradedDeathAttemptsPercentage",
|
||||||
|
"tradedDeathsSucceeded",
|
||||||
|
"tradedDeathsSuccessPercentage",
|
||||||
|
"tradedDeathsOpportunitiesPerRound",
|
||||||
|
"leetifyRating",
|
||||||
|
"personalPerformanceRating",
|
||||||
|
"ctLeetifyRating",
|
||||||
|
"tLeetifyRating",
|
||||||
|
"leetifyUserId",
|
||||||
|
"isCollector",
|
||||||
|
"isProPlan",
|
||||||
|
"isLeetifyStaff",
|
||||||
|
]
|
||||||
|
|
||||||
|
return extract_cols(player_stats, cols)
|
||||||
|
|
||||||
|
|
||||||
|
def cols_from_profile_game(game: dict) -> dict:
|
||||||
|
cols = [
|
||||||
|
"ctLeetifyRating",
|
||||||
|
"ctLeetifyRatingRounds",
|
||||||
|
"dataSource",
|
||||||
|
"elo",
|
||||||
|
"gameFinishedAt",
|
||||||
|
"gameId",
|
||||||
|
"isCs2",
|
||||||
|
"mapName",
|
||||||
|
"matchResult",
|
||||||
|
"scores",
|
||||||
|
"skillLevel",
|
||||||
|
"tLeetifyRating",
|
||||||
|
"tLeetifyRatingRounds",
|
||||||
|
"deaths",
|
||||||
|
"hasBannedPlayer",
|
||||||
|
"kills",
|
||||||
|
"partySize",
|
||||||
|
]
|
||||||
|
return extract_cols(game, cols)
|
||||||
|
|
||||||
|
|
||||||
|
def meta_from_profile(profile: dict) -> dict:
|
||||||
|
meta = profile.get("meta")
|
||||||
|
if not meta:
|
||||||
|
raise Exception("Could not get profile metadata", profile)
|
||||||
|
|
||||||
|
cols = [
|
||||||
|
"name",
|
||||||
|
"steam64Id",
|
||||||
|
"isCollector",
|
||||||
|
"isLeetifyStaff",
|
||||||
|
"isProPlan",
|
||||||
|
"leetifyUserId",
|
||||||
|
"faceitNickname",
|
||||||
|
]
|
||||||
|
|
||||||
|
return extract_cols(meta, cols)
|
||||||
|
|
||||||
|
|
||||||
|
def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]:
|
||||||
|
data[key] = value
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def convert_game_scores(game: dict) -> dict:
|
||||||
|
score = game.get("scores")
|
||||||
|
if not score:
|
||||||
|
raise Exception("Could not get score from prfile game", game)
|
||||||
|
|
||||||
|
score = score_to_text(score)
|
||||||
|
game["scores"] = score
|
||||||
|
return game
|
||||||
|
|
||||||
|
|
||||||
|
def games_from_profile(profile: dict) -> list:
|
||||||
|
games = profile.get("games")
|
||||||
|
# emtpy list is falsy but an acceptable value
|
||||||
|
if not games and not type(games) == list:
|
||||||
|
raise Exception("Could not get games from profile", profile)
|
||||||
|
|
||||||
|
meta = profile.get("meta")
|
||||||
|
if not meta:
|
||||||
|
raise Exception("Could not get profile metadata", profile)
|
||||||
|
|
||||||
|
id = meta.get("leetifyUserId")
|
||||||
|
if not id:
|
||||||
|
raise Exception("Could not get id from profile metadata", meta, profile)
|
||||||
|
|
||||||
|
games = map(cols_from_profile_game, games)
|
||||||
|
games = map(lambda game: insert_value(game, "leetifyUserId", id), games)
|
||||||
|
games = map(convert_game_scores, games)
|
||||||
|
return list(games)
|
||||||
|
|
||||||
|
|
||||||
|
def player_stats_from_game(game: dict) -> list[dict]:
|
||||||
|
stats = game.get("playerStats")
|
||||||
|
if not stats:
|
||||||
|
raise Exception("Could not get stats from game", game)
|
||||||
|
|
||||||
|
|
||||||
|
stats = map(cols_from_player_stats,stats)
|
||||||
|
return list(stats)
|
||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue