andrei-stoica
/
leetify-data
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

prototype extracting

This commit is contained in:
Andrei Stoica 2024-01-29 20:26:56 -05:00
parent 6c41cda563
commit 30497075be
9 changed files with 444 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
requirments.txt
View File

@ -1,3 +1,5 @@
pytest
click
psycopg2
psycopg2-binary
ratelimit
backoff

138
sqitch/deploy/player-stats.sql
View File

@ -5,87 +5,87 @@ BEGIN;
CREATE TABLE IF NOT EXISTS data.player_stats (
id TEXT,
gameId TEXT,
gameFinishedAt TIMESTAMP,
steam64Id TEXT,
game_id TEXT,
game_finished_at TIMESTAMP,
steam64_id TEXT,
name TEXT,
preaim NUMERIC,
reactionTime NUMERIC,
reaction_time NUMERIC,
accuracy NUMERIC,
accuracyEnemySpotted NUMERIC,
accuracyHead NUMERIC,
shotsFiredEnemySpotted NUMERIC,
shotsFired NUMERIC,
shotsHitEnemySpotted NUMERIC,
shotsHitFriend NUMERIC,
shotsHitFriendHead NUMERIC,
shotsHitFoe NUMERIC,
shotsHitFoeHead NUMERIC,
utilityOnDeathAvg NUMERIC,
heFoesDamageAvg NUMERIC,
heFriendsDamageAvg NUMERIC,
heThrown NUMERIC,
molotovThrown NUMERIC,
smokeThrown NUMERIC,
smokeThrownCT NUMERIC,
smokeThrownCTGood NUMERIC,
smokeThrownCTGoodRatio NUMERIC,
smokeThrownCTFoes NUMERIC,
counterStrafingShotsAll NUMERIC,
counterStrafingShotsBad NUMERIC,
counterStrafingShotsGood NUMERIC,
counterStrafingShotsGoodRatio NUMERIC,
flashbangHitFoe NUMERIC,
flashbangLeadingToKill NUMERIC,
flashbangHitFoeAvgDuration NUMERIC,
flashbangHitFriend NUMERIC,
flashbangThrown NUMERIC,
flashAssist NUMERIC,
accuracy_enemy_spotted NUMERIC,
accuracy_head NUMERIC,
shots_fired_enemy_spotted NUMERIC,
shots_fired NUMERIC,
shots_hit_enemy_spotted NUMERIC,
shots_hit_friend NUMERIC,
shots_Hit_Friend_Head NUMERIC,
shots_Hit_Foe NUMERIC,
shots_Hit_Foe_Head NUMERIC,
utility_On_Death_Avg NUMERIC,
he_foes_damage_avg NUMERIC,
he_friends_damage_avg NUMERIC,
he_thrown NUMERIC,
molotov_thrown NUMERIC,
smoke_thrown NUMERIC,
smoke_thrown_ct NUMERIC,
smoke_thrown_ct_good NUMERIC,
smoke_thrown_ct_good_ratio NUMERIC,
smoke_thrown_ct_foes NUMERIC,
counter_strafing_shots_all NUMERIC,
counter_strafing_shots_bad NUMERIC,
counter_strafing_shots_good NUMERIC,
counter_strafing_shots_good_ratio NUMERIC,
flashbang_hit_foe NUMERIC,
flashbang_leading_to_kill NUMERIC,
flashbang_hit_foe_avg_duration NUMERIC,
flashbang_hit_friend NUMERIC,
flashbang_thrown NUMERIC,
flash_assist NUMERIC,
score NUMERIC,
initialTeamNumber NUMERIC,
initial_Team_Number NUMERIC,
mvps NUMERIC,
ctRoundsWon NUMERIC,
ctRoundsLost NUMERIC,
tRoundsWon NUMERIC,
tRoundsLost NUMERIC,
sprayAccuracy NUMERIC,
molotovFoesDamageAvg NUMERIC,
molotovFriendsDamageAvg NUMERIC,
ct_rounds_won NUMERIC,
ct_rounds_lost NUMERIC,
t_rounds_won NUMERIC,
t_rounds_lost NUMERIC,
spray_accuracy NUMERIC,
molotov_foes_damage_avg NUMERIC,
molotov_friends_damage_avg NUMERIC,
color NUMERIC,
totalKills NUMERIC,
totalDeaths NUMERIC,
kdRatio NUMERIC,
total_kills NUMERIC,
total_deaths NUMERIC,
kd_ratio NUMERIC,
multi2k NUMERIC,
multi3k NUMERIC,
multi4k NUMERIC,
multi5k NUMERIC,
hltvRating NUMERIC,
hltv_rating NUMERIC,
hsp NUMERIC,
roundsSurvived NUMERIC,
roundsSurvivedPercentage NUMERIC,
rounds_survived NUMERIC,
rounds_survived_percentage NUMERIC,
dpr NUMERIC,
totalAssists NUMERIC,
totalDamage NUMERIC,
tradeKillOpportunities NUMERIC,
tradeKillAttempts NUMERIC,
tradeKillsSucceeded NUMERIC,
tradeKillAttemptsPercentage NUMERIC,
tradeKillsSuccessPercentage NUMERIC,
tradeKillOpportunitiesPerRound NUMERIC,
tradedDeathOpportunities NUMERIC,
tradedDeathAttempts NUMERIC,
tradedDeathAttemptsPercentage NUMERIC,
tradedDeathsSucceeded NUMERIC,
tradedDeathsSuccessPercentage NUMERIC,
tradedDeathsOpportunitiesPerRound NUMERIC,
leetifyRating NUMERIC,
personalPerformanceRating NUMERIC,
ctLeetifyRating NUMERIC,
tLeetifyRating NUMERIC,
leetifyUserId TEXT,
isCollector BOOL,
isProPlan BOOL,
isLeetifyStaff BOOL
total_assists NUMERIC,
total_damage NUMERIC,
trade_kill_opportunities NUMERIC,
trade_kill_attempts NUMERIC,
trade_kills_succeeded NUMERIC,
trade_kill_attempts_percentage NUMERIC,
trade_kills_success_percentage NUMERIC,
trade_kill_opportunities_per_round NUMERIC,
traded_death_opportunities NUMERIC,
traded_death_attempts NUMERIC,
traded_death_attempts_percentage NUMERIC,
traded_deaths_succeeded NUMERIC,
traded_deaths_success_percentage NUMERIC,
traded_deaths_opportunities_per_round NUMERIC,
leetify_rating NUMERIC,
personal_performance_rating NUMERIC,
ct_leetify_rating NUMERIC,
t_leetify_rating NUMERIC,
leetify_user_id TEXT,
is_collector BOOL,
is_pro_plan BOOL,
is_leetify_staff BOOL
);
COMMIT;

28
sqitch/deploy/profile-game.sql
View File

@ -4,24 +4,24 @@
BEGIN;
CREATE TABLE IF NOT EXISTS data.profile_game (
leetifyUserId TEXT,
ctLeetifyRating NUMERIC,
ctLeetifyRatingRounds NUMERIC,
dataSource NUMERIC,
leetify_user_id TEXT,
ct_Leetify_rating NUMERIC,
ct_Leetify_rating_rounds NUMERIC,
data_source TEXT,
elo NUMERIC,
gameFinishedAt TIMESTAMP,
gameId TEXT,
isCs2 BOOL,
mapName TEXT,
matchResult TEXT,
game_finished_at TIMESTAMP,
game_id TEXT,
is_cs2 BOOL,
map_name TEXT,
match_result TEXT,
scores TEXT, -- has to be extracted from array
skillLevel NUMERIC,
tLeetifyRating NUMERIC,
tLeetifyRatingRounds NUMERIC,
skill_level NUMERIC,
t_leetify_rating NUMERIC,
t_leetify_rating_rounds NUMERIC,
deaths NUMERIC,
hasBannedPlayer BOOL,
has_banned_player BOOL,
kills NUMERIC,
partySize NUMERIC
party_size NUMERIC
);

13
sqitch/deploy/profile-meta.sql
View File

@ -4,12 +4,13 @@
BEGIN;
CREATE TABLE IF NOT EXISTS data.profile_meta (
steam64Id TEXT,
isCollector BOOL,
isLeetifyStaff BOOL,
isProPlan BOOL,
leetifyUserId TEXT,
faceitNickname TEXT
name TEXT,
steam64_id TEXT,
is_collector BOOL,
is_leetify_staff BOOL,
is_pro_plan BOOL,
leetify_user_id TEXT,
faceit_nickname TEXT
);
COMMIT;

45
sqitch/verify/player-stats.sql
View File

@ -3,26 +3,31 @@
BEGIN;
-- XXX Add verifications here.
SELECT id, gameId, gameFinishedAt, steam64Id, name, preaim, reactionTime,
accuracy, accuracyEnemySpotted, accuracyHead, shotsFiredEnemySpotted,
shotsFired, shotsHitEnemySpotted, shotsHitFriend, shotsHitFriendHead,
shotsHitFoe, shotsHitFoeHead, utilityOnDeathAvg, heFoesDamageAvg,
heFriendsDamageAvg, heThrown, molotovThrown, smokeThrown,
smokeThrownCT, smokeThrownCTGood, smokeThrownCTGoodRatio, smokeThrownCTFoes,
counterStrafingShotsAll, counterStrafingShotsBad, counterStrafingShotsGood,
counterStrafingShotsGoodRatio, flashbangHitFoe, flashbangLeadingToKill,
flashbangHitFoeAvgDuration, flashbangHitFriend, flashbangThrown, flashAssist,
score, initialTeamNumber, mvps, ctRoundsWon, ctRoundsLost, tRoundsWon,
tRoundsLost, sprayAccuracy, molotovFoesDamageAvg, molotovFriendsDamageAvg,
color, totalKills, totalDeaths, kdRatio, multi2k, multi3k, multi4k, multi5k,
hltvRating, hsp, roundsSurvived, roundsSurvivedPercentage, dpr, totalAssists,
totalDamage, tradeKillOpportunities, tradeKillAttempts, tradeKillsSucceeded,
tradeKillAttemptsPercentage, tradeKillsSuccessPercentage,
tradeKillOpportunitiesPerRound, tradedDeathOpportunities,
tradedDeathAttempts, tradedDeathAttemptsPercentage, tradedDeathsSucceeded,
tradedDeathsSuccessPercentage, leetifyRating, personalPerformanceRating,
ctLeetifyRating, tLeetifyRating, leetifyUserId, isCollector, isProPlan,
isLeetifyStaff
SELECT id, game_id, game_finished_at, steam64_id, name, preaim, reaction_time,
accuracy, accuracy_enemy_spotted, accuracy_head, shots_fired_enemy_spotted,
shots_fired, shots_hit_enemy_spotted, shots_hit_friend,
shots_Hit_Friend_Head, shots_Hit_Foe, shots_Hit_Foe_Head,
utility_On_Death_Avg, he_foes_damage_avg, he_friends_damage_avg, he_thrown,
molotov_thrown, smoke_thrown, smoke_thrown_ct, smoke_thrown_ct_good,
smoke_thrown_ct_good_ratio, smoke_thrown_ct_foes,
counter_strafing_shots_all, counter_strafing_shots_bad,
counter_strafing_shots_good, counter_strafing_shots_good_ratio,
flashbang_hit_foe, flashbang_leading_to_kill,
flashbang_hit_foe_avg_duration, flashbang_hit_friend, flashbang_thrown,
flash_assist, score, initial_Team_Number, mvps, ct_rounds_won,
ct_rounds_lost, t_rounds_won, t_rounds_lost, spray_accuracy,
molotov_foes_damage_avg, molotov_friends_damage_avg, color,
total_kills, total_deaths, kd_ratio, multi2k, multi3k, multi4k, multi5k,
hltv_rating, hsp, rounds_survived, rounds_survived_percentage, dpr,
total_assists, total_damage, trade_kill_opportunities, trade_kill_attempts,
trade_kills_succeeded, trade_kill_attempts_percentage,
trade_kills_success_percentage, trade_kill_opportunities_per_round,
traded_death_opportunities, traded_death_attempts,
traded_death_attempts_percentage, traded_deaths_succeeded,
traded_deaths_success_percentage, traded_deaths_opportunities_per_round,
leetify_rating, personal_performance_rating, ct_leetify_rating,
t_leetify_rating, leetify_user_id,
is_collector, is_pro_plan, is_leetify_staff
FROM data.player_stats;
ROLLBACK;

8
sqitch/verify/profile-game.sql
View File

@ -3,10 +3,10 @@
BEGIN;
-- XXX Add verifications here.
SELECT leetifyUserId, ctLeetifyRating, ctLeetifyRatingRounds, dataSource, elo,
gameFinishedAt, gameId, isCs2, mapName, matchResult, scores, skillLevel,
tLeetifyRating, tLeetifyRatingRounds, deaths, hasBannedPlayer,
kills, partySize
SELECT leetify_user_id, ct_Leetify_rating, ct_Leetify_rating_rounds,
data_source, elo, game_finished_at, game_id, is_cs2, map_name, match_result,
scores, skill_level, t_leetify_rating, t_leetify_rating_rounds, deaths,
has_banned_player, kills, party_size
FROM data.profile_game
WHERE FALSE;

4
sqitch/verify/profile-meta.sql
View File

@ -2,8 +2,8 @@
BEGIN;
SELECT steam64Id, isCollector, isLeetifyStaff,
isProPlan, leetifyUserId, faceitNickname
SELECT name, steam64_id, is_collector, is_leetify_staff, is_pro_plan,
leetify_user_id, faceit_nickname
FROM data.profile_meta
WHERE FALSE;

316
src/extract.py
View File

@ -1,6 +1,15 @@
import psycopg2 as pg
from .leetify import Leetify
from os import environ as env
from .transform import (
meta_from_profile,
games_from_profile,
player_stats_from_game,
)
import psycopg2 as pg
from typing import Tuple
## TODO seperate out loading from extraction
## this currently handles getting data from api and loading into db
def connect():
return pg.connect(
@ -10,3 +19,308 @@ def connect():
host=env.get("POSTGRES_HOST"),
port=env.get("POSTGRES_PORT"),
)
def vals_in_order(data: dict, keys: list):
vals = [data[key] for key in keys]
return vals
def dict_to_col_val(data: dict) -> Tuple[list, list]:
cols = []
vals = []
for k, v in data.items():
cols.append(k)
vals.append(v)
return cols, vals
def dict_to_sql_str(data: dict) -> Tuple[str, str]:
cols, vals = dict_to_col_val(data)
vals = map(str, vals)
cols = ", ".join(cols)
vals = ", ".join(vals)
return cols, vals
def extract_profile_meta(id, api=Leetify(), conn=None):
if not conn:
conn = connect()
cols_api = [
"name",
"steam64Id",
"isCollector",
"isLeetifyStaff",
"isProPlan",
"leetifyUserId",
"faceitNickname",
]
cols_db = [
"name",
"steam64_id",
"is_collector",
"is_leetify_staff",
"is_pro_plan",
"leetify_user_id",
"faceit_nickname",
]
profile = api.get_profile(id)
meta = meta_from_profile(profile)
vals = vals_in_order(meta, cols_api)
with conn.cursor() as curs:
sql = f"""
INSERT INTO data.profile_meta ({', '.join(cols_db)})
VALUES ({', '.join(['%s'] * len(cols_db))});
"""
curs.execute(sql, vals)
conn.commit()
def extract_profile_games(profile, conn=None):
if not conn:
conn = connect()
cols_api = [
"leetifyUserId",
"ctLeetifyRating",
"ctLeetifyRatingRounds",
"dataSource",
"elo",
"gameFinishedAt",
"gameId",
"isCs2",
"mapName",
"matchResult",
"scores",
"skillLevel",
"tLeetifyRating",
"tLeetifyRatingRounds",
"deaths",
"hasBannedPlayer",
"kills",
"partySize",
]
cols_db = [
"leetify_user_id",
"ct_Leetify_rating",
"ct_Leetify_rating_rounds",
"data_source",
"elo",
"game_finished_at",
"game_id",
"is_cs2",
"map_name",
"match_result",
"scores",
"skill_level",
"t_leetify_rating",
"t_leetify_rating_rounds",
"deaths",
"has_banned_player",
"kills",
"party_size",
]
games = games_from_profile(profile)
games = map(lambda game: vals_in_order(game, cols_api), games)
with conn.cursor() as curs:
curs.executemany(
f"""
INSERT into data.profile_game ({', '.join(cols_db)})
VALUES ({', '.join(['%s'] * len(cols_db))});
""",
games,
)
conn.commit()
def extract_player_stats(game, conn=None):
if not conn:
conn = connect()
cols_api = [
"id",
"gameId",
"gameFinishedAt",
"steam64Id",
"name",
"preaim",
"reactionTime",
"accuracy",
"accuracyEnemySpotted",
"accuracyHead",
"shotsFiredEnemySpotted",
"shotsFired",
"shotsHitEnemySpotted",
"shotsHitFriend",
"shotsHitFriendHead",
"shotsHitFoe",
"shotsHitFoeHead",
"utilityOnDeathAvg",
"heFoesDamageAvg",
"heFriendsDamageAvg",
"heThrown",
"molotovThrown",
"smokeThrown",
"smokeThrownCT",
"smokeThrownCTGood",
"smokeThrownCTGoodRatio",
"smokeThrownCTFoes",
"counterStrafingShotsAll",
"counterStrafingShotsBad",
"counterStrafingShotsGood",
"counterStrafingShotsGoodRatio",
"flashbangHitFoe",
"flashbangLeadingToKill",
"flashbangHitFoeAvgDuration",
"flashbangHitFriend",
"flashbangThrown",
"flashAssist",
"score",
"initialTeamNumber",
"mvps",
"ctRoundsWon",
"ctRoundsLost",
"tRoundsWon",
"tRoundsLost",
"sprayAccuracy",
"molotovFoesDamageAvg",
"molotovFriendsDamageAvg",
"color",
"totalKills",
"totalDeaths",
"kdRatio",
"multi2k",
"multi3k",
"multi4k",
"multi5k",
"hltvRating",
"hsp",
"roundsSurvived",
"roundsSurvivedPercentage",
"dpr",
"totalAssists",
"totalDamage",
"tradeKillOpportunities",
"tradeKillAttempts",
"tradeKillsSucceeded",
"tradeKillAttemptsPercentage",
"tradeKillsSuccessPercentage",
"tradeKillOpportunitiesPerRound",
"tradedDeathOpportunities",
"tradedDeathAttempts",
"tradedDeathAttemptsPercentage",
"tradedDeathsSucceeded",
"tradedDeathsSuccessPercentage",
"tradedDeathsOpportunitiesPerRound",
"leetifyRating",
"personalPerformanceRating",
"ctLeetifyRating",
"tLeetifyRating",
"leetifyUserId",
"isCollector",
"isProPlan",
"isLeetifyStaff",
]
cols_db = [
"id",
"game_id",
"game_finished_at",
"steam64_id",
"name",
"preaim",
"reaction_time",
"accuracy",
"accuracy_enemy_spotted",
"accuracy_head",
"shots_fired_enemy_spotted",
"shots_fired",
"shots_hit_enemy_spotted",
"shots_hit_friend",
"shots_Hit_Friend_Head",
"shots_Hit_Foe",
"shots_Hit_Foe_Head",
"utility_On_Death_Avg",
"he_foes_damage_avg",
"he_friends_damage_avg",
"he_thrown",
"molotov_thrown",
"smoke_thrown",
"smoke_thrown_ct",
"smoke_thrown_ct_good",
"smoke_thrown_ct_good_ratio",
"smoke_thrown_ct_foes",
"counter_strafing_shots_all",
"counter_strafing_shots_bad",
"counter_strafing_shots_good",
"counter_strafing_shots_good_ratio",
"flashbang_hit_foe",
"flashbang_leading_to_kill",
"flashbang_hit_foe_avg_duration",
"flashbang_hit_friend",
"flashbang_thrown",
"flash_assist",
"score",
"initial_Team_Number",
"mvps",
"ct_rounds_won",
"ct_rounds_lost",
"t_rounds_won",
"t_rounds_lost",
"spray_accuracy",
"molotov_foes_damage_avg",
"molotov_friends_damage_avg",
"color",
"total_kills",
"total_deaths",
"kd_ratio",
"multi2k",
"multi3k",
"multi4k",
"multi5k",
"hltv_rating",
"hsp",
"rounds_survived",
"rounds_survived_percentage",
"dpr",
"total_assists",
"total_damage",
"trade_kill_opportunities",
"trade_kill_attempts",
"trade_kills_succeeded",
"trade_kill_attempts_percentage",
"trade_kills_success_percentage",
"trade_kill_opportunities_per_round",
"traded_death_opportunities",
"traded_death_attempts",
"traded_death_attempts_percentage",
"traded_deaths_succeeded",
"traded_deaths_success_percentage",
"traded_deaths_opportunities_per_round",
"leetify_rating",
"personal_performance_rating",
"ct_leetify_rating",
"t_leetify_rating",
"leetify_user_id",
"is_collector",
"is_pro_plan",
"is_leetify_staff",
]
stats = player_stats_from_game(game)
vals = map(lambda game: vals_in_order(game, cols_api), stats)
with conn.cursor() as curs:
sql = f"""
INSERT into data.player_stats ({', '.join(cols_db)})
VALUES ({', '.join(['%s'] * len(cols_db))}); """
curs.executemany(sql, vals)
conn.commit()

5
src/leetify/core.py
View File

@ -1,4 +1,7 @@
from ratelimit.decorators import sleep_and_retry
import requests
from ratelimit import limits, RateLimitException
from backoff import expo, on_exception
class Leetify:
@ -6,6 +9,8 @@ class Leetify:
profile_base_url = f"{api_base_url}/profile"
match_base_url = f"{api_base_url}/games"
@sleep_and_retry
@limits(1, 5)
def __get_page(self, url: str) -> dict:
resp = requests.get(url)
return resp.json()