prototype extracting
This commit is contained in:
parent
6c41cda563
commit
30497075be
|
|
@ -1,3 +1,5 @@
|
||||||
pytest
|
pytest
|
||||||
click
|
click
|
||||||
psycopg2
|
psycopg2-binary
|
||||||
|
ratelimit
|
||||||
|
backoff
|
||||||
|
|
|
||||||
|
|
@ -5,87 +5,87 @@ BEGIN;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS data.player_stats (
|
CREATE TABLE IF NOT EXISTS data.player_stats (
|
||||||
id TEXT,
|
id TEXT,
|
||||||
gameId TEXT,
|
game_id TEXT,
|
||||||
gameFinishedAt TIMESTAMP,
|
game_finished_at TIMESTAMP,
|
||||||
steam64Id TEXT,
|
steam64_id TEXT,
|
||||||
name TEXT,
|
name TEXT,
|
||||||
preaim NUMERIC,
|
preaim NUMERIC,
|
||||||
reactionTime NUMERIC,
|
reaction_time NUMERIC,
|
||||||
accuracy NUMERIC,
|
accuracy NUMERIC,
|
||||||
accuracyEnemySpotted NUMERIC,
|
accuracy_enemy_spotted NUMERIC,
|
||||||
accuracyHead NUMERIC,
|
accuracy_head NUMERIC,
|
||||||
shotsFiredEnemySpotted NUMERIC,
|
shots_fired_enemy_spotted NUMERIC,
|
||||||
shotsFired NUMERIC,
|
shots_fired NUMERIC,
|
||||||
shotsHitEnemySpotted NUMERIC,
|
shots_hit_enemy_spotted NUMERIC,
|
||||||
shotsHitFriend NUMERIC,
|
shots_hit_friend NUMERIC,
|
||||||
shotsHitFriendHead NUMERIC,
|
shots_Hit_Friend_Head NUMERIC,
|
||||||
shotsHitFoe NUMERIC,
|
shots_Hit_Foe NUMERIC,
|
||||||
shotsHitFoeHead NUMERIC,
|
shots_Hit_Foe_Head NUMERIC,
|
||||||
utilityOnDeathAvg NUMERIC,
|
utility_On_Death_Avg NUMERIC,
|
||||||
heFoesDamageAvg NUMERIC,
|
he_foes_damage_avg NUMERIC,
|
||||||
heFriendsDamageAvg NUMERIC,
|
he_friends_damage_avg NUMERIC,
|
||||||
heThrown NUMERIC,
|
he_thrown NUMERIC,
|
||||||
molotovThrown NUMERIC,
|
molotov_thrown NUMERIC,
|
||||||
smokeThrown NUMERIC,
|
smoke_thrown NUMERIC,
|
||||||
smokeThrownCT NUMERIC,
|
smoke_thrown_ct NUMERIC,
|
||||||
smokeThrownCTGood NUMERIC,
|
smoke_thrown_ct_good NUMERIC,
|
||||||
smokeThrownCTGoodRatio NUMERIC,
|
smoke_thrown_ct_good_ratio NUMERIC,
|
||||||
smokeThrownCTFoes NUMERIC,
|
smoke_thrown_ct_foes NUMERIC,
|
||||||
counterStrafingShotsAll NUMERIC,
|
counter_strafing_shots_all NUMERIC,
|
||||||
counterStrafingShotsBad NUMERIC,
|
counter_strafing_shots_bad NUMERIC,
|
||||||
counterStrafingShotsGood NUMERIC,
|
counter_strafing_shots_good NUMERIC,
|
||||||
counterStrafingShotsGoodRatio NUMERIC,
|
counter_strafing_shots_good_ratio NUMERIC,
|
||||||
flashbangHitFoe NUMERIC,
|
flashbang_hit_foe NUMERIC,
|
||||||
flashbangLeadingToKill NUMERIC,
|
flashbang_leading_to_kill NUMERIC,
|
||||||
flashbangHitFoeAvgDuration NUMERIC,
|
flashbang_hit_foe_avg_duration NUMERIC,
|
||||||
flashbangHitFriend NUMERIC,
|
flashbang_hit_friend NUMERIC,
|
||||||
flashbangThrown NUMERIC,
|
flashbang_thrown NUMERIC,
|
||||||
flashAssist NUMERIC,
|
flash_assist NUMERIC,
|
||||||
score NUMERIC,
|
score NUMERIC,
|
||||||
initialTeamNumber NUMERIC,
|
initial_Team_Number NUMERIC,
|
||||||
mvps NUMERIC,
|
mvps NUMERIC,
|
||||||
ctRoundsWon NUMERIC,
|
ct_rounds_won NUMERIC,
|
||||||
ctRoundsLost NUMERIC,
|
ct_rounds_lost NUMERIC,
|
||||||
tRoundsWon NUMERIC,
|
t_rounds_won NUMERIC,
|
||||||
tRoundsLost NUMERIC,
|
t_rounds_lost NUMERIC,
|
||||||
sprayAccuracy NUMERIC,
|
spray_accuracy NUMERIC,
|
||||||
molotovFoesDamageAvg NUMERIC,
|
molotov_foes_damage_avg NUMERIC,
|
||||||
molotovFriendsDamageAvg NUMERIC,
|
molotov_friends_damage_avg NUMERIC,
|
||||||
color NUMERIC,
|
color NUMERIC,
|
||||||
totalKills NUMERIC,
|
total_kills NUMERIC,
|
||||||
totalDeaths NUMERIC,
|
total_deaths NUMERIC,
|
||||||
kdRatio NUMERIC,
|
kd_ratio NUMERIC,
|
||||||
multi2k NUMERIC,
|
multi2k NUMERIC,
|
||||||
multi3k NUMERIC,
|
multi3k NUMERIC,
|
||||||
multi4k NUMERIC,
|
multi4k NUMERIC,
|
||||||
multi5k NUMERIC,
|
multi5k NUMERIC,
|
||||||
hltvRating NUMERIC,
|
hltv_rating NUMERIC,
|
||||||
hsp NUMERIC,
|
hsp NUMERIC,
|
||||||
roundsSurvived NUMERIC,
|
rounds_survived NUMERIC,
|
||||||
roundsSurvivedPercentage NUMERIC,
|
rounds_survived_percentage NUMERIC,
|
||||||
dpr NUMERIC,
|
dpr NUMERIC,
|
||||||
totalAssists NUMERIC,
|
total_assists NUMERIC,
|
||||||
totalDamage NUMERIC,
|
total_damage NUMERIC,
|
||||||
tradeKillOpportunities NUMERIC,
|
trade_kill_opportunities NUMERIC,
|
||||||
tradeKillAttempts NUMERIC,
|
trade_kill_attempts NUMERIC,
|
||||||
tradeKillsSucceeded NUMERIC,
|
trade_kills_succeeded NUMERIC,
|
||||||
tradeKillAttemptsPercentage NUMERIC,
|
trade_kill_attempts_percentage NUMERIC,
|
||||||
tradeKillsSuccessPercentage NUMERIC,
|
trade_kills_success_percentage NUMERIC,
|
||||||
tradeKillOpportunitiesPerRound NUMERIC,
|
trade_kill_opportunities_per_round NUMERIC,
|
||||||
tradedDeathOpportunities NUMERIC,
|
traded_death_opportunities NUMERIC,
|
||||||
tradedDeathAttempts NUMERIC,
|
traded_death_attempts NUMERIC,
|
||||||
tradedDeathAttemptsPercentage NUMERIC,
|
traded_death_attempts_percentage NUMERIC,
|
||||||
tradedDeathsSucceeded NUMERIC,
|
traded_deaths_succeeded NUMERIC,
|
||||||
tradedDeathsSuccessPercentage NUMERIC,
|
traded_deaths_success_percentage NUMERIC,
|
||||||
tradedDeathsOpportunitiesPerRound NUMERIC,
|
traded_deaths_opportunities_per_round NUMERIC,
|
||||||
leetifyRating NUMERIC,
|
leetify_rating NUMERIC,
|
||||||
personalPerformanceRating NUMERIC,
|
personal_performance_rating NUMERIC,
|
||||||
ctLeetifyRating NUMERIC,
|
ct_leetify_rating NUMERIC,
|
||||||
tLeetifyRating NUMERIC,
|
t_leetify_rating NUMERIC,
|
||||||
leetifyUserId TEXT,
|
leetify_user_id TEXT,
|
||||||
isCollector BOOL,
|
is_collector BOOL,
|
||||||
isProPlan BOOL,
|
is_pro_plan BOOL,
|
||||||
isLeetifyStaff BOOL
|
is_leetify_staff BOOL
|
||||||
);
|
);
|
||||||
|
|
||||||
COMMIT;
|
COMMIT;
|
||||||
|
|
|
||||||
|
|
@ -4,24 +4,24 @@
|
||||||
BEGIN;
|
BEGIN;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS data.profile_game (
|
CREATE TABLE IF NOT EXISTS data.profile_game (
|
||||||
leetifyUserId TEXT,
|
leetify_user_id TEXT,
|
||||||
ctLeetifyRating NUMERIC,
|
ct_Leetify_rating NUMERIC,
|
||||||
ctLeetifyRatingRounds NUMERIC,
|
ct_Leetify_rating_rounds NUMERIC,
|
||||||
dataSource NUMERIC,
|
data_source TEXT,
|
||||||
elo NUMERIC,
|
elo NUMERIC,
|
||||||
gameFinishedAt TIMESTAMP,
|
game_finished_at TIMESTAMP,
|
||||||
gameId TEXT,
|
game_id TEXT,
|
||||||
isCs2 BOOL,
|
is_cs2 BOOL,
|
||||||
mapName TEXT,
|
map_name TEXT,
|
||||||
matchResult TEXT,
|
match_result TEXT,
|
||||||
scores TEXT, -- has to be extracted from array
|
scores TEXT, -- has to be extracted from array
|
||||||
skillLevel NUMERIC,
|
skill_level NUMERIC,
|
||||||
tLeetifyRating NUMERIC,
|
t_leetify_rating NUMERIC,
|
||||||
tLeetifyRatingRounds NUMERIC,
|
t_leetify_rating_rounds NUMERIC,
|
||||||
deaths NUMERIC,
|
deaths NUMERIC,
|
||||||
hasBannedPlayer BOOL,
|
has_banned_player BOOL,
|
||||||
kills NUMERIC,
|
kills NUMERIC,
|
||||||
partySize NUMERIC
|
party_size NUMERIC
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,12 +4,13 @@
|
||||||
BEGIN;
|
BEGIN;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS data.profile_meta (
|
CREATE TABLE IF NOT EXISTS data.profile_meta (
|
||||||
steam64Id TEXT,
|
name TEXT,
|
||||||
isCollector BOOL,
|
steam64_id TEXT,
|
||||||
isLeetifyStaff BOOL,
|
is_collector BOOL,
|
||||||
isProPlan BOOL,
|
is_leetify_staff BOOL,
|
||||||
leetifyUserId TEXT,
|
is_pro_plan BOOL,
|
||||||
faceitNickname TEXT
|
leetify_user_id TEXT,
|
||||||
|
faceit_nickname TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
COMMIT;
|
COMMIT;
|
||||||
|
|
|
||||||
|
|
@ -3,26 +3,31 @@
|
||||||
BEGIN;
|
BEGIN;
|
||||||
|
|
||||||
-- XXX Add verifications here.
|
-- XXX Add verifications here.
|
||||||
SELECT id, gameId, gameFinishedAt, steam64Id, name, preaim, reactionTime,
|
SELECT id, game_id, game_finished_at, steam64_id, name, preaim, reaction_time,
|
||||||
accuracy, accuracyEnemySpotted, accuracyHead, shotsFiredEnemySpotted,
|
accuracy, accuracy_enemy_spotted, accuracy_head, shots_fired_enemy_spotted,
|
||||||
shotsFired, shotsHitEnemySpotted, shotsHitFriend, shotsHitFriendHead,
|
shots_fired, shots_hit_enemy_spotted, shots_hit_friend,
|
||||||
shotsHitFoe, shotsHitFoeHead, utilityOnDeathAvg, heFoesDamageAvg,
|
shots_Hit_Friend_Head, shots_Hit_Foe, shots_Hit_Foe_Head,
|
||||||
heFriendsDamageAvg, heThrown, molotovThrown, smokeThrown,
|
utility_On_Death_Avg, he_foes_damage_avg, he_friends_damage_avg, he_thrown,
|
||||||
smokeThrownCT, smokeThrownCTGood, smokeThrownCTGoodRatio, smokeThrownCTFoes,
|
molotov_thrown, smoke_thrown, smoke_thrown_ct, smoke_thrown_ct_good,
|
||||||
counterStrafingShotsAll, counterStrafingShotsBad, counterStrafingShotsGood,
|
smoke_thrown_ct_good_ratio, smoke_thrown_ct_foes,
|
||||||
counterStrafingShotsGoodRatio, flashbangHitFoe, flashbangLeadingToKill,
|
counter_strafing_shots_all, counter_strafing_shots_bad,
|
||||||
flashbangHitFoeAvgDuration, flashbangHitFriend, flashbangThrown, flashAssist,
|
counter_strafing_shots_good, counter_strafing_shots_good_ratio,
|
||||||
score, initialTeamNumber, mvps, ctRoundsWon, ctRoundsLost, tRoundsWon,
|
flashbang_hit_foe, flashbang_leading_to_kill,
|
||||||
tRoundsLost, sprayAccuracy, molotovFoesDamageAvg, molotovFriendsDamageAvg,
|
flashbang_hit_foe_avg_duration, flashbang_hit_friend, flashbang_thrown,
|
||||||
color, totalKills, totalDeaths, kdRatio, multi2k, multi3k, multi4k, multi5k,
|
flash_assist, score, initial_Team_Number, mvps, ct_rounds_won,
|
||||||
hltvRating, hsp, roundsSurvived, roundsSurvivedPercentage, dpr, totalAssists,
|
ct_rounds_lost, t_rounds_won, t_rounds_lost, spray_accuracy,
|
||||||
totalDamage, tradeKillOpportunities, tradeKillAttempts, tradeKillsSucceeded,
|
molotov_foes_damage_avg, molotov_friends_damage_avg, color,
|
||||||
tradeKillAttemptsPercentage, tradeKillsSuccessPercentage,
|
total_kills, total_deaths, kd_ratio, multi2k, multi3k, multi4k, multi5k,
|
||||||
tradeKillOpportunitiesPerRound, tradedDeathOpportunities,
|
hltv_rating, hsp, rounds_survived, rounds_survived_percentage, dpr,
|
||||||
tradedDeathAttempts, tradedDeathAttemptsPercentage, tradedDeathsSucceeded,
|
total_assists, total_damage, trade_kill_opportunities, trade_kill_attempts,
|
||||||
tradedDeathsSuccessPercentage, leetifyRating, personalPerformanceRating,
|
trade_kills_succeeded, trade_kill_attempts_percentage,
|
||||||
ctLeetifyRating, tLeetifyRating, leetifyUserId, isCollector, isProPlan,
|
trade_kills_success_percentage, trade_kill_opportunities_per_round,
|
||||||
isLeetifyStaff
|
traded_death_opportunities, traded_death_attempts,
|
||||||
|
traded_death_attempts_percentage, traded_deaths_succeeded,
|
||||||
|
traded_deaths_success_percentage, traded_deaths_opportunities_per_round,
|
||||||
|
leetify_rating, personal_performance_rating, ct_leetify_rating,
|
||||||
|
t_leetify_rating, leetify_user_id,
|
||||||
|
is_collector, is_pro_plan, is_leetify_staff
|
||||||
FROM data.player_stats;
|
FROM data.player_stats;
|
||||||
|
|
||||||
ROLLBACK;
|
ROLLBACK;
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,10 @@
|
||||||
BEGIN;
|
BEGIN;
|
||||||
|
|
||||||
-- XXX Add verifications here.
|
-- XXX Add verifications here.
|
||||||
SELECT leetifyUserId, ctLeetifyRating, ctLeetifyRatingRounds, dataSource, elo,
|
SELECT leetify_user_id, ct_Leetify_rating, ct_Leetify_rating_rounds,
|
||||||
gameFinishedAt, gameId, isCs2, mapName, matchResult, scores, skillLevel,
|
data_source, elo, game_finished_at, game_id, is_cs2, map_name, match_result,
|
||||||
tLeetifyRating, tLeetifyRatingRounds, deaths, hasBannedPlayer,
|
scores, skill_level, t_leetify_rating, t_leetify_rating_rounds, deaths,
|
||||||
kills, partySize
|
has_banned_player, kills, party_size
|
||||||
FROM data.profile_game
|
FROM data.profile_game
|
||||||
WHERE FALSE;
|
WHERE FALSE;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
BEGIN;
|
BEGIN;
|
||||||
|
|
||||||
SELECT steam64Id, isCollector, isLeetifyStaff,
|
SELECT name, steam64_id, is_collector, is_leetify_staff, is_pro_plan,
|
||||||
isProPlan, leetifyUserId, faceitNickname
|
leetify_user_id, faceit_nickname
|
||||||
FROM data.profile_meta
|
FROM data.profile_meta
|
||||||
WHERE FALSE;
|
WHERE FALSE;
|
||||||
|
|
||||||
|
|
|
||||||
316
src/extract.py
316
src/extract.py
|
|
@ -1,6 +1,15 @@
|
||||||
import psycopg2 as pg
|
from .leetify import Leetify
|
||||||
from os import environ as env
|
from os import environ as env
|
||||||
|
from .transform import (
|
||||||
|
meta_from_profile,
|
||||||
|
games_from_profile,
|
||||||
|
player_stats_from_game,
|
||||||
|
)
|
||||||
|
import psycopg2 as pg
|
||||||
|
from typing import Tuple
|
||||||
|
|
||||||
|
## TODO seperate out loading from extraction
|
||||||
|
## this currently handles getting data from api and loading into db
|
||||||
|
|
||||||
def connect():
|
def connect():
|
||||||
return pg.connect(
|
return pg.connect(
|
||||||
|
|
@ -10,3 +19,308 @@ def connect():
|
||||||
host=env.get("POSTGRES_HOST"),
|
host=env.get("POSTGRES_HOST"),
|
||||||
port=env.get("POSTGRES_PORT"),
|
port=env.get("POSTGRES_PORT"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def vals_in_order(data: dict, keys: list):
|
||||||
|
vals = [data[key] for key in keys]
|
||||||
|
return vals
|
||||||
|
|
||||||
|
|
||||||
|
def dict_to_col_val(data: dict) -> Tuple[list, list]:
|
||||||
|
cols = []
|
||||||
|
vals = []
|
||||||
|
for k, v in data.items():
|
||||||
|
cols.append(k)
|
||||||
|
vals.append(v)
|
||||||
|
|
||||||
|
return cols, vals
|
||||||
|
|
||||||
|
|
||||||
|
def dict_to_sql_str(data: dict) -> Tuple[str, str]:
|
||||||
|
cols, vals = dict_to_col_val(data)
|
||||||
|
vals = map(str, vals)
|
||||||
|
|
||||||
|
cols = ", ".join(cols)
|
||||||
|
vals = ", ".join(vals)
|
||||||
|
|
||||||
|
return cols, vals
|
||||||
|
|
||||||
|
|
||||||
|
def extract_profile_meta(id, api=Leetify(), conn=None):
|
||||||
|
if not conn:
|
||||||
|
conn = connect()
|
||||||
|
|
||||||
|
cols_api = [
|
||||||
|
"name",
|
||||||
|
"steam64Id",
|
||||||
|
"isCollector",
|
||||||
|
"isLeetifyStaff",
|
||||||
|
"isProPlan",
|
||||||
|
"leetifyUserId",
|
||||||
|
"faceitNickname",
|
||||||
|
]
|
||||||
|
cols_db = [
|
||||||
|
"name",
|
||||||
|
"steam64_id",
|
||||||
|
"is_collector",
|
||||||
|
"is_leetify_staff",
|
||||||
|
"is_pro_plan",
|
||||||
|
"leetify_user_id",
|
||||||
|
"faceit_nickname",
|
||||||
|
]
|
||||||
|
|
||||||
|
profile = api.get_profile(id)
|
||||||
|
meta = meta_from_profile(profile)
|
||||||
|
vals = vals_in_order(meta, cols_api)
|
||||||
|
|
||||||
|
with conn.cursor() as curs:
|
||||||
|
sql = f"""
|
||||||
|
INSERT INTO data.profile_meta ({', '.join(cols_db)})
|
||||||
|
VALUES ({', '.join(['%s'] * len(cols_db))});
|
||||||
|
"""
|
||||||
|
curs.execute(sql, vals)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_profile_games(profile, conn=None):
|
||||||
|
if not conn:
|
||||||
|
conn = connect()
|
||||||
|
|
||||||
|
cols_api = [
|
||||||
|
"leetifyUserId",
|
||||||
|
"ctLeetifyRating",
|
||||||
|
"ctLeetifyRatingRounds",
|
||||||
|
"dataSource",
|
||||||
|
"elo",
|
||||||
|
"gameFinishedAt",
|
||||||
|
"gameId",
|
||||||
|
"isCs2",
|
||||||
|
"mapName",
|
||||||
|
"matchResult",
|
||||||
|
"scores",
|
||||||
|
"skillLevel",
|
||||||
|
"tLeetifyRating",
|
||||||
|
"tLeetifyRatingRounds",
|
||||||
|
"deaths",
|
||||||
|
"hasBannedPlayer",
|
||||||
|
"kills",
|
||||||
|
"partySize",
|
||||||
|
]
|
||||||
|
|
||||||
|
cols_db = [
|
||||||
|
"leetify_user_id",
|
||||||
|
"ct_Leetify_rating",
|
||||||
|
"ct_Leetify_rating_rounds",
|
||||||
|
"data_source",
|
||||||
|
"elo",
|
||||||
|
"game_finished_at",
|
||||||
|
"game_id",
|
||||||
|
"is_cs2",
|
||||||
|
"map_name",
|
||||||
|
"match_result",
|
||||||
|
"scores",
|
||||||
|
"skill_level",
|
||||||
|
"t_leetify_rating",
|
||||||
|
"t_leetify_rating_rounds",
|
||||||
|
"deaths",
|
||||||
|
"has_banned_player",
|
||||||
|
"kills",
|
||||||
|
"party_size",
|
||||||
|
]
|
||||||
|
|
||||||
|
games = games_from_profile(profile)
|
||||||
|
games = map(lambda game: vals_in_order(game, cols_api), games)
|
||||||
|
|
||||||
|
with conn.cursor() as curs:
|
||||||
|
curs.executemany(
|
||||||
|
f"""
|
||||||
|
INSERT into data.profile_game ({', '.join(cols_db)})
|
||||||
|
VALUES ({', '.join(['%s'] * len(cols_db))});
|
||||||
|
""",
|
||||||
|
games,
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_player_stats(game, conn=None):
|
||||||
|
if not conn:
|
||||||
|
conn = connect()
|
||||||
|
|
||||||
|
cols_api = [
|
||||||
|
"id",
|
||||||
|
"gameId",
|
||||||
|
"gameFinishedAt",
|
||||||
|
"steam64Id",
|
||||||
|
"name",
|
||||||
|
"preaim",
|
||||||
|
"reactionTime",
|
||||||
|
"accuracy",
|
||||||
|
"accuracyEnemySpotted",
|
||||||
|
"accuracyHead",
|
||||||
|
"shotsFiredEnemySpotted",
|
||||||
|
"shotsFired",
|
||||||
|
"shotsHitEnemySpotted",
|
||||||
|
"shotsHitFriend",
|
||||||
|
"shotsHitFriendHead",
|
||||||
|
"shotsHitFoe",
|
||||||
|
"shotsHitFoeHead",
|
||||||
|
"utilityOnDeathAvg",
|
||||||
|
"heFoesDamageAvg",
|
||||||
|
"heFriendsDamageAvg",
|
||||||
|
"heThrown",
|
||||||
|
"molotovThrown",
|
||||||
|
"smokeThrown",
|
||||||
|
"smokeThrownCT",
|
||||||
|
"smokeThrownCTGood",
|
||||||
|
"smokeThrownCTGoodRatio",
|
||||||
|
"smokeThrownCTFoes",
|
||||||
|
"counterStrafingShotsAll",
|
||||||
|
"counterStrafingShotsBad",
|
||||||
|
"counterStrafingShotsGood",
|
||||||
|
"counterStrafingShotsGoodRatio",
|
||||||
|
"flashbangHitFoe",
|
||||||
|
"flashbangLeadingToKill",
|
||||||
|
"flashbangHitFoeAvgDuration",
|
||||||
|
"flashbangHitFriend",
|
||||||
|
"flashbangThrown",
|
||||||
|
"flashAssist",
|
||||||
|
"score",
|
||||||
|
"initialTeamNumber",
|
||||||
|
"mvps",
|
||||||
|
"ctRoundsWon",
|
||||||
|
"ctRoundsLost",
|
||||||
|
"tRoundsWon",
|
||||||
|
"tRoundsLost",
|
||||||
|
"sprayAccuracy",
|
||||||
|
"molotovFoesDamageAvg",
|
||||||
|
"molotovFriendsDamageAvg",
|
||||||
|
"color",
|
||||||
|
"totalKills",
|
||||||
|
"totalDeaths",
|
||||||
|
"kdRatio",
|
||||||
|
"multi2k",
|
||||||
|
"multi3k",
|
||||||
|
"multi4k",
|
||||||
|
"multi5k",
|
||||||
|
"hltvRating",
|
||||||
|
"hsp",
|
||||||
|
"roundsSurvived",
|
||||||
|
"roundsSurvivedPercentage",
|
||||||
|
"dpr",
|
||||||
|
"totalAssists",
|
||||||
|
"totalDamage",
|
||||||
|
"tradeKillOpportunities",
|
||||||
|
"tradeKillAttempts",
|
||||||
|
"tradeKillsSucceeded",
|
||||||
|
"tradeKillAttemptsPercentage",
|
||||||
|
"tradeKillsSuccessPercentage",
|
||||||
|
"tradeKillOpportunitiesPerRound",
|
||||||
|
"tradedDeathOpportunities",
|
||||||
|
"tradedDeathAttempts",
|
||||||
|
"tradedDeathAttemptsPercentage",
|
||||||
|
"tradedDeathsSucceeded",
|
||||||
|
"tradedDeathsSuccessPercentage",
|
||||||
|
"tradedDeathsOpportunitiesPerRound",
|
||||||
|
"leetifyRating",
|
||||||
|
"personalPerformanceRating",
|
||||||
|
"ctLeetifyRating",
|
||||||
|
"tLeetifyRating",
|
||||||
|
"leetifyUserId",
|
||||||
|
"isCollector",
|
||||||
|
"isProPlan",
|
||||||
|
"isLeetifyStaff",
|
||||||
|
]
|
||||||
|
cols_db = [
|
||||||
|
"id",
|
||||||
|
"game_id",
|
||||||
|
"game_finished_at",
|
||||||
|
"steam64_id",
|
||||||
|
"name",
|
||||||
|
"preaim",
|
||||||
|
"reaction_time",
|
||||||
|
"accuracy",
|
||||||
|
"accuracy_enemy_spotted",
|
||||||
|
"accuracy_head",
|
||||||
|
"shots_fired_enemy_spotted",
|
||||||
|
"shots_fired",
|
||||||
|
"shots_hit_enemy_spotted",
|
||||||
|
"shots_hit_friend",
|
||||||
|
"shots_Hit_Friend_Head",
|
||||||
|
"shots_Hit_Foe",
|
||||||
|
"shots_Hit_Foe_Head",
|
||||||
|
"utility_On_Death_Avg",
|
||||||
|
"he_foes_damage_avg",
|
||||||
|
"he_friends_damage_avg",
|
||||||
|
"he_thrown",
|
||||||
|
"molotov_thrown",
|
||||||
|
"smoke_thrown",
|
||||||
|
"smoke_thrown_ct",
|
||||||
|
"smoke_thrown_ct_good",
|
||||||
|
"smoke_thrown_ct_good_ratio",
|
||||||
|
"smoke_thrown_ct_foes",
|
||||||
|
"counter_strafing_shots_all",
|
||||||
|
"counter_strafing_shots_bad",
|
||||||
|
"counter_strafing_shots_good",
|
||||||
|
"counter_strafing_shots_good_ratio",
|
||||||
|
"flashbang_hit_foe",
|
||||||
|
"flashbang_leading_to_kill",
|
||||||
|
"flashbang_hit_foe_avg_duration",
|
||||||
|
"flashbang_hit_friend",
|
||||||
|
"flashbang_thrown",
|
||||||
|
"flash_assist",
|
||||||
|
"score",
|
||||||
|
"initial_Team_Number",
|
||||||
|
"mvps",
|
||||||
|
"ct_rounds_won",
|
||||||
|
"ct_rounds_lost",
|
||||||
|
"t_rounds_won",
|
||||||
|
"t_rounds_lost",
|
||||||
|
"spray_accuracy",
|
||||||
|
"molotov_foes_damage_avg",
|
||||||
|
"molotov_friends_damage_avg",
|
||||||
|
"color",
|
||||||
|
"total_kills",
|
||||||
|
"total_deaths",
|
||||||
|
"kd_ratio",
|
||||||
|
"multi2k",
|
||||||
|
"multi3k",
|
||||||
|
"multi4k",
|
||||||
|
"multi5k",
|
||||||
|
"hltv_rating",
|
||||||
|
"hsp",
|
||||||
|
"rounds_survived",
|
||||||
|
"rounds_survived_percentage",
|
||||||
|
"dpr",
|
||||||
|
"total_assists",
|
||||||
|
"total_damage",
|
||||||
|
"trade_kill_opportunities",
|
||||||
|
"trade_kill_attempts",
|
||||||
|
"trade_kills_succeeded",
|
||||||
|
"trade_kill_attempts_percentage",
|
||||||
|
"trade_kills_success_percentage",
|
||||||
|
"trade_kill_opportunities_per_round",
|
||||||
|
"traded_death_opportunities",
|
||||||
|
"traded_death_attempts",
|
||||||
|
"traded_death_attempts_percentage",
|
||||||
|
"traded_deaths_succeeded",
|
||||||
|
"traded_deaths_success_percentage",
|
||||||
|
"traded_deaths_opportunities_per_round",
|
||||||
|
"leetify_rating",
|
||||||
|
"personal_performance_rating",
|
||||||
|
"ct_leetify_rating",
|
||||||
|
"t_leetify_rating",
|
||||||
|
"leetify_user_id",
|
||||||
|
"is_collector",
|
||||||
|
"is_pro_plan",
|
||||||
|
"is_leetify_staff",
|
||||||
|
]
|
||||||
|
stats = player_stats_from_game(game)
|
||||||
|
|
||||||
|
vals = map(lambda game: vals_in_order(game, cols_api), stats)
|
||||||
|
|
||||||
|
with conn.cursor() as curs:
|
||||||
|
sql = f"""
|
||||||
|
INSERT into data.player_stats ({', '.join(cols_db)})
|
||||||
|
VALUES ({', '.join(['%s'] * len(cols_db))}); """
|
||||||
|
curs.executemany(sql, vals)
|
||||||
|
conn.commit()
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,7 @@
|
||||||
|
from ratelimit.decorators import sleep_and_retry
|
||||||
import requests
|
import requests
|
||||||
|
from ratelimit import limits, RateLimitException
|
||||||
|
from backoff import expo, on_exception
|
||||||
|
|
||||||
|
|
||||||
class Leetify:
|
class Leetify:
|
||||||
|
|
@ -6,6 +9,8 @@ class Leetify:
|
||||||
profile_base_url = f"{api_base_url}/profile"
|
profile_base_url = f"{api_base_url}/profile"
|
||||||
match_base_url = f"{api_base_url}/games"
|
match_base_url = f"{api_base_url}/games"
|
||||||
|
|
||||||
|
@sleep_and_retry
|
||||||
|
@limits(1, 5)
|
||||||
def __get_page(self, url: str) -> dict:
|
def __get_page(self, url: str) -> dict:
|
||||||
resp = requests.get(url)
|
resp = requests.get(url)
|
||||||
return resp.json()
|
return resp.json()
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue