leetify-data/src/extract.py

from .leetify import Leetify
from os import environ as env
from .transform import (
    meta_from_profile,
    games_from_profile,
    player_stats_from_game,
)
import psycopg2 as pg
from typing import Tuple

def connect():
    return pg.connect(
        dbname=env.get("POSTGRES_DB"),
        user=env.get("POSTGRES_USER"),
        password=env.get("POSTGRES_PASSWORD"),
        host=env.get("POSTGRES_HOST"),
        port=env.get("POSTGRES_PORT"),
    )


def vals_in_order(data: dict, keys: list):
    vals = [data[key] for key in keys]
    return vals


def dict_to_col_val(data: dict) -> Tuple[list, list]:
    cols = []
    vals = []
    for k, v in data.items():
        cols.append(k)
        vals.append(v)

    return cols, vals


def dict_to_sql_str(data: dict) -> Tuple[str, str]:
    cols, vals = dict_to_col_val(data)
    vals = map(str, vals)

    cols = ", ".join(cols)
    vals = ", ".join(vals)

    return cols, vals


def extract_profile_meta(id, api=Leetify(), conn=None):
    if not conn:
        conn = connect()

    cols_api = [
        "name",
        "steam64Id",
        "isCollector",
        "isLeetifyStaff",
        "isProPlan",
        "leetifyUserId",
        "faceitNickname",
    ]
    cols_db = [
        "name",
        "steam64_id",
        "is_collector",
        "is_leetify_staff",
        "is_pro_plan",
        "leetify_user_id",
        "faceit_nickname",
    ]

    profile = api.get_profile(id)
    meta = meta_from_profile(profile)
    vals = vals_in_order(meta, cols_api)

    with conn.cursor() as curs:
        sql = f"""
        INSERT INTO data.profile_meta ({', '.join(cols_db)})
        VALUES ({', '.join(['%s'] * len(cols_db))});
        """
        curs.execute(sql, vals)
    conn.commit()


def extract_profile_games(profile, conn=None):
    if not conn:
        conn = connect()

    cols_api = [
        "leetifyUserId",
        "ctLeetifyRating",
        "ctLeetifyRatingRounds",
        "dataSource",
        "elo",
        "gameFinishedAt",
        "gameId",
        "isCs2",
        "mapName",
        "matchResult",
        "scores",
        "skillLevel",
        "tLeetifyRating",
        "tLeetifyRatingRounds",
        "deaths",
        "hasBannedPlayer",
        "kills",
        "partySize",
    ]

    cols_db = [
        "leetify_user_id",
        "ct_Leetify_rating",
        "ct_Leetify_rating_rounds",
        "data_source",
        "elo",
        "game_finished_at",
        "game_id",
        "is_cs2",
        "map_name",
        "match_result",
        "scores",
        "skill_level",
        "t_leetify_rating",
        "t_leetify_rating_rounds",
        "deaths",
        "has_banned_player",
        "kills",
        "party_size",
    ]

    games = games_from_profile(profile)
    games = map(lambda game: vals_in_order(game, cols_api), games)

    with conn.cursor() as curs:
        curs.executemany(
            f"""
            INSERT into data.profile_game ({', '.join(cols_db)})
            VALUES ({', '.join(['%s'] * len(cols_db))});
            """,
            games,
        )
    conn.commit()


def extract_player_stats(game, conn=None):
    if not conn:
        conn = connect()

    cols_api = [
        "id",
        "gameId",
        "gameFinishedAt",
        "steam64Id",
        "name",
        "preaim",
        "reactionTime",
        "accuracy",
        "accuracyEnemySpotted",
        "accuracyHead",
        "shotsFiredEnemySpotted",
        "shotsFired",
        "shotsHitEnemySpotted",
        "shotsHitFriend",
        "shotsHitFriendHead",
        "shotsHitFoe",
        "shotsHitFoeHead",
        "utilityOnDeathAvg",
        "heFoesDamageAvg",
        "heFriendsDamageAvg",
        "heThrown",
        "molotovThrown",
        "smokeThrown",
        "smokeThrownCT",
        "smokeThrownCTGood",
        "smokeThrownCTGoodRatio",
        "smokeThrownCTFoes",
        "counterStrafingShotsAll",
        "counterStrafingShotsBad",
        "counterStrafingShotsGood",
        "counterStrafingShotsGoodRatio",
        "flashbangHitFoe",
        "flashbangLeadingToKill",
        "flashbangHitFoeAvgDuration",
        "flashbangHitFriend",
        "flashbangThrown",
        "flashAssist",
        "score",
        "initialTeamNumber",
        "mvps",
        "ctRoundsWon",
        "ctRoundsLost",
        "tRoundsWon",
        "tRoundsLost",
        "sprayAccuracy",
        "molotovFoesDamageAvg",
        "molotovFriendsDamageAvg",
        "color",
        "totalKills",
        "totalDeaths",
        "kdRatio",
        "multi2k",
        "multi3k",
        "multi4k",
        "multi5k",
        "hltvRating",
        "hsp",
        "roundsSurvived",
        "roundsSurvivedPercentage",
        "dpr",
        "totalAssists",
        "totalDamage",
        "tradeKillOpportunities",
        "tradeKillAttempts",
        "tradeKillsSucceeded",
        "tradeKillAttemptsPercentage",
        "tradeKillsSuccessPercentage",
        "tradeKillOpportunitiesPerRound",
        "tradedDeathOpportunities",
        "tradedDeathAttempts",
        "tradedDeathAttemptsPercentage",
        "tradedDeathsSucceeded",
        "tradedDeathsSuccessPercentage",
        "tradedDeathsOpportunitiesPerRound",
        "leetifyRating",
        "personalPerformanceRating",
        "ctLeetifyRating",
        "tLeetifyRating",
        "leetifyUserId",
        "isCollector",
        "isProPlan",
        "isLeetifyStaff",
    ]
    cols_db = [
        "id",
        "game_id",
        "game_finished_at",
        "steam64_id",
        "name",
        "preaim",
        "reaction_time",
        "accuracy",
        "accuracy_enemy_spotted",
        "accuracy_head",
        "shots_fired_enemy_spotted",
        "shots_fired",
        "shots_hit_enemy_spotted",
        "shots_hit_friend",
        "shots_Hit_Friend_Head",
        "shots_Hit_Foe",
        "shots_Hit_Foe_Head",
        "utility_On_Death_Avg",
        "he_foes_damage_avg",
        "he_friends_damage_avg",
        "he_thrown",
        "molotov_thrown",
        "smoke_thrown",
        "smoke_thrown_ct",
        "smoke_thrown_ct_good",
        "smoke_thrown_ct_good_ratio",
        "smoke_thrown_ct_foes",
        "counter_strafing_shots_all",
        "counter_strafing_shots_bad",
        "counter_strafing_shots_good",
        "counter_strafing_shots_good_ratio",
        "flashbang_hit_foe",
        "flashbang_leading_to_kill",
        "flashbang_hit_foe_avg_duration",
        "flashbang_hit_friend",
        "flashbang_thrown",
        "flash_assist",
        "score",
        "initial_Team_Number",
        "mvps",
        "ct_rounds_won",
        "ct_rounds_lost",
        "t_rounds_won",
        "t_rounds_lost",
        "spray_accuracy",
        "molotov_foes_damage_avg",
        "molotov_friends_damage_avg",
        "color",
        "total_kills",
        "total_deaths",
        "kd_ratio",
        "multi2k",
        "multi3k",
        "multi4k",
        "multi5k",
        "hltv_rating",
        "hsp",
        "rounds_survived",
        "rounds_survived_percentage",
        "dpr",
        "total_assists",
        "total_damage",
        "trade_kill_opportunities",
        "trade_kill_attempts",
        "trade_kills_succeeded",
        "trade_kill_attempts_percentage",
        "trade_kills_success_percentage",
        "trade_kill_opportunities_per_round",
        "traded_death_opportunities",
        "traded_death_attempts",
        "traded_death_attempts_percentage",
        "traded_deaths_succeeded",
        "traded_deaths_success_percentage",
        "traded_deaths_opportunities_per_round",
        "leetify_rating",
        "personal_performance_rating",
        "ct_leetify_rating",
        "t_leetify_rating",
        "leetify_user_id",
        "is_collector",
        "is_pro_plan",
        "is_leetify_staff",
    ]
    stats = player_stats_from_game(game)

    vals = map(lambda game: vals_in_order(game, cols_api), stats)

    with conn.cursor() as curs:
        sql = f"""
              INSERT into data.player_stats ({', '.join(cols_db)})
              VALUES ({', '.join(['%s'] * len(cols_db))}); """
        curs.executemany(sql, vals)
    conn.commit()