andrei-stoica
/
leetify-data
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

alpha runner

This commit is contained in:
Andrei Stoica 2024-02-01 13:00:41 -05:00
parent bf8977648a
commit 47d8865be8
3 changed files with 48 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/db.py
View File

@ -1,10 +1,10 @@
import psycopg2 as pg
from os import environ as env
from extract import vals_in_order
from transform import vals_in_order
def connect():
print("connecting")
return pg.connect(
dbname=env.get("POSTGRES_DB"),
user=env.get("POSTGRES_USER"),
@ -14,10 +14,17 @@ def connect():
)
def get_all_profile_names(conn=None):
if not conn:
conn = connect()
def connect_by_default(func, conn_func=connect):
def wrapper(*args, **kargs):
if not kargs.get("conn"):
kargs["conn"] = conn_func()
return func(*args, **kargs)
return wrapper
@connect_by_default
def get_all_profile_names(conn=None):
with conn.cursor() as curs:
curs.execute("""SELECT name, leetify_user_id FROM data.profile_meta;""")
data = curs.fetchall()
@ -25,10 +32,8 @@ def get_all_profile_names(conn=None):
return data
@connect_by_default
def get_profile_game_count(user_id, conn=None):
if not conn:
conn = connect()
with conn.cursor() as curs:
curs.execute(
"""SELECT count(*) FROM data.profile_game
@ -42,10 +47,8 @@ def get_profile_game_count(user_id, conn=None):
return data[0]
@connect_by_default
def get_profile_game_ids(user_id, conn=None):
if not conn:
conn = connect()
with conn.cursor() as curs:
curs.execute(
"""SELECT game_id FROM data.profile_game
@ -57,10 +60,8 @@ def get_profile_game_ids(user_id, conn=None):
return [row[0] for row in data]
@connect_by_default
def insert_games(games, conn=None):
if not conn:
conn = connect()
cols_api = [
"leetifyUserId",
"ctLeetifyRating",

89
src/extract.py
View File

@ -3,45 +3,16 @@ from transform import (
meta_from_profile,
games_from_profile,
player_stats_from_game,
vals_in_order,
)
import db
from typing import Tuple
import logging
from itertools import chain
## TODO seperate out loading from extraction
## this currently handles getting data from api and loading into db
def vals_in_order(data: dict, keys: list):
vals = [data[key] for key in keys]
return vals
def dict_to_col_val(data: dict) -> Tuple[list, list]:
cols = []
vals = []
for k, v in data.items():
cols.append(k)
vals.append(v)
return cols, vals
def dict_to_sql_str(data: dict) -> Tuple[str, str]:
cols, vals = dict_to_col_val(data)
vals = map(str, vals)
cols = ", ".join(cols)
vals = ", ".join(vals)
return cols, vals
@db.connect_by_default
def extract_profile_meta(id, api=Leetify(), conn=None):
if not conn:
conn = db.connect()
cols_api = [
"name",
"steam64Id",
@ -74,10 +45,8 @@ def extract_profile_meta(id, api=Leetify(), conn=None):
conn.commit()
@db.connect_by_default
def extract_profile_games(profile, conn=None):
if not conn:
conn = db.connect()
cols_api = [
"leetifyUserId",
"ctLeetifyRating",
@ -134,10 +103,8 @@ def extract_profile_games(profile, conn=None):
conn.commit()
@db.connect_by_default
def extract_player_stats(game, conn=None):
if not conn:
conn = db.connect()
cols_api = [
"id",
"gameId",
@ -318,58 +285,56 @@ def extract_player_stats(game, conn=None):
conn.commit()
@db.connect_by_default
def insert_new_profile_games(games, user_id, conn=None):
if not conn:
conn = db.connect()
game_ids = db.get_profile_game_ids(user_id, conn)
game_ids = db.get_profile_game_ids(user_id, conn=conn)
game_ids = set(game_ids)
new_games = [game for game in games if game.get("gameId") not in game_ids]
logging.info(f"Inserting {len(new_games)} for: {user_id}")
db.insert_games(new_games)
db.insert_games(new_games, conn=conn)
return new_games
def insert_player_stats(game_ids, conn=None, api=Leetify()):
if not conn:
conn = db.connect()
player_stats = map(api.get_match, game_ids)
player_stats = map(player_stats_from_game, player_stats)
player_stats = chain.from_iterable(player_stats)
print(player_stats)
@db.connect_by_default
def insert_player_stats(game_ids, api=Leetify(), conn=None):
logging.info(f"Inserting player_stats from {len(game_ids)} games")
print_every = int(len(game_ids) / 10)
for i, id in enumerate(game_ids):
if i % print_every == 0:
logging.info(f"Inserted player_stats from {i}/{len(game_ids)} games")
game = api.get_match(id)
extract_player_stats(game, conn=conn)
def get_all(api=Leetify()):
conn = db.connect()
data = db.get_all_profile_names(conn)
games = []
data = db.get_all_profile_names(conn=conn)
game_ids = []
for name, id in data:
logging.info(f"Getting data for {name}: {id}")
profile = api.get_profile(id)
games = games_from_profile(profile)
profile_games = games_from_profile(profile)
api_games = len(games)
db_games = db.get_profile_game_count(id, conn)
api_games = len(profile_games)
db_games = db.get_profile_game_count(id, conn=conn)
if api_games > db_games:
logging.info(f"Getting new games for {name}: {id}")
new_games = insert_new_profile_games(games, id, conn)
games.extend(new_games)
new_games = insert_new_profile_games(profile_games, id, conn=conn)
new_game_ids = [game.get("gameId") for game in new_games]
game_ids.extend(new_game_ids)
elif api_games < db_games:
logging.error(f"API returned less games then in DB for {name}: {id}")
logging.info(f"Games synced with Leetify for {name}: {id}")
game_ids = [game.get("game_id") for game in games]
insert_player_stats(game_ids, conn)
# TODO get sync player_stats and profile_games
logging.info(f"Updating player_stats with {len(game_ids)} new games")
insert_player_stats(game_ids, conn=conn)
logging.info("DONE Updating player_stats with new games")
if __name__ == "__main__":

6
src/transform.py
View File

@ -4,6 +4,11 @@ T = TypeVar("T")
U = TypeVar("U")
def vals_in_order(data: dict, keys: list):
vals = [data[key] for key in keys]
return vals
def extract_cols(data: dict, cols: list[str]) -> dict:
return {key: data.get(key) for key in cols}
@ -183,6 +188,5 @@ def player_stats_from_game(game: dict) -> list[dict]:
if not stats:
raise Exception("Could not get stats from game", game)
stats = map(cols_from_player_stats, stats)
return list(stats)