From 4f61fb5ccc551d957c718123b5798b762807ab08 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 11:39:54 -0400 Subject: [PATCH 01/26] moved scripts to module --- recipe_graph/__init__.py | 0 {src => recipe_graph}/db.py | 0 {src => recipe_graph}/insert_sites.py | 0 {src => recipe_graph}/scrape.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 recipe_graph/__init__.py rename {src => recipe_graph}/db.py (100%) rename {src => recipe_graph}/insert_sites.py (100%) rename {src => recipe_graph}/scrape.py (100%) diff --git a/recipe_graph/__init__.py b/recipe_graph/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/db.py b/recipe_graph/db.py similarity index 100% rename from src/db.py rename to recipe_graph/db.py diff --git a/src/insert_sites.py b/recipe_graph/insert_sites.py similarity index 100% rename from src/insert_sites.py rename to recipe_graph/insert_sites.py diff --git a/src/scrape.py b/recipe_graph/scrape.py similarity index 100% rename from src/scrape.py rename to recipe_graph/scrape.py -- 2.40.1 From c30fea1ddc1054c42b75b7b392b9240379973d21 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 11:44:37 -0400 Subject: [PATCH 02/26] added pytest to requirements --- requirements.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/requirements.txt b/requirements.txt index b4b5e06..502ee36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,15 @@ +attrs==22.1.0 beautifulsoup4==4.11.1 greenlet==1.1.2 +iniconfig==1.1.1 +packaging==21.3 +pluggy==1.0.0 psycopg2-binary==2.9.3 +py==1.11.0 PyMySQL==1.0.2 +pyparsing==3.0.9 +pytest==7.1.3 python-dotenv==0.20.0 soupsieve==2.3.2.post1 SQLAlchemy==1.4.39 +tomli==2.0.1 -- 2.40.1 From d96476662b346fd912ff01dc8085be7de5eb2f64 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 13:00:29 -0400 Subject: [PATCH 03/26] updated gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 64b7803..6c17bc8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ data/ *__pycache__ *env *.code-workspace -sandbox/ \ No newline at end of file +sandbox/ +.vscode/ +.pytest_cache/ +*.egg-info/ -- 2.40.1 From a87c0f142ed5ddda7833dc508a6421dd82dcbcf5 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 13:01:19 -0400 Subject: [PATCH 04/26] restructured code for packaging --- pyproject.toml | 6 ++++++ {recipe_graph => src/recipe_graph}/__init__.py | 0 {recipe_graph => src/recipe_graph}/db.py | 0 {recipe_graph => src/recipe_graph}/insert_sites.py | 0 {recipe_graph => src/recipe_graph}/scrape.py | 0 5 files changed, 6 insertions(+) create mode 100644 pyproject.toml rename {recipe_graph => src/recipe_graph}/__init__.py (100%) rename {recipe_graph => src/recipe_graph}/db.py (100%) rename {recipe_graph => src/recipe_graph}/insert_sites.py (100%) rename {recipe_graph => src/recipe_graph}/scrape.py (100%) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a19e8bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[metadata] +name = "recepie_graph" \ No newline at end of file diff --git a/recipe_graph/__init__.py b/src/recipe_graph/__init__.py similarity index 100% rename from recipe_graph/__init__.py rename to src/recipe_graph/__init__.py diff --git a/recipe_graph/db.py b/src/recipe_graph/db.py similarity index 100% rename from recipe_graph/db.py rename to src/recipe_graph/db.py diff --git a/recipe_graph/insert_sites.py b/src/recipe_graph/insert_sites.py similarity index 100% rename from recipe_graph/insert_sites.py rename to src/recipe_graph/insert_sites.py diff --git a/recipe_graph/scrape.py b/src/recipe_graph/scrape.py similarity index 100% rename from recipe_graph/scrape.py rename to src/recipe_graph/scrape.py -- 2.40.1 From c6a75b59ebe2c59d117ec7fc3ebd5fce3aff73db Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 13:01:33 -0400 Subject: [PATCH 05/26] added first test --- test/test_db.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 test/test_db.py diff --git a/test/test_db.py b/test/test_db.py new file mode 100644 index 0000000..6d6bfdd --- /dev/null +++ b/test/test_db.py @@ -0,0 +1,20 @@ +from sqlite3 import connect +from recipe_graph import db +from sqlalchemy.exc import SQLAlchemyError +import sqlalchemy + +import pytest + +@pytest.fixture +def engine() -> sqlalchemy.engine.Engine: + return db.get_engine() + +def test_db_connection(engine): + connected = False + try: + engine.connect() + connected = True + except (SQLAlchemyError): + pass + finally: + assert(connected == True) -- 2.40.1 From 082c342256808e6382017a6a24f27c9b4621ee32 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 13:32:01 -0400 Subject: [PATCH 06/26] fixed .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6c17bc8..10bc129 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,5 @@ data/ *.code-workspace sandbox/ .vscode/ -.pytest_cache/ +*.pytest_cache/ *.egg-info/ -- 2.40.1 From 754cb1235cbe9d033e8aaba7d5b3aab8cc168e23 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 14:40:54 -0400 Subject: [PATCH 07/26] testing db connection and table creation --- src/recipe_graph/db.py | 2 -- test/test_db.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/recipe_graph/db.py b/src/recipe_graph/db.py index 7e2265d..87e97a9 100644 --- a/src/recipe_graph/db.py +++ b/src/recipe_graph/db.py @@ -60,10 +60,8 @@ class IngredientConnection(Base): __tablename__ = 'IngredientConnection' ingredient_a = Column(String, - ForeignKey("RecipeIngredientParts.ingredient"), primary_key = True) ingredient_b = Column(String, - ForeignKey("RecipeIngredientParts.ingredient"), primary_key = True) recipe_count = Column(Integer) UniqueConstraint(ingredient_a, ingredient_b) diff --git a/test/test_db.py b/test/test_db.py index 6d6bfdd..bcda081 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,13 +1,29 @@ from sqlite3 import connect +import inspect from recipe_graph import db +from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session import sqlalchemy import pytest + @pytest.fixture def engine() -> sqlalchemy.engine.Engine: - return db.get_engine() + engine = db.get_engine() + yield engine + db.Base.metadata.drop_all(engine) + + +@pytest.fixture +def tables() -> list[db.Base]: + tables = [] + for _, obj in inspect.getmembers(db): + if inspect.isclass(obj) and issubclass(obj, db.Base) and obj != db.Base: + tables.append(obj) + return tables + def test_db_connection(engine): connected = False @@ -17,4 +33,26 @@ def test_db_connection(engine): except (SQLAlchemyError): pass finally: - assert(connected == True) + assert connected == True + + +def test_db_classes(tables): + assert len(tables) > 0 + + +def test_db_class_creation(tables: list[db.Base], engine: sqlalchemy.engine.Engine): + inspector = sqlalchemy.inspect(engine) + schemas = inspector.get_schema_names() + assert len(list(schemas)) == 2 + assert schemas[0] == "information_schema" + assert schemas[1] == "public" + + db_tables = inspector.get_table_names(schema="public") + assert len(db_tables) == 0 + + db.create_tables(engine) + inspector = sqlalchemy.inspect(engine) + db_tables = inspector.get_table_names(schema="public") + assert len(db_tables) == len(tables) + table_names = [table.__tablename__ for table in tables] + assert sorted(db_tables) == sorted(table_names) \ No newline at end of file -- 2.40.1 From f290f492480c0077e0d21de8f82f423783c0dd50 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sun, 18 Sep 2022 20:58:00 -0400 Subject: [PATCH 08/26] updated test fixtures --- test/test_db.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/test/test_db.py b/test/test_db.py index bcda081..3a41a6b 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -12,10 +12,26 @@ import pytest @pytest.fixture def engine() -> sqlalchemy.engine.Engine: engine = db.get_engine() + # make sure db is empty otherwise might be testing a live db + # this makes sure that we don't drop all on a live db + inspector = sqlalchemy.inspect(engine) + schemas = inspector.get_schema_names() + assert len(list(schemas)) == 2 + assert schemas[0] == "information_schema" + assert schemas[1] == "public" + + db_tables = inspector.get_table_names(schema="public") + assert len(db_tables) == 0 + yield engine db.Base.metadata.drop_all(engine) +def init_db(engine) -> sqlalchemy.engine.Engine: + db.create_tables(engine) + return engine + + @pytest.fixture def tables() -> list[db.Base]: tables = [] @@ -41,18 +57,9 @@ def test_db_classes(tables): def test_db_class_creation(tables: list[db.Base], engine: sqlalchemy.engine.Engine): - inspector = sqlalchemy.inspect(engine) - schemas = inspector.get_schema_names() - assert len(list(schemas)) == 2 - assert schemas[0] == "information_schema" - assert schemas[1] == "public" - - db_tables = inspector.get_table_names(schema="public") - assert len(db_tables) == 0 - db.create_tables(engine) inspector = sqlalchemy.inspect(engine) db_tables = inspector.get_table_names(schema="public") assert len(db_tables) == len(tables) table_names = [table.__tablename__ for table in tables] - assert sorted(db_tables) == sorted(table_names) \ No newline at end of file + assert sorted(db_tables) == sorted(table_names) -- 2.40.1 From c32459f0df17a653fd36464b8ca6fd340b9ab671 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 11:27:12 -0400 Subject: [PATCH 09/26] added test coverage report --- .gitignore | 1 + requirements.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 10bc129..7d40403 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ data/ *env *.code-workspace sandbox/ +htmlcov .vscode/ *.pytest_cache/ *.egg-info/ diff --git a/requirements.txt b/requirements.txt index 502ee36..9bce4cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ attrs==22.1.0 beautifulsoup4==4.11.1 +coverage==6.5.0 greenlet==1.1.2 iniconfig==1.1.1 packaging==21.3 @@ -9,6 +10,7 @@ py==1.11.0 PyMySQL==1.0.2 pyparsing==3.0.9 pytest==7.1.3 +pytest-cov==4.0.0 python-dotenv==0.20.0 soupsieve==2.3.2.post1 SQLAlchemy==1.4.39 -- 2.40.1 From fe911340506ab3b5591be0752d59e5b83e48d852 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 11:47:47 -0400 Subject: [PATCH 10/26] added test coverage reports to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 7d40403..ced2d29 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ data/ *.code-workspace sandbox/ htmlcov +.coverage +*.lcov .vscode/ *.pytest_cache/ *.egg-info/ -- 2.40.1 From 4c96bd8a282ce8db2bb4750b15daecbda98d59e6 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 11:59:34 -0400 Subject: [PATCH 11/26] removed unused imports --- test/test_db.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_db.py b/test/test_db.py index 3a41a6b..9bde118 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,9 +1,7 @@ -from sqlite3 import connect import inspect from recipe_graph import db from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.orm import Session import sqlalchemy import pytest -- 2.40.1 From a5153e240623cca276f4dd2aefc7b4290445789e Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 12:32:58 -0400 Subject: [PATCH 12/26] refactor to facilitate testing --- src/recipe_graph/db.py | 15 +++++++--- src/recipe_graph/insert_sites.py | 49 +++++++++++++++++++------------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/recipe_graph/db.py b/src/recipe_graph/db.py index 87e97a9..07a36c2 100644 --- a/src/recipe_graph/db.py +++ b/src/recipe_graph/db.py @@ -9,7 +9,7 @@ from sqlalchemy import create_engine, Column, Integer, String, Boolean, \ from sqlalchemy.types import ARRAY from sqlalchemy.engine import URL from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker Base = declarative_base() @@ -99,6 +99,11 @@ def get_engine(use_dotenv = True, **kargs): database=DB_NAME) return create_engine(eng_url) +def get_session(**kargs) -> Session: + eng = get_engine(**kargs) + return sessionmaker(eng) + + def create_tables(eng): logging.info(f"Createing DB Tables: {eng.url}") @@ -210,8 +215,10 @@ def update_graph_connectivity(session = None): not_(Recipe.id.in_(graphed)))): session.add(RecipeGraphed(recipe_id=recipe.id, status=True)) - - -if __name__ == "__main__": +def main(): eng = get_engine() create_tables(eng) + + +if __name__ == "__main__": + main() diff --git a/src/recipe_graph/insert_sites.py b/src/recipe_graph/insert_sites.py index a3a7107..886196e 100644 --- a/src/recipe_graph/insert_sites.py +++ b/src/recipe_graph/insert_sites.py @@ -1,29 +1,40 @@ +from pydoc import apropos from sqlalchemy.orm import sessionmaker -import db +from recipe_graph import db import json import argparse import logging -parser = argparse.ArgumentParser(description='Import recipes into database') -parser.add_argument('file', type=str, - help='JSON file with recipe site information') -parser.add_argument('-v', '--verbose', action='store_true') +def load_file(f_name: str): + with open(f_name) as f: + sites = json.load(f) + return sites -args = parser.parse_args() -if args.verbose: - logging.basicConfig(level=logging.INFO) - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) +def setup_argparser() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import recipes into database") + parser.add_argument("file", type=str, help="JSON file with recipe site information") + parser.add_argument("-v", "--verbose", action="store_true") -with open(args.file) as f: - sites = json.load(f) + args = parser.parse_args() + return args -eng = db.get_engine() -S = sessionmaker(eng) +def setup_logging(args: argparse.Namespace): + if args.verbose: + logging.basicConfig(level=logging.INFO) + logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -with S.begin() as session: - for site in sites: - logging.info(f"Adding {site}") - session.add(db.RecipeSite(**site)) - - \ No newline at end of file +def main(): + args = setup_argparser() + setup_logging(args) + + S = db.get_session() + sites = load_file(args.file) + + with S.begin() as session: + for site in sites: + logging.info(f"Adding {site}") + session.add(db.RecipeSite(**site)) + +if __name__ == "__main__": + main() \ No newline at end of file -- 2.40.1 From 3a45cfb02a354a7e6539717f038caed7a27e1bdf Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 12:35:37 -0400 Subject: [PATCH 13/26] formatting --- src/recipe_graph/db.py | 263 +++++++++++++++++-------------- src/recipe_graph/insert_sites.py | 8 +- 2 files changed, 155 insertions(+), 116 deletions(-) diff --git a/src/recipe_graph/db.py b/src/recipe_graph/db.py index 07a36c2..73ccd6c 100644 --- a/src/recipe_graph/db.py +++ b/src/recipe_graph/db.py @@ -3,9 +3,20 @@ import logging from types import NoneType from dotenv import load_dotenv from xmlrpc.client import Boolean -from sqlalchemy import create_engine, Column, Integer, String, Boolean, \ - ForeignKey, UniqueConstraint, func, select, and_, or_, \ - not_ +from sqlalchemy import ( + create_engine, + Column, + Integer, + String, + Boolean, + ForeignKey, + UniqueConstraint, + func, + select, + and_, + or_, + not_, +) from sqlalchemy.types import ARRAY from sqlalchemy.engine import URL from sqlalchemy.ext.declarative import declarative_base @@ -14,41 +25,46 @@ from sqlalchemy.orm import Session, sessionmaker Base = declarative_base() + class Ingredient(Base): - __tablename__ = 'Ingredient' - - id = Column(Integer, primary_key = True) - name = Column(String, nullable = False) + __tablename__ = "Ingredient" + + id = Column(Integer, primary_key=True) + name = Column(String, nullable=False) + class RecipeSite(Base): - __tablename__ = 'RecipeSite' - - id = Column(Integer, primary_key = True) - name = Column(String, nullable = False, unique = True) - ingredient_class = Column(String, nullable = False) - name_class = Column(String, nullable = False) - base_url = Column(String, nullable = False, unique = True) + __tablename__ = "RecipeSite" + + id = Column(Integer, primary_key=True) + name = Column(String, nullable=False, unique=True) + ingredient_class = Column(String, nullable=False) + name_class = Column(String, nullable=False) + base_url = Column(String, nullable=False, unique=True) + class Recipe(Base): - __tablename__ = 'Recipe' - - id = Column(Integer, primary_key = True) + __tablename__ = "Recipe" + + id = Column(Integer, primary_key=True) name = Column(String) - identifier = Column(String, nullable = False) - recipe_site_id = Column(Integer, ForeignKey('RecipeSite.id')) + identifier = Column(String, nullable=False) + recipe_site_id = Column(Integer, ForeignKey("RecipeSite.id")) UniqueConstraint(identifier, recipe_site_id) + class RecipeIngredient(Base): - __tablename__ = 'RecipeIngredient' - - id = Column(Integer, primary_key = True) - text = Column(String, nullable = False) - recipe_id = Column(Integer, ForeignKey('Recipe.id')) + __tablename__ = "RecipeIngredient" + + id = Column(Integer, primary_key=True) + text = Column(String, nullable=False) + recipe_id = Column(Integer, ForeignKey("Recipe.id")) ingredient_id = Column(Integer, ForeignKey("Ingredient.id")) + class RecipeIngredientParts(Base): - __tablename__ = 'RecipeIngredientParts' - + __tablename__ = "RecipeIngredientParts" + id = Column(Integer, ForeignKey("RecipeIngredient.id"), primary_key=True) quantity = Column(String) unit = Column(String) @@ -56,169 +72,188 @@ class RecipeIngredientParts(Base): ingredient = Column(String) supplement = Column(String) + class IngredientConnection(Base): - __tablename__ = 'IngredientConnection' - - ingredient_a = Column(String, - primary_key = True) - ingredient_b = Column(String, - primary_key = True) + __tablename__ = "IngredientConnection" + + ingredient_a = Column(String, primary_key=True) + ingredient_b = Column(String, primary_key=True) recipe_count = Column(Integer) UniqueConstraint(ingredient_a, ingredient_b) + class RecipeConnection(Base): - __tablename__ = 'RecipeConnection' - - recipe_a = Column(Integer, - ForeignKey("Recipe.id"), - primary_key = True) - recipe_b = Column(Integer, - ForeignKey("Recipe.id"), - primary_key = True) + __tablename__ = "RecipeConnection" + + recipe_a = Column(Integer, ForeignKey("Recipe.id"), primary_key=True) + recipe_b = Column(Integer, ForeignKey("Recipe.id"), primary_key=True) ingredient_count = Column(Integer) + class RecipeGraphed(Base): __tablename__ = "RecipeGraphed" - - recipe_id = Column(Integer, ForeignKey("Recipe.id"), primary_key = True) - status = Column(Boolean, nullable = False, default = False) - -def get_engine(use_dotenv = True, **kargs): + recipe_id = Column(Integer, ForeignKey("Recipe.id"), primary_key=True) + status = Column(Boolean, nullable=False, default=False) + + +def get_engine(use_dotenv=True, **kargs): if use_dotenv: - load_dotenv() + load_dotenv() DB_URL = os.getenv("POSTGRES_URL") - DB_USER = os.getenv("POSTGRES_USER") + DB_USER = os.getenv("POSTGRES_USER") DB_PASSWORD = os.getenv("POSTGRES_PASSWORD") - DB_NAME = os.getenv("POSTGRES_DB") + DB_NAME = os.getenv("POSTGRES_DB") - eng_url = URL.create('postgresql', - username=DB_USER, - password=DB_PASSWORD, - host=DB_URL, - database=DB_NAME) + eng_url = URL.create( + "postgresql", + username=DB_USER, + password=DB_PASSWORD, + host=DB_URL, + database=DB_NAME, + ) return create_engine(eng_url) + def get_session(**kargs) -> Session: eng = get_engine(**kargs) return sessionmaker(eng) - def create_tables(eng): logging.info(f"Createing DB Tables: {eng.url}") Base.metadata.create_all(eng, checkfirst=True) -def pair_query(pairable, groupable, recipe_ids = None, pair_type = String): - pair_func= func.text_pairs - if pair_type == Integer: - pair_func=func.int_pairs - new_pairs = select(groupable, - pair_func(func.array_agg(pairable.distinct()), - type_=ARRAY(pair_type)).label("pair"))\ - .join(RecipeIngredientParts) +def pair_query(pairable, groupable, recipe_ids=None, pair_type=String): + pair_func = func.text_pairs + if pair_type == Integer: + pair_func = func.int_pairs + + new_pairs = select( + groupable, + pair_func(func.array_agg(pairable.distinct()), type_=ARRAY(pair_type)).label( + "pair" + ), + ).join(RecipeIngredientParts) if not type(recipe_ids) == NoneType: new_pairs = new_pairs.where(RecipeIngredient.recipe_id.in_(recipe_ids)) - new_pairs = new_pairs.group_by(groupable)\ - .cte() + new_pairs = new_pairs.group_by(groupable).cte() return new_pairs -def pair_count_query(pairs, countable, recipe_ids = None): + +def pair_count_query(pairs, countable, recipe_ids=None): new_counts = select(pairs, func.count(func.distinct(countable))) - + if not type(recipe_ids) == NoneType: - new_counts = new_counts.where(or_(pairs[0].in_(recipe_ids), - pairs[1].in_(recipe_ids))) - - + new_counts = new_counts.where( + or_(pairs[0].in_(recipe_ids), pairs[1].in_(recipe_ids)) + ) + new_counts = new_counts.group_by(pairs) - + return new_counts -def update_graph_connectivity(session = None): + +def update_graph_connectivity(session=None): # this is pure SQLAlchemy so it is more portable # This would have been simpler if I utilized Postgres specific feature if not session: session = Session(get_engine()) with session.begin(): - ids = select(Recipe.id)\ - .join(RecipeGraphed, isouter = True)\ - .where(RecipeGraphed.status.is_not(True)) + ids = ( + select(Recipe.id) + .join(RecipeGraphed, isouter=True) + .where(RecipeGraphed.status.is_not(True)) + ) - num_recipes = session.execute(select(func.count('*')).select_from(ids.cte())).fetchone()[0] + num_recipes = session.execute( + select(func.count("*")).select_from(ids.cte()) + ).fetchone()[0] if num_recipes <= 0: logging.info("no new recipies") return logging.info(f"adding {num_recipes} recipes to the graphs") - - new_pairs = pair_query(RecipeIngredientParts.ingredient, - RecipeIngredient.recipe_id, - recipe_ids = ids) - - new_counts = pair_count_query(new_pairs.c.pair, - new_pairs.c.recipe_id) - + new_pairs = pair_query( + RecipeIngredientParts.ingredient, RecipeIngredient.recipe_id, recipe_ids=ids + ) + + new_counts = pair_count_query(new_pairs.c.pair, new_pairs.c.recipe_id) + logging.info("addeing new ingredient connections") for pair, count in session.execute(new_counts): - connection = session.query(IngredientConnection)\ - .where(and_(IngredientConnection.ingredient_a == pair[0], - IngredientConnection.ingredient_b == pair[1]))\ - .first() + connection = ( + session.query(IngredientConnection) + .where( + and_( + IngredientConnection.ingredient_a == pair[0], + IngredientConnection.ingredient_b == pair[1], + ) + ) + .first() + ) if connection: connection.recipe_count += count session.merge(connection) else: - session.add(IngredientConnection(ingredient_a = pair[0], - ingredient_b = pair[1], - recipe_count = count)) - + session.add( + IngredientConnection( + ingredient_a=pair[0], ingredient_b=pair[1], recipe_count=count + ) + ) + # update RecipeConnection logging.info("adding new recipe connections") - all_pairs = pair_query(RecipeIngredient.recipe_id, - RecipeIngredientParts.ingredient, - pair_type=Integer) - - new_counts = pair_count_query(all_pairs.c.pair, - all_pairs.c.ingredient, - recipe_ids=ids) - + all_pairs = pair_query( + RecipeIngredient.recipe_id, + RecipeIngredientParts.ingredient, + pair_type=Integer, + ) + + new_counts = pair_count_query( + all_pairs.c.pair, all_pairs.c.ingredient, recipe_ids=ids + ) + i = 0 for pair, count in session.execute(new_counts): - session.add(RecipeConnection(recipe_a = pair[0], - recipe_b = pair[1], - ingredient_count = count)) + session.add( + RecipeConnection( + recipe_a=pair[0], recipe_b=pair[1], ingredient_count=count + ) + ) # flush often to reduce memory usage i += 1 if (i % 100000) == 0: session.flush() - + # update RecipeGraphed.status logging.info("updating existing RecipeGraphed rows") - for recipeGraphed in session.query(RecipeGraphed)\ - .where(RecipeGraphed.recipe_id.in_(ids)): + for recipeGraphed in session.query(RecipeGraphed).where( + RecipeGraphed.recipe_id.in_(ids) + ): recipeGraphed.status = True session.merge(recipeGraphed) - + graphed = select(RecipeGraphed.recipe_id) - - # add recipies that aren't in the table + + # add recipies that aren't in the table logging.info("adding new RecipeGraphed rows") - for recipe in session.query(Recipe)\ - .where(and_(Recipe.id.in_(ids), - not_(Recipe.id.in_(graphed)))): + for recipe in session.query(Recipe).where( + and_(Recipe.id.in_(ids), not_(Recipe.id.in_(graphed))) + ): session.add(RecipeGraphed(recipe_id=recipe.id, status=True)) - + + def main(): eng = get_engine() create_tables(eng) - + if __name__ == "__main__": main() diff --git a/src/recipe_graph/insert_sites.py b/src/recipe_graph/insert_sites.py index 886196e..50db413 100644 --- a/src/recipe_graph/insert_sites.py +++ b/src/recipe_graph/insert_sites.py @@ -11,6 +11,7 @@ def load_file(f_name: str): sites = json.load(f) return sites + def setup_argparser() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Import recipes into database") parser.add_argument("file", type=str, help="JSON file with recipe site information") @@ -19,11 +20,13 @@ def setup_argparser() -> argparse.Namespace: args = parser.parse_args() return args + def setup_logging(args: argparse.Namespace): if args.verbose: logging.basicConfig(level=logging.INFO) logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) + def main(): args = setup_argparser() setup_logging(args) @@ -35,6 +38,7 @@ def main(): for site in sites: logging.info(f"Adding {site}") session.add(db.RecipeSite(**site)) - + + if __name__ == "__main__": - main() \ No newline at end of file + main() -- 2.40.1 From 6189de8039ef50816bfad3213d44a59e50bbd5cd Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 12:51:08 -0400 Subject: [PATCH 14/26] added test for db.get_session() --- test/test_db.py | 6 ++++++ test/test_insert_sites.py | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 test/test_insert_sites.py diff --git a/test/test_db.py b/test/test_db.py index 9bde118..f40c5ce 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -2,6 +2,7 @@ import inspect from recipe_graph import db from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import sessionmaker import sqlalchemy import pytest @@ -49,6 +50,11 @@ def test_db_connection(engine): finally: assert connected == True +def test_get_session(): + session = db.get_session() + eng = db.get_engine() + session == sessionmaker(eng) + def test_db_classes(tables): assert len(tables) > 0 diff --git a/test/test_insert_sites.py b/test/test_insert_sites.py new file mode 100644 index 0000000..a792f5f --- /dev/null +++ b/test/test_insert_sites.py @@ -0,0 +1,7 @@ +import inspect +from recipe_graph import insert_sites +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +import sqlalchemy + +import pytest -- 2.40.1 From 9b9e629548641601bbbac276ddf70b2df7b348d2 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 13:17:23 -0400 Subject: [PATCH 15/26] added test for loading json file --- test/test_insert_sites.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/test_insert_sites.py b/test/test_insert_sites.py index a792f5f..cba9585 100644 --- a/test/test_insert_sites.py +++ b/test/test_insert_sites.py @@ -1,7 +1,28 @@ import inspect +import json +import os from recipe_graph import insert_sites from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError import sqlalchemy import pytest + + +@pytest.fixture +def json_data() -> list[dict]: + return [{"key": "value"}, {"test": "value1", "test2": "value2"}] + + +@pytest.fixture +def json_file(json_data: list[dict]) -> str: + f_path = "test.json" + with open(f_path, 'w') as f: + json.dump(json_data, f) + yield f_path + if os.path.exists(f_path): + os.remove(f_path) + +def test_load_file(json_file: str, json_data): + test_data = insert_sites.load_file(json_file) + assert test_data == json_data \ No newline at end of file -- 2.40.1 From c34af93533845d0e41a43ead497e3ee946017cdf Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 14:19:15 -0400 Subject: [PATCH 16/26] moved to logger --- src/recipe_graph/insert_sites.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/recipe_graph/insert_sites.py b/src/recipe_graph/insert_sites.py index 50db413..a4a1c2b 100644 --- a/src/recipe_graph/insert_sites.py +++ b/src/recipe_graph/insert_sites.py @@ -4,6 +4,7 @@ from recipe_graph import db import json import argparse import logging +import sys def load_file(f_name: str): @@ -12,31 +13,32 @@ def load_file(f_name: str): return sites -def setup_argparser() -> argparse.Namespace: +def setup_argparser(args) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Import recipes into database") parser.add_argument("file", type=str, help="JSON file with recipe site information") parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - return args + return parser.parse_args(args) -def setup_logging(args: argparse.Namespace): +def setup_logging(args: argparse.Namespace) -> logging.Logger: + logger = logging.Logger("insert_sites", logging.WARNING) if args.verbose: - logging.basicConfig(level=logging.INFO) + logger.setLevel(logging.INFO) logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) + return logger def main(): - args = setup_argparser() - setup_logging(args) + args = setup_argparser(sys.argv[1:]) + logger = setup_logging(args) S = db.get_session() sites = load_file(args.file) with S.begin() as session: for site in sites: - logging.info(f"Adding {site}") + logger.info(f"Adding {site}") session.add(db.RecipeSite(**site)) -- 2.40.1 From 88b9707201dcb88c25a796990e4e74da3f9d1121 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 14:27:40 -0400 Subject: [PATCH 17/26] added test for logging setup --- test/test_insert_sites.py | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/test/test_insert_sites.py b/test/test_insert_sites.py index cba9585..2b01be0 100644 --- a/test/test_insert_sites.py +++ b/test/test_insert_sites.py @@ -1,10 +1,8 @@ -import inspect import json import os from recipe_graph import insert_sites from sqlalchemy import select -from sqlalchemy.exc import SQLAlchemyError -import sqlalchemy +import logging import pytest @@ -25,4 +23,35 @@ def json_file(json_data: list[dict]) -> str: def test_load_file(json_file: str, json_data): test_data = insert_sites.load_file(json_file) - assert test_data == json_data \ No newline at end of file + assert test_data == json_data + + +def test_setup_argparser(): + file_name = "test" + args = insert_sites.setup_argparser([file_name]) + assert len(vars(args)) == 2 + assert args.file == file_name + assert args.verbose == False + + + args = insert_sites.setup_argparser([file_name, "-v"]) + assert args.file == file_name + assert args.verbose == True + + args = insert_sites.setup_argparser([file_name, "--verbose"]) + assert args.file == file_name + assert args.verbose == True + +def test_setup_logging(): + args = insert_sites.setup_argparser(["test"]) + logger = insert_sites.setup_logging(args) + assert logger.level == logging.WARNING + + args = insert_sites.setup_argparser(["test", "-v"]) + logger = insert_sites.setup_logging(args) + assert logger.level == logging.INFO + + args = insert_sites.setup_argparser(["test", "--verbose"]) + logger = insert_sites.setup_logging(args) + assert logger.level == logging.INFO + \ No newline at end of file -- 2.40.1 From 294231dd48e1df1f8d5956ddb5361fd95b16cf5d Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 14:33:45 -0400 Subject: [PATCH 18/26] added test exclusion to main functions --- src/recipe_graph/db.py | 4 ++-- src/recipe_graph/insert_sites.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/recipe_graph/db.py b/src/recipe_graph/db.py index 73ccd6c..8dc7ad6 100644 --- a/src/recipe_graph/db.py +++ b/src/recipe_graph/db.py @@ -250,10 +250,10 @@ def update_graph_connectivity(session=None): session.add(RecipeGraphed(recipe_id=recipe.id, status=True)) -def main(): +def main(): # pragma: no cover eng = get_engine() create_tables(eng) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover main() diff --git a/src/recipe_graph/insert_sites.py b/src/recipe_graph/insert_sites.py index a4a1c2b..812bd76 100644 --- a/src/recipe_graph/insert_sites.py +++ b/src/recipe_graph/insert_sites.py @@ -29,7 +29,7 @@ def setup_logging(args: argparse.Namespace) -> logging.Logger: return logger -def main(): +def main(): # pragma: no cover args = setup_argparser(sys.argv[1:]) logger = setup_logging(args) @@ -42,5 +42,5 @@ def main(): session.add(db.RecipeSite(**site)) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover main() -- 2.40.1 From 9a15f6c03103cffbb89cc55a0cdf1202666c6f8d Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 14:40:16 -0400 Subject: [PATCH 19/26] refactoring --- src/recipe_graph/scrape.py | 112 ++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/src/recipe_graph/scrape.py b/src/recipe_graph/scrape.py index 06ca61f..bdc4519 100644 --- a/src/recipe_graph/scrape.py +++ b/src/recipe_graph/scrape.py @@ -1,9 +1,7 @@ -from ast import alias -from dis import Instruction -import db +import sys +from recipe_graph import db import re from sqlalchemy import select, desc, exists, not_, except_ -from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import sessionmaker import bs4 from urllib.request import urlopen @@ -129,62 +127,64 @@ def parse_recipe(session, recipe, site): return recipe +def main(): + parser = ArgumentParser(description="Scrape a recipe site for recipies") + parser.add_argument('site', + help='Name of site') + parser.add_argument('-id', '--identifier', dest='id', + help='url of recipe(reletive to base url of site) or commma seperated list') + parser.add_argument('-a', '--auto', action='store', dest='n', + help='automaticaly generate identifier(must supply number of recipies to scrape)') + parser.add_argument('-v', '--verbose', action='store_true') -parser = ArgumentParser(description="Scrape a recipe site for recipies") -parser.add_argument('site', - help='Name of site') -parser.add_argument('-id', '--identifier', dest='id', - help='url of recipe(reletive to base url of site) or commma seperated list') -parser.add_argument('-a', '--auto', action='store', dest='n', - help='automaticaly generate identifier(must supply number of recipies to scrape)') -parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args(sys.argv) + if args.verbose: + logging.basicConfig(level=logging.INFO) + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) -args = parser.parse_args() -if args.verbose: - logging.basicConfig(level=logging.INFO) - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + eng = db.get_engine() + S = sessionmaker(eng) -eng = db.get_engine() -S = sessionmaker(eng) - -with S.begin() as sess: - site = sess.query(db.RecipeSite).where(db.RecipeSite.name == args.site).one() - site_id = site.id + with S.begin() as sess: + site = sess.query(db.RecipeSite).where(db.RecipeSite.name == args.site).one() + site_id = site.id + + recipe_ids = [] + starting_id = 0 + if args.id and not args.n: + recipe_ids.append(args.id) + logging.info(f'Retreiving single recipe: {args.id}') + elif args.n: + if not args.id: + last_recipe = sess.query(db.Recipe).\ + where(db.Recipe.recipe_site_id == site.id).\ + order_by(desc(db.Recipe.identifier)).\ + limit(1).\ + scalar() + starting_id = int(last_recipe.identifier) + 1 + else: + starting_id = int(args.id) + recipe_ids = range(starting_id, starting_id+int(args.n)) + logging.info(f'Retreving {args.n} recipes from {site.base_url} starting at {starting_id}') - recipe_ids = [] - starting_id = 0 - if args.id and not args.n: - recipe_ids.append(args.id) - logging.info(f'Retreiving single recipe: {args.id}') - elif args.n: - if not args.id: - last_recipe = sess.query(db.Recipe).\ - where(db.Recipe.recipe_site_id == site.id).\ - order_by(desc(db.Recipe.identifier)).\ - limit(1).\ - scalar() - starting_id = int(last_recipe.identifier) + 1 - else: - starting_id = int(args.id) - recipe_ids = range(starting_id, starting_id+int(args.n)) - logging.info(f'Retreving {args.n} recipes from {site.base_url} starting at {starting_id}') - - - - for recipe_id in recipe_ids: - try: - savepoint = sess.begin_nested() + + + for recipe_id in recipe_ids: + try: + savepoint = sess.begin_nested() - recipe = db.Recipe(identifier = recipe_id, recipe_site_id = site.id) - parse_recipe(sess, recipe, site) + recipe = db.Recipe(identifier = recipe_id, recipe_site_id = site.id) + parse_recipe(sess, recipe, site) - savepoint.commit() - except KeyboardInterrupt as e: - savepoint.rollback() - break - except Exception as e: - savepoint.rollback() - logging.error(e) - continue + savepoint.commit() + except KeyboardInterrupt as e: + savepoint.rollback() + break + except Exception as e: + savepoint.rollback() + logging.error(e) + continue - \ No newline at end of file + +if __name__ == "__main__": # pragma: no cover + main() \ No newline at end of file -- 2.40.1 From 98f96543e6c8673a785fe6018f8377f27e0a5832 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 14:40:42 -0400 Subject: [PATCH 20/26] added file for testing scrape script --- test/test_scrape.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 test/test_scrape.py diff --git a/test/test_scrape.py b/test/test_scrape.py new file mode 100644 index 0000000..e7ac014 --- /dev/null +++ b/test/test_scrape.py @@ -0,0 +1,3 @@ +from recipe_graph import scrape + +import pytest \ No newline at end of file -- 2.40.1 From a568fb244e1ef83400c569313ea3cd071f39c99e Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Wed, 19 Apr 2023 17:43:35 -0400 Subject: [PATCH 21/26] added dist folder to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ced2d29..5458304 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ htmlcov .vscode/ *.pytest_cache/ *.egg-info/ +dist/ -- 2.40.1 From b9e754c9841cdd912450fdc0574a374a22cac912 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Wed, 19 Apr 2023 17:44:57 -0400 Subject: [PATCH 22/26] added example of sites.json to readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 006be47..620cf65 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,18 @@ Start database docker-compose up ``` +Example `sites.json` +```json +[ + { + "name": "Example Site Name", + "ingredient_class": "example-ingredients-item-name", + "name_class" : "example-heading-content", + "base_url" : "https://www.example.com/recipe/" + } +] +``` + Initialize database and recipe sites ```sh python src/db.py -- 2.40.1 From 51d631daf6ff824ca29886f546dce6e7af26b3f5 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Wed, 19 Apr 2023 18:42:56 -0400 Subject: [PATCH 23/26] added blurb about testing --- README.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 620cf65..5f7cd02 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ POSTGRES_DB=rgraph Start database ```sh -docker-compose up +docker-compose -p recipe-dev up ``` Example `sites.json` @@ -41,6 +41,11 @@ python src/db.py python src/insert_sites.py data/sites.json ``` +Shutdown database +```sh +docker-compose -p recipe-dev down +``` + ## Usage ### Scrape import new recipes @@ -72,6 +77,31 @@ options: -v, --verbose ``` +## Testing +For testing create a new set up docker containers. Tests will fail if +the database is already initiated. + +Starting testing db +```sh +docker-compose -p recipe-test up +``` + +running tests +```sh +pytest +``` + +**WARNINING**: If you get `ERROR at setup of test_db_connection` and +`ERROR at setup of test_db_class_creation`, please check if testing database is +already initiated. Testing is destructive and should be done on a fresh database. + + +Shutting down testing db +```sh +docker-compose -p recipe-test down +``` + + ## TODO > ☑ automate scraping\ > ☑ extracting quantity and name (via regex)\ -- 2.40.1 From a9970552d608e4cf37bc50b6ce58c66c852ff1fb Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Sat, 15 Oct 2022 15:07:08 -0400 Subject: [PATCH 24/26] added testing to readme --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 5f7cd02..28a7c8e 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,19 @@ docker-compose -p recipe-test down ``` +Test are written in pytest framework. Currently focused on unittest. +Integration tests to come. + +To run test use: +``` +pytest --cov=src/recipe_graph --cov-report lcov --cov-report html +``` + +The html report is under `htmlcov/` and can be viewed through any browser. +The `lcov` file can be used for the [Coverage Gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) +plugin for VS Code to view coverage in your editor. + + ## TODO > ☑ automate scraping\ > ☑ extracting quantity and name (via regex)\ -- 2.40.1 From 01b8a2be621fe1fa82a8a699ba1db82289dce653 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Mon, 15 May 2023 11:25:26 -0400 Subject: [PATCH 25/26] drone-ci testing --- .drone.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..6969f03 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,21 @@ +--- +kind: pipeline +name: test + +steps: +- name: db-up + image: compose:1.23.2 + volumes: + - name: docker_sock + path: /var/run/docker.sock + commands: + - up -p rgraph-test -f docker-compose.yaml + environment: + - POSTGRES_USER=${TESTING_USER} + - POSTGRES_PASSWORD=${TESTING_PASSWORD} + - POSTGRES_DB=${TESTING_DB} + secrets: [TESTING_USER, TESTING_PASSWORD, TESTING_DB] +volumes: + - name: docker_sock + host: + path: /var/run/docker.sock \ No newline at end of file -- 2.40.1 From 3359ce4cf6b9c67fccbf02d5aac443dddf034d10 Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Mon, 15 May 2023 23:24:51 -0400 Subject: [PATCH 26/26] drone-ci testing --- .drone.yml | 70 ++++++++++++++++++++++++++++++++++++++++++++---------- .gitignore | 1 + 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/.drone.yml b/.drone.yml index 6969f03..caf5ab6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,21 +1,65 @@ --- kind: pipeline name: test +environment: + project_name: rgraph +trigger: + event: + include: + - pull_request steps: -- name: db-up - image: compose:1.23.2 - volumes: - - name: docker_sock - path: /var/run/docker.sock - commands: - - up -p rgraph-test -f docker-compose.yaml - environment: - - POSTGRES_USER=${TESTING_USER} - - POSTGRES_PASSWORD=${TESTING_PASSWORD} - - POSTGRES_DB=${TESTING_DB} - secrets: [TESTING_USER, TESTING_PASSWORD, TESTING_DB] + - name: db-up + image: docker/compose:alpine-1.29.2 + environment: + POSTGRES_USER: + from_secret: TESTING_USER + POSTGRES_PASSWORD: + from_secret: TESTING_PASSWORD + POSTGRES_DB: + from_secret: TESTING_DB + volumes: + - name: docker_sock + path: /var/run/docker.sock + commands: + - docker-compose -p rgraph-test up -d + + - name: requirements + image: python:3.10-alpine + commands: + - python -m venv .venv + - . .venv/bin/activate + - pip install -r requirements.txt + + - name: build + image: python:3.10-alpine + commands: + - . .venv/bin/activate + - pip install . + + - name: test + image: python:3.10-alpine + environment: + POSTGRES_USER: + from_secret: TESTING_USER + POSTGRES_PASSWORD: + from_secret: TESTING_PASSWORD + POSTGRES_DB: + from_secret: TESTING_DB + commands: + - hostip=$(ip route show | awk '/default/ {print $3}') + - export POSTGRES_URL=$hostip + - . .venv/bin/activate + - pytest + - name: db-cleanup + image: docker/compose:alpine-1.29.2 + volumes: + - name: docker_sock + path: /var/run/docker.sock + commands: + - docker-compose -p rgraph-test down + - docker volume rm rgraph-test_dbdata volumes: - name: docker_sock host: - path: /var/run/docker.sock \ No newline at end of file + path: /var/run/docker.sock diff --git a/.gitignore b/.gitignore index 5458304..44010e0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ htmlcov *.pytest_cache/ *.egg-info/ dist/ +build/ -- 2.40.1