From 4ddb2ea8fff8f1b346e85ec31581d450dc3b7cda Mon Sep 17 00:00:00 2001 From: Andrei Stoica Date: Fri, 16 Feb 2024 00:30:19 -0500 Subject: [PATCH] api recipe and ingredient, GET and PUT --- .gitignore | 162 ++++++++++++++++++++++++++ api.py | 48 ++++++++ data.py | 85 ++++++++++++++ db.py | 176 +++++++++++++++++++++++++++++ docker-compose.yml | 11 ++ requirements.txt | 6 + sqitch.conf | 8 ++ sqitch/deploy/ingredient-table.sql | 21 ++++ sqitch/deploy/recipe-schema.sql | 7 ++ sqitch/deploy/recipe-table.sql | 18 +++ sqitch/revert/ingredient-table.sql | 7 ++ sqitch/revert/recipe-schema.sql | 7 ++ sqitch/revert/recipe-table.sql | 7 ++ sqitch/sqitch.plan | 6 + sqitch/verify/ingredient-table.sql | 11 ++ sqitch/verify/recipe-schema.sql | 7 ++ sqitch/verify/recipe-table.sql | 11 ++ test/put_recipe.json | 12 ++ 18 files changed, 610 insertions(+) create mode 100644 .gitignore create mode 100644 api.py create mode 100644 data.py create mode 100644 db.py create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 sqitch.conf create mode 100644 sqitch/deploy/ingredient-table.sql create mode 100644 sqitch/deploy/recipe-schema.sql create mode 100644 sqitch/deploy/recipe-table.sql create mode 100644 sqitch/revert/ingredient-table.sql create mode 100644 sqitch/revert/recipe-schema.sql create mode 100644 sqitch/revert/recipe-table.sql create mode 100644 sqitch/sqitch.plan create mode 100644 sqitch/verify/ingredient-table.sql create mode 100644 sqitch/verify/recipe-schema.sql create mode 100644 sqitch/verify/recipe-table.sql create mode 100644 test/put_recipe.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50d319c --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +data/ diff --git a/api.py b/api.py new file mode 100644 index 0000000..ce55d9f --- /dev/null +++ b/api.py @@ -0,0 +1,48 @@ +from fastapi import FastAPI, HTTPException +from data import Recipe, Ingredient +import db +from psycopg2.errors import UniqueViolation + +app = FastAPI() +conn = db.connect() + + +@app.put("/recipe") +def new_recipe(recipe: Recipe) -> Recipe | None: + try: + recipe = db.insert_recipe(recipe, conn) + except UniqueViolation: + raise HTTPException(status_code=400, detail="id conflict") + return recipe + + +@app.get("/recipe/{item_id}") +def get_recipe(item_id: int): + recipe = db.get_recipe(item_id, conn) + if not recipe: + raise HTTPException( + status_code=404, + detail=f"Recipe not found: id = {id}", + ) + return recipe + + +@app.put("/ingredient") +def new_ingredient(ingredient: Ingredient) -> Ingredient | None: + try: + ingredient = db.insert_ingredient(ingredient, conn) + except UniqueViolation: + raise HTTPException(status_code=400, detail="id conflict") + return ingredient + + +@app.get("/ingredient/{item_id}") +def get_ingredient(id): + ingredient = db.get_ingredient(id, conn) + print(ingredient) + if not ingredient: + raise HTTPException( + status_code=404, + detail=f"Ingredient not found: id = {id}", + ) + return ingredient diff --git a/data.py b/data.py new file mode 100644 index 0000000..e40720a --- /dev/null +++ b/data.py @@ -0,0 +1,85 @@ +from pydantic import BaseModel +from recipe_scrapers._abstract import AbstractScraper +from ingredient_parser.postprocess import ParsedIngredient + + +class Recipe(BaseModel): + id: int | None = None + title: str | None + url: str | None + site_name: str + cuisine: str + category: str + description: str + cook_time: float + prep_time: float + yields: str + + +class Ingredient(BaseModel): + id: int | None = None + recipe_id: int | None = None + text: str | None = None + purpose: str | None = None + name: str | None + name_conf: float | None + amount_quantity: list[str] | None + amount_unit: list[str] | None + amount_conf: list[float] | None + preparation: str | None + preparation_conf: float | None + comment: str | None + comment_conf: float | None + + +def parsed_ingredient_to_ingredient( + parsed_ingredient: ParsedIngredient, + purpose=None, + recipe_id=None, +) -> Ingredient: + data = {} + data["purpose"] = purpose + data["recipe_id"] = recipe_id + fields = ["name", "preparation", "comment"] + + data["text"] = parsed_ingredient.sentence + + for field in fields: + attr = getattr(parsed_ingredient, field) + data[field] = attr.text if attr else None + data[field + "_conf"] = attr.confidence if attr else None + + amounts = parsed_ingredient.amount or [] + if amounts: + data["amount_quantity"] = [] + data["amount_unit"] = [] + data["amount_conf"] = [] + for amount in amounts: + data["amount_quantity"].append(amount.quantity if amount else "") + data["amount_unit"].append(amount.unit if amount else "") + data["amount_conf"].append(amount.confidence if amount else 0) + else: + data["amount_quantity"] = None + data["amount_unit"] = None + data["amount_conf"] = None + + return Ingredient(**data) + + +def scraped_reciepe_to_recipe(recipe: AbstractScraper) -> Recipe: + site_name = recipe.site_name() + if not site_name: + site_name = "" + site_name = str(site_name) + + return Recipe( + title=recipe.title(), + site_name=site_name, + url=recipe.url, + cuisine=recipe.cuisine(), + category=recipe.category(), + description=recipe.description(), + cook_time=recipe.cook_time(), + prep_time=recipe.prep_time(), + yields=recipe.yields(), + ) diff --git a/db.py b/db.py new file mode 100644 index 0000000..6fa0a9e --- /dev/null +++ b/db.py @@ -0,0 +1,176 @@ +from pydantic import BaseModel +import psycopg2 as pg +from psycopg2 import sql +from psycopg2.extras import RealDictCursor +from os import environ as env, stat +from functools import wraps +from inspect import getfullargspec +from typing import Callable, Type +from data import Recipe, Ingredient + + +class NoConnectionException(Exception): + pass + + +def connect(): + return pg.connect( + dbname=env.get("POSTGRES_DB"), + user=env.get("POSTGRES_USER"), + password=env.get("POSTGRES_PASSWORD"), + host=env.get("POSTGRES_HOST"), + port=env.get("POSTGRES_PORT"), + ) + + +def connect_by_default( + func: Callable, + kword: str = "conn", + conn_func: Callable[..., pg.extensions.connection] = connect, +) -> Callable: + @wraps(func) + def wrapped(*args, **kargs): + for i, arg in enumerate(args): + kargs[getfullargspec(func).args[i]] = arg + if not kargs.get(kword): + kargs[kword] = conn_func() + return func(**kargs) + + return wrapped + + +@connect_by_default +def select_by_id( + schema_name: str, + table_name: str, + id: int, + columns: list[str] | None = None, + conn: pg.extensions.connection | None = None, +) -> dict | None: + if not conn: + raise NoConnectionException() + + query = sql.SQL( + f""" + SELECT {'{}' if columns else '*'} + FROM {{}}.{{}} + WHERE id = %s;""" + ) + if columns: + cols = sql.SQL(",").join([sql.Identifier(col) for col in columns]) + query = query.format( + cols, sql.Identifier(schema_name), sql.Identifier(table_name) + ) + else: + query = query.format( + sql.Identifier(schema_name), sql.Identifier(table_name) + ) + + with conn.cursor(cursor_factory=RealDictCursor) as curs: + curs.execute(query, (id,)) + data = curs.fetchone() + + if not data: + return + + return data + + +@connect_by_default +def insert_model( + schema_name: str, + table_name: str, + model: BaseModel, + conn: pg.extensions.connection | None = None, +) -> dict | None: + if not conn: + raise NoConnectionException() + + data = model.model_dump(exclude_none=True) + cols = data.keys() + vals = data.values() + + query = sql.SQL( + f""" + INSERT INTO {{schema}}.{{table}} ( + {{fields}} + ) VALUES ( + {", ".join(["%s" for _ in vals])} + ) + RETURNING id; + """ + ) + + with conn.cursor() as curs: + try: + curs.execute( + query.format( + schema=sql.Identifier(schema_name), + table=sql.Identifier(table_name), + fields=sql.SQL(",").join( + [sql.Identifier(col) for col in cols] + ), + ), + (*vals,), + ) + row = curs.fetchone() + conn.commit() + print(row) + except pg.errors.UniqueViolation as error: + conn.rollback() + raise error + + if not row: + return + id = row[0] + + data["id"] = id + return data + + +@connect_by_default +def insert_recipe( + recipe: Recipe, + conn: pg.extensions.connection | None = None, +) -> Recipe | None: + data = insert_model("recipe", "recipe", recipe, conn) + if not data: + return + return Recipe(**data) + + +@connect_by_default +def get_recipe( + id: int, + conn: pg.extensions.connection | None = None, +) -> Recipe | None: + data = select_by_id("recipe", "recipe", id, conn=conn) + if not data: + return + + recipe = Recipe(**data) + return recipe + + +@connect_by_default +def insert_ingredient( + ingredient: Ingredient, + conn: pg.extensions.connection | None = None, +) -> Ingredient | None: + data = insert_model("recipe", "ingredient", ingredient, conn) + if not data: + return + return Ingredient(**data) + + +@connect_by_default +def get_ingredient( + id: int, + conn: pg.extensions.connection | None = None, +) -> Ingredient | None: + data = select_by_id("recipe", "ingredient", id, conn=conn) + if not data: + return + + ingredient = Ingredient(**data) + return ingredient diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..195637b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + psql: + image: postgres + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - "${POSTGRES_PORT}:5432" + volumes: + - "./data/${COMPOSE_PROJECT_NAME}:/var/lib/postgresql/data" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6644c55 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +pandas +psycopg2-binary +fastapi +recipe-scrapers +ingredient-parser-nlp +pydantic diff --git a/sqitch.conf b/sqitch.conf new file mode 100644 index 0000000..0e20ef7 --- /dev/null +++ b/sqitch.conf @@ -0,0 +1,8 @@ +[core] + engine = pg + top_dir = sqitch + # plan_file = sqitch/sqitch.plan +# [engine "pg"] + # target = db:pg: + # registry = sqitch + # client = psql diff --git a/sqitch/deploy/ingredient-table.sql b/sqitch/deploy/ingredient-table.sql new file mode 100644 index 0000000..06d94e0 --- /dev/null +++ b/sqitch/deploy/ingredient-table.sql @@ -0,0 +1,21 @@ +-- Deploy rsugest:ingredient-table to pg +-- requires: recipe-schema + +BEGIN; + +CREATE TABLE recipe.ingredient ( + id SERIAL PRIMARY KEY, + recipe_id INT, + text TEXT, + purpose TEXT, + name TEXT, + name_conf FLOAT, + amount_quantity TEXT[], + amount_unit TEXT[], + amount_conf FLOAT[], + preparation TEXT, + preparation_conf FLOAT, + comment TEXT, + comment_conf FLOAT); + +COMMIT; diff --git a/sqitch/deploy/recipe-schema.sql b/sqitch/deploy/recipe-schema.sql new file mode 100644 index 0000000..ad62908 --- /dev/null +++ b/sqitch/deploy/recipe-schema.sql @@ -0,0 +1,7 @@ +-- Deploy rsugest:recipe-schema to pg + +BEGIN; + +CREATE SCHEMA recipe; + +COMMIT; diff --git a/sqitch/deploy/recipe-table.sql b/sqitch/deploy/recipe-table.sql new file mode 100644 index 0000000..ed15f97 --- /dev/null +++ b/sqitch/deploy/recipe-table.sql @@ -0,0 +1,18 @@ +-- Deploy rsugest:recipe-table to pg +-- requires: recipe-schema + +BEGIN; + +CREATE TABLE recipe.recipe ( + id SERIAL PRIMARY KEY, + title TEXT, + url TEXT, + site_name TEXT, + cuisine TEXT, + category TEXT, + description TEXT, + cook_time REAL, + prep_time REAL, + yields TEXT); + +COMMIT; diff --git a/sqitch/revert/ingredient-table.sql b/sqitch/revert/ingredient-table.sql new file mode 100644 index 0000000..4b36d38 --- /dev/null +++ b/sqitch/revert/ingredient-table.sql @@ -0,0 +1,7 @@ +-- Revert rsugest:ingredient-table from pg + +BEGIN; + +DROP TABLE recipe.ingredient; + +COMMIT; diff --git a/sqitch/revert/recipe-schema.sql b/sqitch/revert/recipe-schema.sql new file mode 100644 index 0000000..f30e603 --- /dev/null +++ b/sqitch/revert/recipe-schema.sql @@ -0,0 +1,7 @@ +-- Revert rsugest:recipe-schema from pg + +BEGIN; + +DROP SCHEMA recipe; + +COMMIT; diff --git a/sqitch/revert/recipe-table.sql b/sqitch/revert/recipe-table.sql new file mode 100644 index 0000000..663993b --- /dev/null +++ b/sqitch/revert/recipe-table.sql @@ -0,0 +1,7 @@ +-- Revert rsugest:recipe-table from pg + +BEGIN; + +DROP TABLE recipe.recipe; + +COMMIT; diff --git a/sqitch/sqitch.plan b/sqitch/sqitch.plan new file mode 100644 index 0000000..0281a88 --- /dev/null +++ b/sqitch/sqitch.plan @@ -0,0 +1,6 @@ +%syntax-version=1.0.0 +%project=rsugest + +recipe-schema 2024-02-07T01:24:03Z Andrei,,, # adding schema for recipes +recipe-table [recipe-schema] 2024-02-07T01:42:28Z Andrei,,, # adding table for recipes +ingredient-table [recipe-schema] 2024-02-07T03:50:20Z Andrei,,, # adding table for ingredients diff --git a/sqitch/verify/ingredient-table.sql b/sqitch/verify/ingredient-table.sql new file mode 100644 index 0000000..39d9d84 --- /dev/null +++ b/sqitch/verify/ingredient-table.sql @@ -0,0 +1,11 @@ +-- Verify rsugest:ingredient-table on pg + +BEGIN; + +SELECT id, recipe_id, text, purpose, name, name_conf, + amount_quantity, amount_unit, amount_conf, preparation, + preparation_conf, comment, comment_conf +FROM recipe.ingredient +WHERE FALSE; + +ROLLBACK; diff --git a/sqitch/verify/recipe-schema.sql b/sqitch/verify/recipe-schema.sql new file mode 100644 index 0000000..5d3e9f2 --- /dev/null +++ b/sqitch/verify/recipe-schema.sql @@ -0,0 +1,7 @@ +-- Verify rsugest:recipe-schema on pg + +BEGIN; + +SELECT pg_catalog.has_schema_privilege('recipe', 'usage'); + +ROLLBACK; diff --git a/sqitch/verify/recipe-table.sql b/sqitch/verify/recipe-table.sql new file mode 100644 index 0000000..9d9e9ce --- /dev/null +++ b/sqitch/verify/recipe-table.sql @@ -0,0 +1,11 @@ +-- Verify rsugest:recipe-table on pg + +BEGIN; + + +select id, title, url, site_name, cuisine, category, description, + cook_time, prep_time, yields +FROM recipe.recipe +WHERE FALSE; + +ROLLBACK; diff --git a/test/put_recipe.json b/test/put_recipe.json new file mode 100644 index 0000000..2389bff --- /dev/null +++ b/test/put_recipe.json @@ -0,0 +1,12 @@ +{ + "id": 0, + "title": "string", + "url": "string", + "site_name": "string", + "cuisine": "string", + "category": "string", + "description": "string", + "cook_time": 0, + "prep_time": 0, + "yields": "string" +}