andrei-stoica
/
recipe-graph
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactoring

This commit is contained in:
Andrei Stoica 2022-10-15 14:40:16 -04:00
parent 294231dd48
commit 9a15f6c031
1 changed files with 56 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/recipe_graph/scrape.py
View File

@ -1,9 +1,7 @@
from ast import alias import sys
from dis import Instruction from recipe_graph import db
import db
import re import re
from sqlalchemy import select, desc, exists, not_, except_ from sqlalchemy import select, desc, exists, not_, except_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
import bs4 import bs4
from urllib.request import urlopen from urllib.request import urlopen
@ -129,7 +127,7 @@ def parse_recipe(session, recipe, site):
return recipe return recipe
def main():
parser = ArgumentParser(description="Scrape a recipe site for recipies") parser = ArgumentParser(description="Scrape a recipe site for recipies")
parser.add_argument('site', parser.add_argument('site',
help='Name of site') help='Name of site')
@ -139,7 +137,7 @@ parser.add_argument('-a', '--auto', action='store', dest='n',
help='automaticaly generate identifier(must supply number of recipies to scrape)') help='automaticaly generate identifier(must supply number of recipies to scrape)')
parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('-v', '--verbose', action='store_true')
args = parser.parse_args() args = parser.parse_args(sys.argv)
if args.verbose: if args.verbose:
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
@ -188,3 +186,5 @@ with S.begin() as sess:
continue continue
if __name__ == "__main__": # pragma: no cover
main()