From 4cdee0976a826551be8d4e89dafc679db5a78eef Mon Sep 17 00:00:00 2001 From: Paul Haedrich <38889179+berrysauce@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:00:27 +0200 Subject: [PATCH] Removed cached queries and stats (for now) --- ingredients.py | 28 ++++---------- main.py | 77 +++++++++++++------------------------- requirements.txt | 4 +- ui/src/routes/+page.svelte | 3 ++ 4 files changed, 38 insertions(+), 74 deletions(-) diff --git a/ingredients.py b/ingredients.py index 75d38dd..1035591 100644 --- a/ingredients.py +++ b/ingredients.py @@ -2,12 +2,6 @@ import bs4 import json import httpx -from deta import Deta - - -deta = Deta() -db = deta.Base("ingredients-stats") - # ---------------------------------------- # FUNCTIONS @@ -21,8 +15,11 @@ def add_ingredient(category: str, ingredient: str): matching_ingredients.append(f"{category}/{ingredient}") categories = os.listdir("ingredients") + if "categories.json" in categories: categories.remove("categories.json") + if ".DS_Store" in categories: + categories.remove(".DS_Store") # macOS only, improves compatibility try: r = httpx.get(url, follow_redirects=True) @@ -44,6 +41,9 @@ def add_ingredient(category: str, ingredient: str): for category in categories: ingredients = os.listdir(f"ingredients/{category}") + if ".DS_Store" in ingredients: + ingredients.remove(".DS_Store") # macOS only, improves compatibility + for ingredient in ingredients: with open(f"ingredients/{category}/{ingredient}", "r") as f: ingredient_data = json.loads(f.read()) @@ -51,21 +51,7 @@ def add_ingredient(category: str, ingredient: str): # ----- STATS ----- # increment total scans for each ingredient - db_ingredient = db.get(ingredient.replace(".json", "")) - if db_ingredient == None: - try: - db.insert(key=ingredient.replace(".json", ""), data={ - "total_scans": 1, - "matching_scans": 0 - }) - except Exception: - # Deta hasn't defined a specific exception for this error - # just ignore it - pass - else: - db.update(key=ingredient.replace(".json", ""), updates={ - "total_scans": db_ingredient["total_scans"] + 1 - }) + # # ----------------- diff --git a/main.py b/main.py index abb89a6..5f4830c 100644 --- a/main.py +++ b/main.py @@ -1,35 +1,41 @@ -import os +#import os import json import httpx import uvicorn -from deta import Deta from typing import Optional from fastapi import FastAPI, Response, HTTPException from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse from fastapi.middleware.cors import CORSMiddleware from urllib.parse import urlparse +#import redis +#from dotenv import load_dotenv # local imports import ingredients +# env variables +""" +load_dotenv() +REDIS_HOST = os.getenv("REDIS_HOST") +REDIS_PORT = os.getenv("REDIS_PORT") +REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") -deta = Deta() -#deta = Deta(project_key=os.environ.get("DETA_PROJECT_KEY"), project_id=os.environ.get("DETA_PROJECT_ID")) - -db = deta.Base("ingredients-stats") -cache_db = deta.Base("ingredients-cache") +r = redis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + password=REDIS_PASSWORD, + ssl=True +) +""" app = FastAPI( title="Ingredients – API", - license_info="https://github.com/berrysauce/ingredients/blob/main/LICENSE.md", - docs_url=None, + #license_info="https://github.com/berrysauce/ingredients/blob/main/LICENSE.md", + #docs_url=None, redoc_url=None ) origins = [ - "https://ingredients.tech", - "https://dev.ingredients.tech", - "https://cdn.ingredients.tech", "https://ingredients.work", "https://dev.ingredients.work", "https://cdn.ingredients.work" @@ -50,9 +56,11 @@ def get_root(): return HTMLResponse(content=f.read()) +""" @app.get("/docs") def get_docs(): return RedirectResponse(url="https://github.com/berrysauce/ingredients/blob/main/README.md#-using-the-api", status_code=301) +""" @app.get("/ingredients", response_class=JSONResponse) @@ -67,21 +75,6 @@ def get_scan(url: str, includeCategories: Optional[bool] = False): r = urlparse(url) parsed_url = r.scheme + "://" + r.netloc.split(":")[0] + r.path - # ----- CACHE ----- - # request data from cache - try: - cache_data = cache_db.get(key=url) - if cache_data != None: - if cache_data["categories"].get("other"): - # move "other" to the end of the dict - cache_data["categories"]["other"] = cache_data["categories"].pop("other") - return cache_data - except Exception: - # Deta hasn't defined a specific exception for this error - # just ignore it - pass - # ----------------- - try: data = ingredients.scan(parsed_url) @@ -105,19 +98,10 @@ def get_scan(url: str, includeCategories: Optional[bool] = False): ingredient_data = json.loads(f.read()) ingredient_name = ingredient.split("/")[1].replace(".json", "") - + # ----- STATS ----- # increment matching scans for each ingredient - db_ingredient = db.get(ingredient_name) - - try: - db.update(key=ingredient_name, updates={ - "matching_scans": int(db_ingredient["matching_scans"]) + 1 - }) - except Exception: - # Deta hasn't defined a specific exception for this error - # just ignore it - pass + # # ----------------- return_data["matches"][ingredient.split("/")[0]].append( @@ -126,32 +110,21 @@ def get_scan(url: str, includeCategories: Optional[bool] = False): "name": ingredient_data["name"], "description": ingredient_data["description"], "icon": ingredient_data["icon"], - "match_percentage": round(((db_ingredient["matching_scans"] + 1) / db_ingredient["total_scans"]) * 100, 1) + #"match_percentage": round(((db_ingredient["matching_scans"] + 1) / db_ingredient["total_scans"]) * 100, 1) } ) if includeCategories: with open("ingredients/categories.json", "r") as f: return_data["categories"] = json.loads(f.read()) - - # ----- CACHE ----- - # add data to cache - # expiry: 15 minutes (900 seconds) - try: - cache_db.put(key=url, data=return_data, expire_in=900) - except Exception: - # Deta hasn't defined a specific exception for this error - # just ignore it - pass - # ----------------- return return_data except httpx.InvalidURL as e: raise HTTPException(status_code=400, detail=str(e)) except httpx.RequestError as e: raise HTTPException(status_code=400, detail=str(e)) - except: - raise HTTPException(status_code=500, detail=f"Unknown error") + #except: + # raise HTTPException(status_code=500, detail=f"Unknown error") @app.get("/icon/{icon}") diff --git a/requirements.txt b/requirements.txt index af8f96b..eb4fc1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ anyio==3.7.1 beautifulsoup4==4.12.2 certifi==2024.7.4 click==8.1.7 -deta==1.2.0 +#deta==1.2.0 fastapi==0.111.0 h11==0.14.0 httpcore==1.0.1 @@ -16,3 +16,5 @@ soupsieve==2.5 starlette==0.37.2 typing_extensions==4.8.0 uvicorn==0.24.0.post1 +redis==5.1.1 +python-dotenv==1.0.1 \ No newline at end of file diff --git a/ui/src/routes/+page.svelte b/ui/src/routes/+page.svelte index 62a7d53..d4a7d0b 100644 --- a/ui/src/routes/+page.svelte +++ b/ui/src/routes/+page.svelte @@ -157,11 +157,14 @@

{ ingredient.description }

+ +