From 96f4b74eead85a4a991068c131facbd4467e2b06 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Wed, 24 Jun 2026 14:07:31 +0400 Subject: [PATCH 01/12] restructure of the recommender service --- .../app/entry/{main.py => grpc_server.py} | 40 +- recommender/app/entry/kafka_consumer.py | 44 ++ recommender/app/entry/sync.py | 64 --- .../app/{db/__init__.py => entry/train.py} | 0 recommender/app/services/recommender.py | 52 +-- .../{ => app/shared}/config/settings.py | 0 recommender/app/{ => shared}/db/models.py | 18 +- recommender/app/shared/db/repo.py | 85 ++++ recommender/app/{ => shared}/db/session.py | 8 +- recommender/app/shared/kafka/models.py | 8 + recommender/app/shared/kafka/utils.py | 8 + recommender/app/shared/product/utils.py | 11 + recommender/config/__init__.py | 0 .../{app/entry/__init__.py => main.py} | 0 recommender/pyproject.toml | 33 ++ recommender/uv.lock | 435 ++++++++++++++++++ 16 files changed, 671 insertions(+), 135 deletions(-) rename recommender/app/entry/{main.py => grpc_server.py} (71%) create mode 100644 recommender/app/entry/kafka_consumer.py delete mode 100644 recommender/app/entry/sync.py rename recommender/app/{db/__init__.py => entry/train.py} (100%) rename recommender/{ => app/shared}/config/settings.py (100%) rename recommender/app/{ => shared}/db/models.py (66%) create mode 100644 recommender/app/shared/db/repo.py rename recommender/app/{ => shared}/db/session.py (64%) create mode 100644 recommender/app/shared/kafka/models.py create mode 100644 recommender/app/shared/kafka/utils.py create mode 100644 recommender/app/shared/product/utils.py delete mode 100644 recommender/config/__init__.py rename recommender/{app/entry/__init__.py => main.py} (100%) create mode 100644 recommender/pyproject.toml create mode 100644 recommender/uv.lock diff --git a/recommender/app/entry/main.py b/recommender/app/entry/grpc_server.py similarity index 71% rename from recommender/app/entry/main.py rename to recommender/app/entry/grpc_server.py index fdfb072..50974e3 100644 --- a/recommender/app/entry/main.py +++ b/recommender/app/entry/grpc_server.py @@ -3,28 +3,9 @@ from generated.pb import recommender_pb2, recommender_pb2_grpc from app.services.recommender import recommender -from app.db.models import Product -from app.db.session import ReplicaSession - -def _fetch_grpc_products(product_ids): - """Helper method to fetch products from the DB and return gRPC ProductReplica objects.""" - with ReplicaSession() as session: - products = ( - session.query(Product) - .filter(Product.id.in_(product_ids)) - .all() - ) - - grpc_products = [ - recommender_pb2.ProductReplica( - id=product.id, - name=product.name, - description=product.description, - price=product.price, - ) - for product in products - ] - return grpc_products +from recommender.app.shared.db.repo import get_products_by_ids +from recommender.app.shared.db.session import get_session +from recommender.app.shared.product.utils import fetch_product_by_id def _handle_exception(context, error_message): @@ -49,9 +30,13 @@ def GetRecommendations(self, request, context): ) # Fetch product details as gRPC objects - grpc_products = _fetch_grpc_products(recommended_product_ids) + recommended_products = [] + with get_session() as session: + products = get_products_by_ids(session, recommended_product_ids) + recommended_products = [product.to_grpc_model() for product in products] + return recommender_pb2.RecommendationResponse( - recommended_products=grpc_products + recommended_products=recommended_products ) except Exception as e: @@ -71,9 +56,12 @@ def GetRecommendationsBasedOnViewed(self, request, context): ) # Fetch product details as gRPC objects - grpc_products = _fetch_grpc_products(recommended_product_ids) + recommended_products = [] + with get_session() as session: + products = get_products_by_ids(session, recommended_product_ids) + recommended_products = [product.to_grpc_model() for product in products] return recommender_pb2.RecommendationResponse( - recommended_products=grpc_products + recommended_products=recommended_products ) except Exception as e: diff --git a/recommender/app/entry/kafka_consumer.py b/recommender/app/entry/kafka_consumer.py new file mode 100644 index 0000000..19148b1 --- /dev/null +++ b/recommender/app/entry/kafka_consumer.py @@ -0,0 +1,44 @@ +import json + +from kafka import KafkaConsumer +import requests +from loguru import logger + +from recommender.app.shared.product.utils import fetch_product_by_id +from shared.db.session import get_session +from shared.db.repo import create_or_update_product, create_product, delete_product_by_id, get_product_by_id, record_interaction +from shared.kafka.utils import product_is_created_or_updated, product_is_deleted +from shared.config.settings import KAFKA_SERVER + +def sync_products(): + consumer = KafkaConsumer("product_events", bootstrap_servers=KAFKA_SERVER) + for message in consumer: + event = json.loads(message.value) + with get_session() as session: + if product_is_created_or_updated(event): + product_data = event["data"] + logger.info("Processing product event {} for product ID: {}", event['type'], product_data['product_id']) + create_or_update_product(session, product_data) + + elif product_is_deleted(event): + delete_product_by_id(session, event["data"]["product_id"]) + +def process_interactions(): + consumer = KafkaConsumer("interaction_events", bootstrap_servers=KAFKA_SERVER) + for message in consumer: + event = json.loads(message.value) + with get_session() as session: + record_interaction(session, event) + product = get_product_by_id(session, event["data"]["product_id"]) + if not product: + try: + product = fetch_product_by_id(event["data"]["product_id"]) + create_product(session, product) + session.commit() + except requests.RequestException as e: + logger.error("Failed to fetch product {} for interaction event {}: {}", event['product_id'], event['type'], e) + session.rollback() + +if __name__ == "__main__": + sync_products() + process_interactions() \ No newline at end of file diff --git a/recommender/app/entry/sync.py b/recommender/app/entry/sync.py deleted file mode 100644 index 0b06165..0000000 --- a/recommender/app/entry/sync.py +++ /dev/null @@ -1,64 +0,0 @@ -from kafka import KafkaConsumer -import json -import requests -from app.db.session import ReplicaSession -from app.db.models import Product, Interaction -from config.settings import PRODUCT_API, KAFKA_SERVER - -def sync_products(): - consumer = KafkaConsumer("product_events", bootstrap_servers=KAFKA_SERVER) - for message in consumer: - event = json.loads(message.value) - with ReplicaSession() as session: - if event["type"] in ["product_created", "product_updated"]: - product_data = event["data"] - print(f"Product data: {product_data}") - print(f"Processing product event: {event['type']} for product ID: {product_data['product_id']}") - product = session.query(Product).filter_by(id=product_data["product_id"]).first() - if product: - product.name = product_data["name"] - product.description = product_data["description"] - product.price = product_data["price"] - product.account_id = product_data["account_id"] - else: - product = Product( - id=product_data["product_id"], - name=product_data["name"], - description=product_data["description"], - price=product_data["price"], - account_id=product_data["accountID"] - ) - session.add(product) - session.commit() - elif event["type"] == "product_deleted": - product = session.query(Product).filter_by(id=event["data"]["product_id"]).first() - if product: - session.delete(product) - session.commit() - -def process_interactions(): - consumer = KafkaConsumer("interaction_events", bootstrap_servers=KAFKA_SERVER) - for message in consumer: - event = json.loads(message.value) - with ReplicaSession() as session: - interaction = Interaction( - user_id=event["data"]["user_id"], - product_id=event["data"]["product_id"], - interaction_type=event["type"] - ) - session.add(interaction) - product = session.query(Product).filter_by(id=event["product_id"]).first() - if not product: - try: - response = requests.get(f"{PRODUCT_API}/{event['product_id']}") - response.raise_for_status() - product_data = response.json() - product = Product(**product_data) - session.add(product) - except requests.RequestException as e: - print(f"Failed to fetch product {event['product_id']}: {e}") - session.commit() - -if __name__ == "__main__": - sync_products() - process_interactions() \ No newline at end of file diff --git a/recommender/app/db/__init__.py b/recommender/app/entry/train.py similarity index 100% rename from recommender/app/db/__init__.py rename to recommender/app/entry/train.py diff --git a/recommender/app/services/recommender.py b/recommender/app/services/recommender.py index 2f3c0be..dcf69bb 100644 --- a/recommender/app/services/recommender.py +++ b/recommender/app/services/recommender.py @@ -1,43 +1,11 @@ from surprise import SVD, Dataset, Reader -import pandas as pd from app.db.session import ReplicaSession -from app.db.models import Interaction, Product - -def fetch_interactions() -> pd.DataFrame: - with ReplicaSession() as session: - interactions = session.query(Interaction).all() - data = [ - { - "user_id": i.user_id, - "product_id": i.product_id, - "rating": 3.0 if i.interaction_type == "purchase" else 1.0 - } - for i in interactions - ] - return pd.DataFrame(data) - -def _get_all_product_ids(session): - """Fetch all product IDs from the database.""" - return {p.id for p in session.query(Product.id).all()} - -def _get_interacted_ids_for_user(session, user_id: str): - """Get set of product IDs that the user has interacted with.""" - return { - i.product_id - for i in session.query(Interaction.product_id) - .filter(Interaction.user_id == user_id) - .all() - } - -def _get_interacted_ids_for_viewed(session, viewed_ids: list[str]): - """Get set of product IDs among 'viewed_ids' that have existing interactions.""" - return { - i.product_id - for i in session.query(Interaction.product_id) - .filter(Interaction.product_id.in_(viewed_ids)) - .all() - } - +from shared.repo import ( + fetch_interactions, + get_all_product_ids, + get_interacted_ids_for_user, + get_interacted_ids_for_viewed +) class Recommender: def __init__(self): @@ -65,8 +33,8 @@ def _predict_and_sort(self, user_id: str, candidates: list[str]) -> list: def recommend_on_user_id(self, user_id: str, skip: int = 0, take: int = 5) -> list[str]: """Recommend based on user interactions.""" with ReplicaSession() as session: - all_product_ids = _get_all_product_ids(session) - interacted_ids = _get_interacted_ids_for_user(session, user_id) + all_product_ids = get_all_product_ids(session) + interacted_ids = get_interacted_ids_for_user(session, user_id) candidates = [pid for pid in all_product_ids if pid not in interacted_ids] @@ -79,8 +47,8 @@ def recommend_on_user_id(self, user_id: str, skip: int = 0, take: int = 5) -> li def recommend_on_viewed_ids(self, viewed_ids: list[str], skip: int = 0, take: int = 5) -> list[str]: """Recommend based on a set of viewed product IDs.""" with ReplicaSession() as session: - all_product_ids = _get_all_product_ids(session) - interacted_ids = _get_interacted_ids_for_viewed(session, viewed_ids) + all_product_ids = get_all_product_ids(session) + interacted_ids = get_interacted_ids_for_viewed(session, viewed_ids) candidates = [pid for pid in all_product_ids if pid not in interacted_ids] diff --git a/recommender/config/settings.py b/recommender/app/shared/config/settings.py similarity index 100% rename from recommender/config/settings.py rename to recommender/app/shared/config/settings.py diff --git a/recommender/app/db/models.py b/recommender/app/shared/db/models.py similarity index 66% rename from recommender/app/db/models.py rename to recommender/app/shared/db/models.py index e393bfc..da8dfe6 100644 --- a/recommender/app/db/models.py +++ b/recommender/app/shared/db/models.py @@ -1,6 +1,7 @@ from sqlalchemy import String, Float, Integer, DateTime, func, ForeignKey from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Mapped, mapped_column, relationship +from generated.pb import recommender_pb2 Base = declarative_base() @@ -15,6 +16,14 @@ class Product(Base): interactions: Mapped[list["Interaction"]] = relationship( "Interaction", back_populates="product", cascade="all, delete-orphan") + def to_grpc_model(self) -> recommender_pb2.ProductReplica: + return recommender_pb2.ProductReplica( + id=self.id, + name=self.name, + description=self.description, + price=self.price + ) + class Interaction(Base): __tablename__ = "interactions" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) @@ -24,4 +33,11 @@ class Interaction(Base): timestamp: Mapped[DateTime] = mapped_column(DateTime, default=func.now()) product: Mapped["Product"] = relationship( - "Product", back_populates="interactions") \ No newline at end of file + "Product", back_populates="interactions") + + def to_grpc_model(self) -> recommender_pb2.Interaction: + return recommender_pb2.Interaction( + user_id=self.user_id, + product_id=self.product_id, + interaction_type=self.interaction_type + ) \ No newline at end of file diff --git a/recommender/app/shared/db/repo.py b/recommender/app/shared/db/repo.py new file mode 100644 index 0000000..4cf0dc9 --- /dev/null +++ b/recommender/app/shared/db/repo.py @@ -0,0 +1,85 @@ +import pandas as pd + +from shared.db.session import ReplicaSession +from shared.db.models import Interaction, Product + +def fetch_interactions() -> pd.DataFrame: + with ReplicaSession() as session: + interactions = session.query(Interaction).all() + data = [ + { + "user_id": i.user_id, + "product_id": i.product_id, + "rating": 3.0 if i.interaction_type == "purchase" else 1.0 + } + for i in interactions + ] + return pd.DataFrame(data) + +def get_all_product_ids(session): + """Fetch all product IDs from the database.""" + return {p.id for p in session.query(Product.id).all()} + +def get_product_by_id(session, product_id: str): + return session.query(Product).filter(Product.id == product_id).first() + +def get_products_by_ids(session, product_ids: list[str]): + return session.query(Product).filter(Product.id.in_(product_ids)).all() + +def get_interacted_ids_for_user(session, user_id: str): + """Get set of product IDs that the user has interacted with.""" + return { + i.product_id + for i in session.query(Interaction.product_id) + .filter(Interaction.user_id == user_id) + .all() + } + +def get_interacted_ids_for_viewed(session, viewed_ids: list[str]): + """Get set of product IDs among 'viewed_ids' that have existing interactions.""" + return { + i.product_id + for i in session.query(Interaction.product_id) + .filter(Interaction.product_id.in_(viewed_ids)) + .all() + } + +def record_interaction(session, event: dict): + interaction = Interaction( + user_id=event["data"]["user_id"], + product_id=event["data"]["product_id"], + interaction_type=event["type"] + ) + session.add(interaction) + +def create_product(session, product_data: dict): + product = Product( + id=product_data["product_id"], + name=product_data["name"], + description=product_data["description"], + price=product_data["price"], + account_id=product_data["accountID"] + ) + session.add(product) + +def update_product(session, product_data: dict): + product = get_product_by_id(session, product_data["product_id"]) + product.name = product_data["name"] + product.description = product_data["description"] + product.price = product_data["price"] + product.account_id = product_data["account_id"] + +def create_or_update_product(session, product_data: dict): + product = get_product_by_id(session, product_data["product_id"]) + if product: + update_product(session, product_data) + else: + create_product(session, product_data) + session.commit() + + +def delete_product_by_id(session, product_id: str): + product = get_product_by_id(session, product_id) + if product: + session.delete(product) + session.commit() \ No newline at end of file diff --git a/recommender/app/db/session.py b/recommender/app/shared/db/session.py similarity index 64% rename from recommender/app/db/session.py rename to recommender/app/shared/db/session.py index 854adb4..196e75c 100644 --- a/recommender/app/db/session.py +++ b/recommender/app/shared/db/session.py @@ -1,9 +1,13 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from app.db.models import Base -from config.settings import DATABASE_URL + +from shared.db.models import Base +from shared.config.settings import DATABASE_URL replica_engine = create_engine(DATABASE_URL, pool_pre_ping=True) ReplicaSession = sessionmaker(bind=replica_engine) Base.metadata.create_all(replica_engine) + +def get_session(): + return ReplicaSession() \ No newline at end of file diff --git a/recommender/app/shared/kafka/models.py b/recommender/app/shared/kafka/models.py new file mode 100644 index 0000000..6ee0ba9 --- /dev/null +++ b/recommender/app/shared/kafka/models.py @@ -0,0 +1,8 @@ +from enum import Enum + +class EventType(Enum): + PRODUCT_CREATED = "product_created" + PRODUCT_UPDATED = "product_updated" + PRODUCT_DELETED = "product_deleted" + INTERACTION_CREATED = "interaction_created" + INTERACTION_UPDATED = "interaction_updated" diff --git a/recommender/app/shared/kafka/utils.py b/recommender/app/shared/kafka/utils.py new file mode 100644 index 0000000..b69e945 --- /dev/null +++ b/recommender/app/shared/kafka/utils.py @@ -0,0 +1,8 @@ +from recommender.app.shared.kafka.models import EventType + + +def product_is_created_or_updated(event: dict) -> bool: + return event["type"] == EventType.PRODUCT_CREATED or event["type"] == EventType.PRODUCT_UPDATED + +def product_is_deleted(event: dict) -> bool: + return event["type"] == EventType.PRODUCT_DELETED \ No newline at end of file diff --git a/recommender/app/shared/product/utils.py b/recommender/app/shared/product/utils.py new file mode 100644 index 0000000..3910b7f --- /dev/null +++ b/recommender/app/shared/product/utils.py @@ -0,0 +1,11 @@ +import requests + +from shared.config.settings import PRODUCT_API + + +def fetch_product_by_id(product_id: str) -> dict: + """Fetch a product data by its ID from the product API.""" + response = requests.get(f"{PRODUCT_API}/{product_id}") + response.raise_for_status() + product_data = response.json() + return product_data diff --git a/recommender/config/__init__.py b/recommender/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/recommender/app/entry/__init__.py b/recommender/main.py similarity index 100% rename from recommender/app/entry/__init__.py rename to recommender/main.py diff --git a/recommender/pyproject.toml b/recommender/pyproject.toml new file mode 100644 index 0000000..29e5e17 --- /dev/null +++ b/recommender/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "recommender" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.11,<3.12" +dependencies = [ + "certifi==2025.1.31", + "charset-normalizer==3.4.1", + "greenlet==3.1.1", + "grpcio==1.71.0", + "grpcio-tools==1.71.0", + "idna==3.10", + "joblib==1.4.2", + "kafka-python==2.1.4", + "loguru>=0.7.3", + "numpy==1.26.4", + "pandas==2.2.3", + "protobuf==5.29.4", + "psycopg2-binary==2.9.10", + "python-dateutil==2.9.0.post0", + "python-dotenv==1.1.0", + "pytz==2025.2", + "requests==2.32.3", + "scikit-surprise==1.1.4", + "scipy==1.15.2", + "six==1.17.0", + "sqlalchemy==2.0.40", + "surprise==0.1", + "typing-extensions==4.13.0", + "tzdata==2025.2", + "urllib3==2.3.0", +] diff --git a/recommender/uv.lock b/recommender/uv.lock new file mode 100644 index 0000000..16c2da8 --- /dev/null +++ b/recommender/uv.lock @@ -0,0 +1,435 @@ +version = 1 +revision = 3 +requires-python = "==3.11.*" + +[[package]] +name = "certifi" +version = "2025.1.31" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", size = 167577, upload-time = "2025-01-31T02:16:47.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393, upload-time = "2025-01-31T02:16:45.015Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", size = 123188, upload-time = "2024-12-24T18:12:35.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/80/41ef5d5a7935d2d3a773e3eaebf0a9350542f2cab4eac59a7a4741fbbbbe/charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125", size = 194995, upload-time = "2024-12-24T18:10:12.838Z" }, + { url = "https://files.pythonhosted.org/packages/7a/28/0b9fefa7b8b080ec492110af6d88aa3dea91c464b17d53474b6e9ba5d2c5/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1", size = 139471, upload-time = "2024-12-24T18:10:14.101Z" }, + { url = "https://files.pythonhosted.org/packages/71/64/d24ab1a997efb06402e3fc07317e94da358e2585165930d9d59ad45fcae2/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3", size = 149831, upload-time = "2024-12-24T18:10:15.512Z" }, + { url = "https://files.pythonhosted.org/packages/37/ed/be39e5258e198655240db5e19e0b11379163ad7070962d6b0c87ed2c4d39/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd", size = 142335, upload-time = "2024-12-24T18:10:18.369Z" }, + { url = "https://files.pythonhosted.org/packages/88/83/489e9504711fa05d8dde1574996408026bdbdbd938f23be67deebb5eca92/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00", size = 143862, upload-time = "2024-12-24T18:10:19.743Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c7/32da20821cf387b759ad24627a9aca289d2822de929b8a41b6241767b461/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12", size = 145673, upload-time = "2024-12-24T18:10:21.139Z" }, + { url = "https://files.pythonhosted.org/packages/68/85/f4288e96039abdd5aeb5c546fa20a37b50da71b5cf01e75e87f16cd43304/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77", size = 140211, upload-time = "2024-12-24T18:10:22.382Z" }, + { url = "https://files.pythonhosted.org/packages/28/a3/a42e70d03cbdabc18997baf4f0227c73591a08041c149e710045c281f97b/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146", size = 148039, upload-time = "2024-12-24T18:10:24.802Z" }, + { url = "https://files.pythonhosted.org/packages/85/e4/65699e8ab3014ecbe6f5c71d1a55d810fb716bbfd74f6283d5c2aa87febf/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd", size = 151939, upload-time = "2024-12-24T18:10:26.124Z" }, + { url = "https://files.pythonhosted.org/packages/b1/82/8e9fe624cc5374193de6860aba3ea8070f584c8565ee77c168ec13274bd2/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6", size = 149075, upload-time = "2024-12-24T18:10:30.027Z" }, + { url = "https://files.pythonhosted.org/packages/3d/7b/82865ba54c765560c8433f65e8acb9217cb839a9e32b42af4aa8e945870f/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8", size = 144340, upload-time = "2024-12-24T18:10:32.679Z" }, + { url = "https://files.pythonhosted.org/packages/b5/b6/9674a4b7d4d99a0d2df9b215da766ee682718f88055751e1e5e753c82db0/charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b", size = 95205, upload-time = "2024-12-24T18:10:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ab/45b180e175de4402dcf7547e4fb617283bae54ce35c27930a6f35b6bef15/charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76", size = 102441, upload-time = "2024-12-24T18:10:37.574Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767, upload-time = "2024-12-24T18:12:32.852Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "greenlet" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/ff/df5fede753cc10f6a5be0931204ea30c35fa2f2ea7a35b25bdaf4fe40e46/greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467", size = 186022, upload-time = "2024-09-20T18:21:04.506Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/62/1c2665558618553c42922ed47a4e6d6527e2fa3516a8256c2f431c5d0441/greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70", size = 272479, upload-time = "2024-09-20T17:07:22.332Z" }, + { url = "https://files.pythonhosted.org/packages/76/9d/421e2d5f07285b6e4e3a676b016ca781f63cfe4a0cd8eaecf3fd6f7a71ae/greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159", size = 640404, upload-time = "2024-09-20T17:36:45.588Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/6e05f5c59262a584e502dd3d261bbdd2c97ab5416cc9c0b91ea38932a901/greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e", size = 652813, upload-time = "2024-09-20T17:39:19.052Z" }, + { url = "https://files.pythonhosted.org/packages/49/93/d5f93c84241acdea15a8fd329362c2c71c79e1a507c3f142a5d67ea435ae/greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1", size = 648517, upload-time = "2024-09-20T17:44:24.101Z" }, + { url = "https://files.pythonhosted.org/packages/15/85/72f77fc02d00470c86a5c982b8daafdf65d38aefbbe441cebff3bf7037fc/greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383", size = 647831, upload-time = "2024-09-20T17:08:40.577Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4b/1c9695aa24f808e156c8f4813f685d975ca73c000c2a5056c514c64980f6/greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a", size = 602413, upload-time = "2024-09-20T17:08:31.728Z" }, + { url = "https://files.pythonhosted.org/packages/76/70/ad6e5b31ef330f03b12559d19fda2606a522d3849cde46b24f223d6d1619/greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511", size = 1129619, upload-time = "2024-09-20T17:44:14.222Z" }, + { url = "https://files.pythonhosted.org/packages/f4/fb/201e1b932e584066e0f0658b538e73c459b34d44b4bd4034f682423bc801/greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395", size = 1155198, upload-time = "2024-09-20T17:09:23.903Z" }, + { url = "https://files.pythonhosted.org/packages/12/da/b9ed5e310bb8b89661b80cbcd4db5a067903bbcd7fc854923f5ebb4144f0/greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39", size = 298930, upload-time = "2024-09-20T17:25:18.656Z" }, +] + +[[package]] +name = "grpcio" +version = "1.71.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/95/aa11fc09a85d91fbc7dd405dcb2a1e0256989d67bf89fa65ae24b3ba105a/grpcio-1.71.0.tar.gz", hash = "sha256:2b85f7820475ad3edec209d3d89a7909ada16caab05d3f2e08a7e8ae3200a55c", size = 12549828, upload-time = "2025-03-10T19:28:49.203Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/04/a085f3ad4133426f6da8c1becf0749872a49feb625a407a2e864ded3fb12/grpcio-1.71.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:d6aa986318c36508dc1d5001a3ff169a15b99b9f96ef5e98e13522c506b37eef", size = 5210453, upload-time = "2025-03-10T19:24:33.342Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d5/0bc53ed33ba458de95020970e2c22aa8027b26cc84f98bea7fcad5d695d1/grpcio-1.71.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:d2c170247315f2d7e5798a22358e982ad6eeb68fa20cf7a820bb74c11f0736e7", size = 11347567, upload-time = "2025-03-10T19:24:35.215Z" }, + { url = "https://files.pythonhosted.org/packages/e3/6d/ce334f7e7a58572335ccd61154d808fe681a4c5e951f8a1ff68f5a6e47ce/grpcio-1.71.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:e6f83a583ed0a5b08c5bc7a3fe860bb3c2eac1f03f1f63e0bc2091325605d2b7", size = 5696067, upload-time = "2025-03-10T19:24:37.988Z" }, + { url = "https://files.pythonhosted.org/packages/05/4a/80befd0b8b1dc2b9ac5337e57473354d81be938f87132e147c4a24a581bd/grpcio-1.71.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4be74ddeeb92cc87190e0e376dbc8fc7736dbb6d3d454f2fa1f5be1dee26b9d7", size = 6348377, upload-time = "2025-03-10T19:24:40.361Z" }, + { url = "https://files.pythonhosted.org/packages/c7/67/cbd63c485051eb78663355d9efd1b896cfb50d4a220581ec2cb9a15cd750/grpcio-1.71.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dd0dfbe4d5eb1fcfec9490ca13f82b089a309dc3678e2edabc144051270a66e", size = 5940407, upload-time = "2025-03-10T19:24:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/7a11aa4326d7faa499f764eaf8a9b5a0eb054ce0988ee7ca34897c2b02ae/grpcio-1.71.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a2242d6950dc892afdf9e951ed7ff89473aaf744b7d5727ad56bdaace363722b", size = 6030915, upload-time = "2025-03-10T19:24:44.463Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a2/cdae2d0e458b475213a011078b0090f7a1d87f9a68c678b76f6af7c6ac8c/grpcio-1.71.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0fa05ee31a20456b13ae49ad2e5d585265f71dd19fbd9ef983c28f926d45d0a7", size = 6648324, upload-time = "2025-03-10T19:24:46.287Z" }, + { url = "https://files.pythonhosted.org/packages/27/df/f345c8daaa8d8574ce9869f9b36ca220c8845923eb3087e8f317eabfc2a8/grpcio-1.71.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3d081e859fb1ebe176de33fc3adb26c7d46b8812f906042705346b314bde32c3", size = 6197839, upload-time = "2025-03-10T19:24:48.565Z" }, + { url = "https://files.pythonhosted.org/packages/f2/2c/cd488dc52a1d0ae1bad88b0d203bc302efbb88b82691039a6d85241c5781/grpcio-1.71.0-cp311-cp311-win32.whl", hash = "sha256:d6de81c9c00c8a23047136b11794b3584cdc1460ed7cbc10eada50614baa1444", size = 3619978, upload-time = "2025-03-10T19:24:50.518Z" }, + { url = "https://files.pythonhosted.org/packages/ee/3f/cf92e7e62ccb8dbdf977499547dfc27133124d6467d3a7d23775bcecb0f9/grpcio-1.71.0-cp311-cp311-win_amd64.whl", hash = "sha256:24e867651fc67717b6f896d5f0cac0ec863a8b5fb7d6441c2ab428f52c651c6b", size = 4282279, upload-time = "2025-03-10T19:24:52.313Z" }, +] + +[[package]] +name = "grpcio-tools" +version = "1.71.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/d2/c0866a48c355a6a4daa1f7e27e210c7fa561b1f3b7c0bce2671e89cfa31e/grpcio_tools-1.71.0.tar.gz", hash = "sha256:38dba8e0d5e0fb23a034e09644fdc6ed862be2371887eee54901999e8f6792a8", size = 5326008, upload-time = "2025-03-10T19:29:03.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/ec/73b9797ffec80e1faf039ce3e2f0513e26e1a68eedc525ed294ae2a44d03/grpcio_tools-1.71.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:1f19b16b49afa5d21473f49c0966dd430c88d089cd52ac02404d8cef67134efb", size = 2385557, upload-time = "2025-03-10T19:27:27.62Z" }, + { url = "https://files.pythonhosted.org/packages/bf/87/42c6e192b7b09c9610a53e771797f7826aee4f6e769683985ae406a2d862/grpcio_tools-1.71.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:459c8f5e00e390aecd5b89de67deb3ec7188a274bc6cb50e43cef35ab3a3f45d", size = 5954404, upload-time = "2025-03-10T19:27:29.835Z" }, + { url = "https://files.pythonhosted.org/packages/25/30/3fd385a56d32dce34cde09a64dbaf7cf85d395f2bcd86dd41e4b4ee5938f/grpcio_tools-1.71.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:edab7e6518de01196be37f96cb1e138c3819986bf5e2a6c9e1519b4d716b2f5a", size = 2352061, upload-time = "2025-03-10T19:27:31.624Z" }, + { url = "https://files.pythonhosted.org/packages/87/eb/e9971c7693a2d85e7f55760f7906211a95ff74af4d41b05d187849d7fb58/grpcio_tools-1.71.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b93b9f6adc7491d4c10144c0643409db298e5e63c997106a804f6f0248dbaf4", size = 2745033, upload-time = "2025-03-10T19:27:33.787Z" }, + { url = "https://files.pythonhosted.org/packages/15/72/4e69beae87a1b334f80da9e93c8e2f5c8fe4860c956a781246a092dc4c97/grpcio_tools-1.71.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ae5f2efa9e644c10bf1021600bfc099dfbd8e02b184d2d25dc31fcd6c2bc59e", size = 2476743, upload-time = "2025-03-10T19:27:35.896Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f3/336d2c83f1bfc00a5376bf20dd2273d7aa891b03dd91b11c71ca47392351/grpcio_tools-1.71.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:65aa082f4435571d65d5ce07fc444f23c3eff4f3e34abef599ef8c9e1f6f360f", size = 2853693, upload-time = "2025-03-10T19:27:37.624Z" }, + { url = "https://files.pythonhosted.org/packages/62/ba/cc7ace518c11501a4b8620df5edb8188e81470e5b82dc6829212f3e9b2ff/grpcio_tools-1.71.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1331e726e08b7bdcbf2075fcf4b47dff07842b04845e6e220a08a4663e232d7f", size = 3304474, upload-time = "2025-03-10T19:27:39.351Z" }, + { url = "https://files.pythonhosted.org/packages/00/0d/4b843654af3d5aa2f1a5775df1d583e6e3471e6d569106fd3213ad185a98/grpcio_tools-1.71.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6693a7d3ba138b0e693b3d1f687cdd9db9e68976c3fa2b951c17a072fea8b583", size = 2916147, upload-time = "2025-03-10T19:27:41.022Z" }, + { url = "https://files.pythonhosted.org/packages/e4/14/047e1c817422bc3d434247b9c640c51fd51ca4e047583ff31d927c3dea73/grpcio_tools-1.71.0-cp311-cp311-win32.whl", hash = "sha256:6d11ed3ff7b6023b5c72a8654975324bb98c1092426ba5b481af406ff559df00", size = 949374, upload-time = "2025-03-10T19:27:42.999Z" }, + { url = "https://files.pythonhosted.org/packages/86/cb/739a1b6d517672693796022c0f9061f63eaa243ec70cbbfa59bf881ed9fb/grpcio_tools-1.71.0-cp311-cp311-win_amd64.whl", hash = "sha256:072b2a5805ac97e4623b3aa8f7818275f3fb087f4aa131b0fce00471065f6eaa", size = 1120786, upload-time = "2025-03-10T19:27:44.706Z" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "joblib" +version = "1.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/64/33/60135848598c076ce4b231e1b1895170f45fbcaeaa2c9d5e38b04db70c35/joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e", size = 2116621, upload-time = "2024-05-02T12:15:05.765Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/29/df4b9b42f2be0b623cbd5e2140cafcaa2bef0759a00b7b70104dcfe2fb51/joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6", size = 301817, upload-time = "2024-05-02T12:15:00.765Z" }, +] + +[[package]] +name = "kafka-python" +version = "2.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/20/d0b75cdba5aa06d9423dac34f5215fb66c273d474ddfc9b96bf28251a059/kafka_python-2.1.4.tar.gz", hash = "sha256:7e45be44cc7eb3dcad8b432cc514411454d05971ad26411092577584ee15d67a", size = 306782, upload-time = "2025-03-28T18:03:43.267Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/01/e5a16a51093febcee7cd833a6408eb2bcf44f3282cf96a4e731c036454ff/kafka_python-2.1.4-py2.py3-none-any.whl", hash = "sha256:23340cdcd403d4f73564c068fd3b1f4b4be3fc1a89f3ef1a76d46ea889878190", size = 276618, upload-time = "2025-03-28T18:03:41.483Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, + { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, + { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, + { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, + { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, +] + +[[package]] +name = "pandas" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/44/d9502bf0ed197ba9bf1103c9867d5904ddcaf869e52329787fc54ed70cc8/pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039", size = 12602222, upload-time = "2024-09-20T13:08:56.254Z" }, + { url = "https://files.pythonhosted.org/packages/52/11/9eac327a38834f162b8250aab32a6781339c69afe7574368fffe46387edf/pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd", size = 11321274, upload-time = "2024-09-20T13:08:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/45/fb/c4beeb084718598ba19aa9f5abbc8aed8b42f90930da861fcb1acdb54c3a/pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698", size = 15579836, upload-time = "2024-09-20T19:01:57.571Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5f/4dba1d39bb9c38d574a9a22548c540177f78ea47b32f99c0ff2ec499fac5/pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc", size = 13058505, upload-time = "2024-09-20T13:09:01.501Z" }, + { url = "https://files.pythonhosted.org/packages/b9/57/708135b90391995361636634df1f1130d03ba456e95bcf576fada459115a/pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3", size = 16744420, upload-time = "2024-09-20T19:02:00.678Z" }, + { url = "https://files.pythonhosted.org/packages/86/4a/03ed6b7ee323cf30404265c284cee9c65c56a212e0a08d9ee06984ba2240/pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32", size = 14440457, upload-time = "2024-09-20T13:09:04.105Z" }, + { url = "https://files.pythonhosted.org/packages/ed/8c/87ddf1fcb55d11f9f847e3c69bb1c6f8e46e2f40ab1a2d2abadb2401b007/pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5", size = 11617166, upload-time = "2024-09-20T13:09:06.917Z" }, +] + +[[package]] +name = "protobuf" +version = "5.29.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/17/7d/b9dca7365f0e2c4fa7c193ff795427cfa6290147e5185ab11ece280a18e7/protobuf-5.29.4.tar.gz", hash = "sha256:4f1dfcd7997b31ef8f53ec82781ff434a28bf71d9102ddde14d076adcfc78c99", size = 424902, upload-time = "2025-03-19T21:23:24.25Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/b2/043a1a1a20edd134563699b0e91862726a0dc9146c090743b6c44d798e75/protobuf-5.29.4-cp310-abi3-win32.whl", hash = "sha256:13eb236f8eb9ec34e63fc8b1d6efd2777d062fa6aaa68268fb67cf77f6839ad7", size = 422709, upload-time = "2025-03-19T21:23:08.293Z" }, + { url = "https://files.pythonhosted.org/packages/79/fc/2474b59570daa818de6124c0a15741ee3e5d6302e9d6ce0bdfd12e98119f/protobuf-5.29.4-cp310-abi3-win_amd64.whl", hash = "sha256:bcefcdf3976233f8a502d265eb65ea740c989bacc6c30a58290ed0e519eb4b8d", size = 434506, upload-time = "2025-03-19T21:23:11.253Z" }, + { url = "https://files.pythonhosted.org/packages/46/de/7c126bbb06aa0f8a7b38aaf8bd746c514d70e6a2a3f6dd460b3b7aad7aae/protobuf-5.29.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:307ecba1d852ec237e9ba668e087326a67564ef83e45a0189a772ede9e854dd0", size = 417826, upload-time = "2025-03-19T21:23:13.132Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b5/bade14ae31ba871a139aa45e7a8183d869efe87c34a4850c87b936963261/protobuf-5.29.4-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:aec4962f9ea93c431d5714ed1be1c93f13e1a8618e70035ba2b0564d9e633f2e", size = 319574, upload-time = "2025-03-19T21:23:14.531Z" }, + { url = "https://files.pythonhosted.org/packages/46/88/b01ed2291aae68b708f7d334288ad5fb3e7aa769a9c309c91a0d55cb91b0/protobuf-5.29.4-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:d7d3f7d1d5a66ed4942d4fefb12ac4b14a29028b209d4bfb25c68ae172059922", size = 319672, upload-time = "2025-03-19T21:23:15.839Z" }, + { url = "https://files.pythonhosted.org/packages/12/fb/a586e0c973c95502e054ac5f81f88394f24ccc7982dac19c515acd9e2c93/protobuf-5.29.4-py3-none-any.whl", hash = "sha256:3fde11b505e1597f71b875ef2fc52062b6a9740e5f7c8997ce878b6009145862", size = 172551, upload-time = "2025-03-19T21:23:22.682Z" }, +] + +[[package]] +name = "psycopg2-binary" +version = "2.9.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/bdc8274dc0585090b4e3432267d7be4dfbfd8971c0fa59167c711105a6bf/psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", size = 385764, upload-time = "2024-10-16T11:24:58.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/8f/9feb01291d0d7a0a4c6a6bab24094135c2b59c6a81943752f632c75896d6/psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff", size = 3043397, upload-time = "2024-10-16T11:19:40.033Z" }, + { url = "https://files.pythonhosted.org/packages/15/30/346e4683532011561cd9c8dfeac6a8153dd96452fee0b12666058ab7893c/psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c", size = 3274806, upload-time = "2024-10-16T11:19:43.5Z" }, + { url = "https://files.pythonhosted.org/packages/66/6e/4efebe76f76aee7ec99166b6c023ff8abdc4e183f7b70913d7c047701b79/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c", size = 2851370, upload-time = "2024-10-16T11:19:46.986Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fd/ff83313f86b50f7ca089b161b8e0a22bb3c319974096093cd50680433fdb/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb", size = 3080780, upload-time = "2024-10-16T11:19:50.242Z" }, + { url = "https://files.pythonhosted.org/packages/e6/c4/bfadd202dcda8333a7ccafdc51c541dbdfce7c2c7cda89fa2374455d795f/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341", size = 3264583, upload-time = "2024-10-16T11:19:54.424Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f1/09f45ac25e704ac954862581f9f9ae21303cc5ded3d0b775532b407f0e90/psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a", size = 3019831, upload-time = "2024-10-16T11:19:57.762Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2e/9beaea078095cc558f215e38f647c7114987d9febfc25cb2beed7c3582a5/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b", size = 2871822, upload-time = "2024-10-16T11:20:04.693Z" }, + { url = "https://files.pythonhosted.org/packages/01/9e/ef93c5d93f3dc9fc92786ffab39e323b9aed066ba59fdc34cf85e2722271/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7", size = 2820975, upload-time = "2024-10-16T11:20:11.401Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f0/049e9631e3268fe4c5a387f6fc27e267ebe199acf1bc1bc9cbde4bd6916c/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e", size = 2919320, upload-time = "2024-10-16T11:20:17.959Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9a/bcb8773b88e45fb5a5ea8339e2104d82c863a3b8558fbb2aadfe66df86b3/psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68", size = 2957617, upload-time = "2024-10-16T11:20:24.711Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6b/144336a9bf08a67d217b3af3246abb1d027095dab726f0687f01f43e8c03/psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392", size = 1024618, upload-time = "2024-10-16T11:20:27.718Z" }, + { url = "https://files.pythonhosted.org/packages/61/69/3b3d7bd583c6d3cbe5100802efa5beacaacc86e37b653fc708bf3d6853b8/psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4", size = 1163816, upload-time = "2024-10-16T11:20:30.777Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/2c/7bb1416c5620485aa793f2de31d3df393d3686aa8a8506d11e10e13c5baf/python_dotenv-1.1.0.tar.gz", hash = "sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5", size = 39920, upload-time = "2025-03-25T10:14:56.835Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/18/98a99ad95133c6a6e2005fe89faedf294a748bd5dc803008059409ac9b1e/python_dotenv-1.1.0-py3-none-any.whl", hash = "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d", size = 20256, upload-time = "2025-03-25T10:14:55.034Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "recommender" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "greenlet" }, + { name = "grpcio" }, + { name = "grpcio-tools" }, + { name = "idna" }, + { name = "joblib" }, + { name = "kafka-python" }, + { name = "loguru" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "psycopg2-binary" }, + { name = "python-dateutil" }, + { name = "python-dotenv" }, + { name = "pytz" }, + { name = "requests" }, + { name = "scikit-surprise" }, + { name = "scipy" }, + { name = "six" }, + { name = "sqlalchemy" }, + { name = "surprise" }, + { name = "typing-extensions" }, + { name = "tzdata" }, + { name = "urllib3" }, +] + +[package.metadata] +requires-dist = [ + { name = "certifi", specifier = "==2025.1.31" }, + { name = "charset-normalizer", specifier = "==3.4.1" }, + { name = "greenlet", specifier = "==3.1.1" }, + { name = "grpcio", specifier = "==1.71.0" }, + { name = "grpcio-tools", specifier = "==1.71.0" }, + { name = "idna", specifier = "==3.10" }, + { name = "joblib", specifier = "==1.4.2" }, + { name = "kafka-python", specifier = "==2.1.4" }, + { name = "loguru", specifier = ">=0.7.3" }, + { name = "numpy", specifier = "==1.26.4" }, + { name = "pandas", specifier = "==2.2.3" }, + { name = "protobuf", specifier = "==5.29.4" }, + { name = "psycopg2-binary", specifier = "==2.9.10" }, + { name = "python-dateutil", specifier = "==2.9.0.post0" }, + { name = "python-dotenv", specifier = "==1.1.0" }, + { name = "pytz", specifier = "==2025.2" }, + { name = "requests", specifier = "==2.32.3" }, + { name = "scikit-surprise", specifier = "==1.1.4" }, + { name = "scipy", specifier = "==1.15.2" }, + { name = "six", specifier = "==1.17.0" }, + { name = "sqlalchemy", specifier = "==2.0.40" }, + { name = "surprise", specifier = "==0.1" }, + { name = "typing-extensions", specifier = "==4.13.0" }, + { name = "tzdata", specifier = "==2025.2" }, + { name = "urllib3", specifier = "==2.3.0" }, +] + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218, upload-time = "2024-05-29T15:37:49.536Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload-time = "2024-05-29T15:37:47.027Z" }, +] + +[[package]] +name = "scikit-surprise" +version = "1.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/8d/40ac32e703f3808159f9e2b33760cfbd6224cc7783eb663091eddc9581c2/scikit_surprise-1.1.4.tar.gz", hash = "sha256:130c45feaee9de4b8cba0aff413ad9b51b2d5c5c90d41aee4759e00059913752", size = 154434, upload-time = "2024-05-19T14:25:59.527Z" } + +[[package]] +name = "scipy" +version = "1.15.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/b9/31ba9cd990e626574baf93fbc1ac61cf9ed54faafd04c479117517661637/scipy-1.15.2.tar.gz", hash = "sha256:cd58a314d92838f7e6f755c8a2167ead4f27e1fd5c1251fd54289569ef3495ec", size = 59417316, upload-time = "2025-02-17T00:42:24.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/1f/bf0a5f338bda7c35c08b4ed0df797e7bafe8a78a97275e9f439aceb46193/scipy-1.15.2-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:92233b2df6938147be6fa8824b8136f29a18f016ecde986666be5f4d686a91a4", size = 38703651, upload-time = "2025-02-17T00:30:31.09Z" }, + { url = "https://files.pythonhosted.org/packages/de/54/db126aad3874601048c2c20ae3d8a433dbfd7ba8381551e6f62606d9bd8e/scipy-1.15.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:62ca1ff3eb513e09ed17a5736929429189adf16d2d740f44e53270cc800ecff1", size = 30102038, upload-time = "2025-02-17T00:30:40.219Z" }, + { url = "https://files.pythonhosted.org/packages/61/d8/84da3fffefb6c7d5a16968fe5b9f24c98606b165bb801bb0b8bc3985200f/scipy-1.15.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4c6676490ad76d1c2894d77f976144b41bd1a4052107902238047fb6a473e971", size = 22375518, upload-time = "2025-02-17T00:30:47.547Z" }, + { url = "https://files.pythonhosted.org/packages/44/78/25535a6e63d3b9c4c90147371aedb5d04c72f3aee3a34451f2dc27c0c07f/scipy-1.15.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a8bf5cb4a25046ac61d38f8d3c3426ec11ebc350246a4642f2f315fe95bda655", size = 25142523, upload-time = "2025-02-17T00:30:56.002Z" }, + { url = "https://files.pythonhosted.org/packages/e0/22/4b4a26fe1cd9ed0bc2b2cb87b17d57e32ab72c346949eaf9288001f8aa8e/scipy-1.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a8e34cf4c188b6dd004654f88586d78f95639e48a25dfae9c5e34a6dc34547e", size = 35491547, upload-time = "2025-02-17T00:31:07.599Z" }, + { url = "https://files.pythonhosted.org/packages/32/ea/564bacc26b676c06a00266a3f25fdfe91a9d9a2532ccea7ce6dd394541bc/scipy-1.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28a0d2c2075946346e4408b211240764759e0fabaeb08d871639b5f3b1aca8a0", size = 37634077, upload-time = "2025-02-17T00:31:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/43/c2/bfd4e60668897a303b0ffb7191e965a5da4056f0d98acfb6ba529678f0fb/scipy-1.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:42dabaaa798e987c425ed76062794e93a243be8f0f20fff6e7a89f4d61cb3d40", size = 37231657, upload-time = "2025-02-17T00:31:22.041Z" }, + { url = "https://files.pythonhosted.org/packages/4a/75/5f13050bf4f84c931bcab4f4e83c212a36876c3c2244475db34e4b5fe1a6/scipy-1.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6f5e296ec63c5da6ba6fa0343ea73fd51b8b3e1a300b0a8cae3ed4b1122c7462", size = 40035857, upload-time = "2025-02-17T00:31:29.836Z" }, + { url = "https://files.pythonhosted.org/packages/b9/8b/7ec1832b09dbc88f3db411f8cdd47db04505c4b72c99b11c920a8f0479c3/scipy-1.15.2-cp311-cp311-win_amd64.whl", hash = "sha256:597a0c7008b21c035831c39927406c6181bcf8f60a73f36219b69d010aa04737", size = 41217654, upload-time = "2025-02-17T00:31:43.65Z" }, +] + +[[package]] +name = "setuptools" +version = "82.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.40" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/c3/3f2bfa5e4dcd9938405fe2fab5b6ab94a9248a4f9536ea2fd497da20525f/sqlalchemy-2.0.40.tar.gz", hash = "sha256:d827099289c64589418ebbcaead0145cd19f4e3e8a93919a0100247af245fa00", size = 9664299, upload-time = "2025-03-27T17:52:31.876Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/7e/55044a9ec48c3249bb38d5faae93f09579c35e862bb318ebd1ed7a1994a5/sqlalchemy-2.0.40-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f6bacab7514de6146a1976bc56e1545bee247242fab030b89e5f70336fc0003e", size = 2114025, upload-time = "2025-03-27T18:49:29.456Z" }, + { url = "https://files.pythonhosted.org/packages/77/0f/dcf7bba95f847aec72f638750747b12d37914f71c8cc7c133cf326ab945c/sqlalchemy-2.0.40-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5654d1ac34e922b6c5711631f2da497d3a7bffd6f9f87ac23b35feea56098011", size = 2104419, upload-time = "2025-03-27T18:49:30.75Z" }, + { url = "https://files.pythonhosted.org/packages/75/70/c86a5c20715e4fe903dde4c2fd44fc7e7a0d5fb52c1b954d98526f65a3ea/sqlalchemy-2.0.40-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35904d63412db21088739510216e9349e335f142ce4a04b69e2528020ee19ed4", size = 3222720, upload-time = "2025-03-27T18:44:29.871Z" }, + { url = "https://files.pythonhosted.org/packages/12/cf/b891a8c1d0c27ce9163361664c2128c7a57de3f35000ea5202eb3a2917b7/sqlalchemy-2.0.40-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7a80ed86d6aaacb8160a1caef6680d4ddd03c944d985aecee940d168c411d1", size = 3222682, upload-time = "2025-03-27T18:55:20.097Z" }, + { url = "https://files.pythonhosted.org/packages/15/3f/7709d8c8266953d945435a96b7f425ae4172a336963756b58e996fbef7f3/sqlalchemy-2.0.40-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:519624685a51525ddaa7d8ba8265a1540442a2ec71476f0e75241eb8263d6f51", size = 3159542, upload-time = "2025-03-27T18:44:31.333Z" }, + { url = "https://files.pythonhosted.org/packages/85/7e/717eaabaf0f80a0132dc2032ea8f745b7a0914451c984821a7c8737fb75a/sqlalchemy-2.0.40-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2ee5f9999a5b0e9689bed96e60ee53c3384f1a05c2dd8068cc2e8361b0df5b7a", size = 3179864, upload-time = "2025-03-27T18:55:21.784Z" }, + { url = "https://files.pythonhosted.org/packages/e4/cc/03eb5dfcdb575cbecd2bd82487b9848f250a4b6ecfb4707e834b4ce4ec07/sqlalchemy-2.0.40-cp311-cp311-win32.whl", hash = "sha256:c0cae71e20e3c02c52f6b9e9722bca70e4a90a466d59477822739dc31ac18b4b", size = 2084675, upload-time = "2025-03-27T18:48:55.915Z" }, + { url = "https://files.pythonhosted.org/packages/9a/48/440946bf9dc4dc231f4f31ef0d316f7135bf41d4b86aaba0c0655150d370/sqlalchemy-2.0.40-cp311-cp311-win_amd64.whl", hash = "sha256:574aea2c54d8f1dd1699449f332c7d9b71c339e04ae50163a3eb5ce4c4325ee4", size = 2110099, upload-time = "2025-03-27T18:48:57.45Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7c/5fc8e802e7506fe8b55a03a2e1dab156eae205c91bee46305755e086d2e2/sqlalchemy-2.0.40-py3-none-any.whl", hash = "sha256:32587e2e1e359276957e6fe5dad089758bc042a971a8a09ae8ecf7a8fe23d07a", size = 1903894, upload-time = "2025-03-27T18:40:43.796Z" }, +] + +[[package]] +name = "surprise" +version = "0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "scikit-surprise" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/de/e5cba8682201fcf9c3719a6fdda95693468ed061945493dea2dd37c5618b/surprise-0.1-py2.py3-none-any.whl", hash = "sha256:0b5ad9b2e1c07ad74b93ece2e74beca83a05e32c7bd2ff2dcff7261e4cad0305", size = 1821, upload-time = "2017-01-14T13:39:47.865Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/3e/b00a62db91a83fff600de219b6ea9908e6918664899a2d85db222f4fbf19/typing_extensions-4.13.0.tar.gz", hash = "sha256:0a4ac55a5820789d87e297727d229866c9650f6521b64206413c4fbada24d95b", size = 106520, upload-time = "2025-03-26T03:49:41.628Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/86/39b65d676ec5732de17b7e3c476e45bb80ec64eb50737a8dce1a4178aba1/typing_extensions-4.13.0-py3-none-any.whl", hash = "sha256:c8dd92cc0d6425a97c18fbb9d1954e5ff92c1ca881a309c45f06ebc0b79058e5", size = 45683, upload-time = "2025-03-26T03:49:40.35Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "urllib3" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/63/e53da845320b757bf29ef6a9062f5c669fe997973f966045cb019c3f4b66/urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d", size = 307268, upload-time = "2024-12-22T07:47:30.032Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369, upload-time = "2024-12-22T07:47:28.074Z" }, +] + +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] From 098520fce884b2397c675d270ff7b81f002d726b Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Wed, 24 Jun 2026 15:24:27 +0400 Subject: [PATCH 02/12] updated entry --- recommender/app/entry/kafka_consumer.py | 50 ++++++++++++++----------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/recommender/app/entry/kafka_consumer.py b/recommender/app/entry/kafka_consumer.py index 19148b1..dc69a71 100644 --- a/recommender/app/entry/kafka_consumer.py +++ b/recommender/app/entry/kafka_consumer.py @@ -6,12 +6,25 @@ from recommender.app.shared.product.utils import fetch_product_by_id from shared.db.session import get_session -from shared.db.repo import create_or_update_product, create_product, delete_product_by_id, get_product_by_id, record_interaction -from shared.kafka.utils import product_is_created_or_updated, product_is_deleted +from shared.db.repo import ( + get_product_by_id, + create_or_update_product, + create_product, + record_interaction, + delete_product_by_id, + ) +from shared.kafka.utils import ( + product_is_created_or_updated, + product_is_deleted, + is_interaction_event) from shared.config.settings import KAFKA_SERVER -def sync_products(): - consumer = KafkaConsumer("product_events", bootstrap_servers=KAFKA_SERVER) +def start_kafka_consumer(): + consumer = KafkaConsumer( + "product_events", + "interaction_events", + bootstrap_servers=KAFKA_SERVER) + for message in consumer: event = json.loads(message.value) with get_session() as session: @@ -23,22 +36,17 @@ def sync_products(): elif product_is_deleted(event): delete_product_by_id(session, event["data"]["product_id"]) -def process_interactions(): - consumer = KafkaConsumer("interaction_events", bootstrap_servers=KAFKA_SERVER) - for message in consumer: - event = json.loads(message.value) - with get_session() as session: - record_interaction(session, event) - product = get_product_by_id(session, event["data"]["product_id"]) - if not product: - try: - product = fetch_product_by_id(event["data"]["product_id"]) - create_product(session, product) - session.commit() - except requests.RequestException as e: - logger.error("Failed to fetch product {} for interaction event {}: {}", event['product_id'], event['type'], e) - session.rollback() + elif is_interaction_event(event): + record_interaction(session, event) + product = get_product_by_id(session, event["data"]["product_id"]) + if not product: + try: + product = fetch_product_by_id(event["data"]["product_id"]) + create_product(session, product) + session.commit() + except requests.RequestException as e: + logger.error("Failed to fetch product {} for interaction event {}: {}", event["data"]["product_id"], event["type"], e) + session.rollback() if __name__ == "__main__": - sync_products() - process_interactions() \ No newline at end of file + start_kafka_consumer() \ No newline at end of file From 6f2f5f18f1a494d98283a08bfb394790781ca26a Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Wed, 24 Jun 2026 15:28:59 +0400 Subject: [PATCH 03/12] added file manager interface and implementations for local disk and bucket --- recommender/app/shared/config/settings.py | 9 +++ .../app/shared/file_manager/__init__.py | 11 ++++ recommender/app/shared/file_manager/base.py | 32 ++++++++++ recommender/app/shared/file_manager/bucket.py | 60 +++++++++++++++++++ .../app/shared/file_manager/factory.py | 27 +++++++++ recommender/app/shared/file_manager/local.py | 41 +++++++++++++ 6 files changed, 180 insertions(+) create mode 100644 recommender/app/shared/file_manager/__init__.py create mode 100644 recommender/app/shared/file_manager/base.py create mode 100644 recommender/app/shared/file_manager/bucket.py create mode 100644 recommender/app/shared/file_manager/factory.py create mode 100644 recommender/app/shared/file_manager/local.py diff --git a/recommender/app/shared/config/settings.py b/recommender/app/shared/config/settings.py index 2f892b9..ad38ce4 100644 --- a/recommender/app/shared/config/settings.py +++ b/recommender/app/shared/config/settings.py @@ -7,3 +7,12 @@ KAFKA_SERVER = os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'kafka:9092') DATABASE_URL = os.getenv("DATABASE_URL") + +ARTIFACTS_BACKEND = os.getenv("ARTIFACTS_BACKEND", "local") +ARTIFACTS_LOCAL_PATH = os.getenv("ARTIFACTS_LOCAL_PATH", "artifacts") + +S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME") +S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL") +S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY") +S3_REGION = os.getenv("S3_REGION", "auto") diff --git a/recommender/app/shared/file_manager/__init__.py b/recommender/app/shared/file_manager/__init__.py new file mode 100644 index 0000000..63074c7 --- /dev/null +++ b/recommender/app/shared/file_manager/__init__.py @@ -0,0 +1,11 @@ +from shared.file_manager.base import FileManagerBase +from shared.file_manager.bucket import BucketFileManager +from shared.file_manager.factory import get_file_manager +from shared.file_manager.local import LocalFileManager + +__all__ = [ + "FileManagerBase", + "LocalFileManager", + "BucketFileManager", + "get_file_manager", +] diff --git a/recommender/app/shared/file_manager/base.py b/recommender/app/shared/file_manager/base.py new file mode 100644 index 0000000..337f32e --- /dev/null +++ b/recommender/app/shared/file_manager/base.py @@ -0,0 +1,32 @@ +from abc import ABC, abstractmethod + + +class FileManagerBase(ABC): + @abstractmethod + def upload_file(self, key: str, file: bytes) -> str: + """ + Upload a file under the given key. + Returns the key/path where the file was stored. + """ + pass + + @abstractmethod + def download_file(self, file_path: str) -> bytes: + """ + Download a file by key/path. + """ + pass + + @abstractmethod + def delete_file(self, file_path: str) -> None: + """ + Delete a file by key/path. + """ + pass + + @abstractmethod + def file_exists(self, file_path: str) -> bool: + """ + Return True if a file exists at the given key/path. + """ + pass diff --git a/recommender/app/shared/file_manager/bucket.py b/recommender/app/shared/file_manager/bucket.py new file mode 100644 index 0000000..052ba7b --- /dev/null +++ b/recommender/app/shared/file_manager/bucket.py @@ -0,0 +1,60 @@ +import boto3 +from botocore.exceptions import ClientError + +from shared.file_manager.base import FileManagerBase + + +class BucketFileManager(FileManagerBase): + def __init__( + self, + bucket_name: str, + *, + endpoint_url: str | None = None, + access_key_id: str | None = None, + secret_access_key: str | None = None, + region_name: str | None = None, + ): + if not bucket_name: + raise ValueError("bucket_name is required") + + self._bucket_name = bucket_name + session_kwargs: dict = {} + if access_key_id and secret_access_key: + session_kwargs["aws_access_key_id"] = access_key_id + session_kwargs["aws_secret_access_key"] = secret_access_key + if region_name: + session_kwargs["region_name"] = region_name + + session = boto3.session.Session(**session_kwargs) + client_kwargs: dict = {} + if endpoint_url: + client_kwargs["endpoint_url"] = endpoint_url + + self._client = session.client("s3", **client_kwargs) + + def upload_file(self, key: str, file: bytes) -> str: + self._client.put_object(Bucket=self._bucket_name, Key=key, Body=file) + return key + + def download_file(self, file_path: str) -> bytes: + try: + response = self._client.get_object(Bucket=self._bucket_name, Key=file_path) + except ClientError as exc: + error_code = exc.response.get("Error", {}).get("Code") + if error_code in {"NoSuchKey", "404", "NotFound"}: + raise FileNotFoundError(f"File not found: {file_path}") from exc + raise + return response["Body"].read() + + def delete_file(self, file_path: str) -> None: + self._client.delete_object(Bucket=self._bucket_name, Key=file_path) + + def file_exists(self, file_path: str) -> bool: + try: + self._client.head_object(Bucket=self._bucket_name, Key=file_path) + except ClientError as exc: + error_code = exc.response.get("Error", {}).get("Code") + if error_code in {"404", "NoSuchKey", "NotFound"}: + return False + raise + return True diff --git a/recommender/app/shared/file_manager/factory.py b/recommender/app/shared/file_manager/factory.py new file mode 100644 index 0000000..8e3ee71 --- /dev/null +++ b/recommender/app/shared/file_manager/factory.py @@ -0,0 +1,27 @@ +from shared.config.settings import ( + ARTIFACTS_BACKEND, + ARTIFACTS_LOCAL_PATH, + S3_ACCESS_KEY_ID, + S3_BUCKET_NAME, + S3_ENDPOINT_URL, + S3_REGION, + S3_SECRET_ACCESS_KEY, +) +from shared.file_manager.base import FileManagerBase +from shared.file_manager.bucket import BucketFileManager +from shared.file_manager.local import LocalFileManager + + +def get_file_manager() -> FileManagerBase: + if ARTIFACTS_BACKEND == "bucket": + if not S3_BUCKET_NAME: + raise ValueError("S3_BUCKET_NAME is required when ARTIFACTS_BACKEND=bucket") + return BucketFileManager( + bucket_name=S3_BUCKET_NAME, + endpoint_url=S3_ENDPOINT_URL, + access_key_id=S3_ACCESS_KEY_ID, + secret_access_key=S3_SECRET_ACCESS_KEY, + region_name=S3_REGION, + ) + + return LocalFileManager(base_path=ARTIFACTS_LOCAL_PATH) diff --git a/recommender/app/shared/file_manager/local.py b/recommender/app/shared/file_manager/local.py new file mode 100644 index 0000000..f3ab2d9 --- /dev/null +++ b/recommender/app/shared/file_manager/local.py @@ -0,0 +1,41 @@ +from pathlib import Path + +from shared.file_manager.base import FileManagerBase + + +class LocalFileManager(FileManagerBase): + def __init__(self, base_path: str | Path): + self._base_path = Path(base_path).resolve() + self._base_path.mkdir(parents=True, exist_ok=True) + + def _resolve_key(self, key: str) -> Path: + normalized = Path(key) + if normalized.is_absolute(): + raise ValueError(f"Absolute paths are not allowed: {key}") + + destination = (self._base_path / normalized).resolve() + if not str(destination).startswith(str(self._base_path)): + raise ValueError(f"Path traversal detected: {key}") + + return destination + + def upload_file(self, key: str, file: bytes) -> str: + destination = self._resolve_key(key) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(file) + return key + + def download_file(self, file_path: str) -> bytes: + destination = self._resolve_key(file_path) + if not destination.is_file(): + raise FileNotFoundError(f"File not found: {file_path}") + return destination.read_bytes() + + def delete_file(self, file_path: str) -> None: + destination = self._resolve_key(file_path) + if destination.is_file(): + destination.unlink() + + def file_exists(self, file_path: str) -> bool: + destination = self._resolve_key(file_path) + return destination.is_file() From 3324109e52f94147307042d7e490f6e8ecdc82d6 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Wed, 24 Jun 2026 15:29:25 +0400 Subject: [PATCH 04/12] new utility functions for kafka consumer --- recommender/app/shared/kafka/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/recommender/app/shared/kafka/utils.py b/recommender/app/shared/kafka/utils.py index b69e945..ae126f9 100644 --- a/recommender/app/shared/kafka/utils.py +++ b/recommender/app/shared/kafka/utils.py @@ -5,4 +5,7 @@ def product_is_created_or_updated(event: dict) -> bool: return event["type"] == EventType.PRODUCT_CREATED or event["type"] == EventType.PRODUCT_UPDATED def product_is_deleted(event: dict) -> bool: - return event["type"] == EventType.PRODUCT_DELETED \ No newline at end of file + return event["type"] == EventType.PRODUCT_DELETED + +def is_interaction_event(event: dict) -> bool: + return event["type"] == EventType.INTERACTION_CREATED or event["type"] == EventType.INTERACTION_UPDATED \ No newline at end of file From c2071e094889962917ab07afbd9a7d3ee274bb21 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Wed, 24 Jun 2026 22:22:22 +0400 Subject: [PATCH 05/12] new recommendation service --- recommender/app/entry/train.py | 0 recommender/app/recommendations/__init__.py | 5 + recommender/app/recommendations/load.py | 27 +++++ recommender/app/recommendations/rankers.py | 78 ++++++++++++++ recommender/app/recommendations/service.py | 54 ++++++++++ recommender/app/recommendations/storage.py | 112 +++++++++++++++++++ recommender/app/recommendations/train.py | 114 ++++++++++++++++++++ recommender/app/recommendations/weights.py | 9 ++ recommender/app/services/__init__.py | 0 recommender/app/services/recommender.py | 62 ----------- recommender/pyproject.toml | 4 +- recommender/uv.lock | 96 ++++++++++++++--- 12 files changed, 483 insertions(+), 78 deletions(-) delete mode 100644 recommender/app/entry/train.py create mode 100644 recommender/app/recommendations/__init__.py create mode 100644 recommender/app/recommendations/load.py create mode 100644 recommender/app/recommendations/rankers.py create mode 100644 recommender/app/recommendations/service.py create mode 100644 recommender/app/recommendations/storage.py create mode 100644 recommender/app/recommendations/train.py create mode 100644 recommender/app/recommendations/weights.py delete mode 100644 recommender/app/services/__init__.py delete mode 100644 recommender/app/services/recommender.py diff --git a/recommender/app/entry/train.py b/recommender/app/entry/train.py deleted file mode 100644 index e69de29..0000000 diff --git a/recommender/app/recommendations/__init__.py b/recommender/app/recommendations/__init__.py new file mode 100644 index 0000000..4e3d7d6 --- /dev/null +++ b/recommender/app/recommendations/__init__.py @@ -0,0 +1,5 @@ +from recommendations.load import load_service +from recommendations.service import RecommendationService +from recommendations.train import train_and_save + +__all__ = ["RecommendationService", "load_service", "train_and_save"] diff --git a/recommender/app/recommendations/load.py b/recommender/app/recommendations/load.py new file mode 100644 index 0000000..9ba38eb --- /dev/null +++ b/recommender/app/recommendations/load.py @@ -0,0 +1,27 @@ +from loguru import logger + +from recommendations.service import RecommendationService +from recommendations.storage import TrainedModel, has_model, load_model +from shared.db.repo import get_all_product_ids +from shared.db.session import get_session +from shared.file_manager.factory import get_file_manager + + +def load_service() -> RecommendationService: + files = get_file_manager() + + if not has_model(files): + logger.warning("No trained model found; using product catalog as fallback") + with get_session() as session: + product_ids = sorted(get_all_product_ids(session)) + model = TrainedModel( + als=None, + similar_products={}, + popular_products=product_ids, + metadata={"source": "fallback"}, + ) + return RecommendationService(model) + + model = load_model(files) + logger.info("Loaded model trained at {}", model.metadata.get("trained_at", "unknown")) + return RecommendationService(model) diff --git a/recommender/app/recommendations/rankers.py b/recommender/app/recommendations/rankers.py new file mode 100644 index 0000000..6ad45e7 --- /dev/null +++ b/recommender/app/recommendations/rankers.py @@ -0,0 +1,78 @@ +from collections import defaultdict + +import numpy as np + + +class UserRanker: + """Personalized recommendations for a known user (ALS).""" + + def __init__( + self, + user_ids: list[str], + product_ids: list[str], + user_factors: np.ndarray, + item_factors: np.ndarray, + ): + self._user_index = {user_id: i for i, user_id in enumerate(user_ids)} + self._product_ids = product_ids + self._user_factors = user_factors + self._item_factors = item_factors + + def knows_user(self, user_id: str) -> bool: + return user_id in self._user_index + + def recommend(self, user_id: str, *, exclude: set[str], limit: int) -> list[str]: + if not self.knows_user(user_id): + return [] + + scores = self._user_factors[self._user_index[user_id]] @ self._item_factors.T + ranked = np.argsort(scores)[::-1] + + results: list[str] = [] + for index in ranked: + product_id = self._product_ids[index] + if product_id in exclude: + continue + results.append(product_id) + if len(results) >= limit: + break + return results + + +class SimilarProductsRanker: + """Recommend products similar to a set of viewed products.""" + + def __init__(self, similar_products: dict[str, list[tuple[str, float]]]): + self._similar_products = similar_products + + def has_data_for(self, product_ids: list[str]) -> bool: + return any(pid in self._similar_products for pid in product_ids) + + def recommend(self, viewed_ids: list[str], *, exclude: set[str], limit: int) -> list[str]: + scores: dict[str, float] = defaultdict(float) + + for product_id in viewed_ids: + for neighbor_id, score in self._similar_products.get(product_id, []): + if neighbor_id in exclude or neighbor_id in viewed_ids: + continue + scores[neighbor_id] += score + + ranked = sorted(scores.items(), key=lambda item: item[1], reverse=True) + return [product_id for product_id, _ in ranked[:limit]] + + +class PopularProductsRanker: + """Fallback: return best-selling products.""" + + def __init__(self, product_ids: list[str]): + self._product_ids = product_ids + + def recommend(self, *, exclude: set[str], limit: int) -> list[str]: + results: list[str] = [] + for product_id in self._product_ids: + if product_id in exclude: + continue + results.append(product_id) + if len(results) >= limit: + break + return results diff --git a/recommender/app/recommendations/service.py b/recommender/app/recommendations/service.py new file mode 100644 index 0000000..a385af8 --- /dev/null +++ b/recommender/app/recommendations/service.py @@ -0,0 +1,54 @@ +from recommendations.rankers import PopularProductsRanker, SimilarProductsRanker, UserRanker +from recommendations.storage import TrainedModel +from shared.db.repo import get_interacted_ids_for_user +from shared.db.session import get_session + + +class RecommendationService: + """Main API: recommend products for a user or for a browsing session.""" + + def __init__(self, model: TrainedModel): + self._user_ranker = ( + UserRanker( + model.als.user_ids, + model.als.product_ids, + model.als.user_factors, + model.als.item_factors, + ) + if model.als + else None + ) + self._similar_ranker = SimilarProductsRanker(model.similar_products) + self._popular_ranker = PopularProductsRanker(model.popular_products) + + def for_user(self, user_id: str, skip: int = 0, take: int = 5) -> list[str]: + exclude = _products_user_already_saw(user_id) + limit = skip + take + + if self._user_ranker and self._user_ranker.knows_user(user_id): + ids = self._user_ranker.recommend(user_id, exclude=exclude, limit=limit) + else: + ids = self._popular_ranker.recommend(exclude=exclude, limit=limit) + + return ids[skip : skip + take] + + def for_viewed_products( + self, + product_ids: list[str], + skip: int = 0, + take: int = 5, + ) -> list[str]: + exclude = set(product_ids) + limit = skip + take + + if product_ids and self._similar_ranker.has_data_for(product_ids): + ids = self._similar_ranker.recommend(product_ids, exclude=exclude, limit=limit) + else: + ids = self._popular_ranker.recommend(exclude=exclude, limit=limit) + + return ids[skip : skip + take] + + +def _products_user_already_saw(user_id: str) -> set[str]: + with get_session() as session: + return get_interacted_ids_for_user(session, user_id) diff --git a/recommender/app/recommendations/storage.py b/recommender/app/recommendations/storage.py new file mode 100644 index 0000000..4bc56db --- /dev/null +++ b/recommender/app/recommendations/storage.py @@ -0,0 +1,112 @@ +import io +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone + +import numpy as np + +from recommendations.weights import ( + ALS_FILE, + METADATA_FILE, + POPULAR_PRODUCTS_FILE, + SIMILAR_PRODUCTS_FILE, +) +from shared.file_manager.base import FileManagerBase + + +@dataclass +class ALSData: + user_ids: list[str] + product_ids: list[str] + user_factors: np.ndarray + item_factors: np.ndarray + + +@dataclass +class TrainedModel: + als: ALSData | None + similar_products: dict[str, list[tuple[str, float]]] + popular_products: list[str] + metadata: dict = field(default_factory=dict) + + +def save_model(files: FileManagerBase, model: TrainedModel) -> None: + if model.als is not None: + buffer = io.BytesIO() + np.savez( + buffer, + user_ids=np.array(model.als.user_ids, dtype=object), + product_ids=np.array(model.als.product_ids, dtype=object), + user_factors=model.als.user_factors, + item_factors=model.als.item_factors, + ) + files.upload_file(ALS_FILE, buffer.getvalue()) + + buffer = io.BytesIO() + product_ids = np.array(list(model.similar_products.keys()), dtype=object) + neighbor_lists = np.array( + [json.dumps(neighbors) for neighbors in model.similar_products.values()], + dtype=object, + ) + np.savez(buffer, product_ids=product_ids, neighbor_lists=neighbor_lists) + files.upload_file(SIMILAR_PRODUCTS_FILE, buffer.getvalue()) + + files.upload_file( + POPULAR_PRODUCTS_FILE, + json.dumps(model.popular_products).encode("utf-8"), + ) + + metadata = {**model.metadata, "saved_at": datetime.now(timezone.utc).isoformat()} + files.upload_file(METADATA_FILE, json.dumps(metadata, indent=2).encode("utf-8")) + + +def has_model(files: FileManagerBase) -> bool: + return files.file_exists(POPULAR_PRODUCTS_FILE) + + +def load_model(files: FileManagerBase) -> TrainedModel: + als = _load_als(files) if files.file_exists(ALS_FILE) else None + return TrainedModel( + als=als, + similar_products=_load_similar_products(files), + popular_products=_load_popular_products(files), + metadata=_load_metadata(files), + ) + + +def _load_als(files: FileManagerBase) -> ALSData: + payload = np.load(io.BytesIO(files.download_file(ALS_FILE)), allow_pickle=True) + return ALSData( + user_ids=payload["user_ids"].tolist(), + product_ids=payload["product_ids"].tolist(), + user_factors=payload["user_factors"], + item_factors=payload["item_factors"], + ) + + +def _load_similar_products(files: FileManagerBase) -> dict[str, list[tuple[str, float]]]: + if not files.file_exists(SIMILAR_PRODUCTS_FILE): + return {} + + payload = np.load( + io.BytesIO(files.download_file(SIMILAR_PRODUCTS_FILE)), + allow_pickle=True, + ) + product_ids = payload["product_ids"].tolist() + neighbor_lists = payload["neighbor_lists"].tolist() + return { + product_id: [tuple(neighbor) for neighbor in json.loads(neighbors)] + for product_id, neighbors in zip(product_ids, neighbor_lists) + } + + +def _load_popular_products(files: FileManagerBase) -> list[str]: + if not files.file_exists(POPULAR_PRODUCTS_FILE): + return [] + return json.loads(files.download_file(POPULAR_PRODUCTS_FILE).decode("utf-8")) + + +def _load_metadata(files: FileManagerBase) -> dict: + if not files.file_exists(METADATA_FILE): + return {} + return json.loads(files.download_file(METADATA_FILE).decode("utf-8")) diff --git a/recommender/app/recommendations/train.py b/recommender/app/recommendations/train.py new file mode 100644 index 0000000..e9cdfef --- /dev/null +++ b/recommender/app/recommendations/train.py @@ -0,0 +1,114 @@ +from collections import defaultdict +from datetime import datetime, timezone + +import implicit +import pandas as pd +from loguru import logger +from scipy.sparse import coo_matrix + +from recommendations.storage import ALSData, TrainedModel, save_model +from shared.db.repo import fetch_interactions, get_all_product_ids +from shared.db.session import get_session +from shared.file_manager.factory import get_file_manager + + +def train_and_save() -> None: + interactions = fetch_interactions() + model = _train(interactions) + + if not model.popular_products: + with get_session() as session: + model.popular_products = sorted(get_all_product_ids(session)) + + save_model(get_file_manager(), model) + logger.info("Model saved: {}", model.metadata.get("metrics", {})) + + +def _train(interactions: pd.DataFrame) -> TrainedModel: + if interactions.empty: + logger.warning("No interactions found; saving empty model") + return TrainedModel( + als=None, + similar_products={}, + popular_products=[], + metadata=_metadata(interactions, {}), + ) + + als = _train_als(interactions) + similar_products = _train_similar_products(interactions) + popular_products = _train_popular_products(interactions) + metrics = { + "interactions": len(interactions), + "users": int(interactions["user_id"].nunique()), + "products": int(interactions["product_id"].nunique()), + "purchases": int((interactions["interaction_type"] == "purchase").sum()), + } + + return TrainedModel( + als=als, + similar_products=similar_products, + popular_products=popular_products, + metadata=_metadata(interactions, metrics), + ) + + +def _train_als(interactions: pd.DataFrame) -> ALSData | None: + aggregated = interactions.groupby(["user_id", "product_id"], as_index=False)["confidence"].sum() + if aggregated.empty: + return None + + user_ids = aggregated["user_id"].unique().tolist() + product_ids = aggregated["product_id"].unique().tolist() + user_index = {user_id: i for i, user_id in enumerate(user_ids)} + product_index = {product_id: i for i, product_id in enumerate(product_ids)} + + matrix = coo_matrix( + ( + aggregated["confidence"].astype(float).tolist(), + ( + [user_index[u] for u in aggregated["user_id"]], + [product_index[p] for p in aggregated["product_id"]], + ), + ), + shape=(len(user_ids), len(product_ids)), + ).tocsr() + + model = implicit.als.AlternatingLeastSquares(factors=50, random_state=42, iterations=15) + model.fit(matrix) + + return ALSData(user_ids, product_ids, model.user_factors, model.item_factors) + + +def _train_similar_products( + interactions: pd.DataFrame, + top_k: int = 50, +) -> dict[str, list[tuple[str, float]]]: + pairs: dict[str, dict[str, float]] = defaultdict(lambda: defaultdict(float)) + + for _, group in interactions.groupby("user_id"): + items = group.drop_duplicates("product_id")[["product_id", "confidence"]].to_dict("records") + for i, left in enumerate(items): + for right in items[i + 1 :]: + weight = min(left["confidence"], right["confidence"]) + pairs[left["product_id"]][right["product_id"]] += weight + pairs[right["product_id"]][left["product_id"]] += weight + + return { + product_id: sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k] + for product_id, scores in pairs.items() + } + + +def _train_popular_products(interactions: pd.DataFrame) -> list[str]: + purchases = interactions[interactions["interaction_type"] == "purchase"] + source = purchases if not purchases.empty else interactions + return ( + source.groupby("product_id")["confidence"] + .sum() + .sort_values(ascending=False) + .index.tolist() + ) + + +def _metadata(interactions: pd.DataFrame, metrics: dict) -> dict: + return {"trained_at": datetime.now(timezone.utc).isoformat(), "metrics": metrics} diff --git a/recommender/app/recommendations/weights.py b/recommender/app/recommendations/weights.py new file mode 100644 index 0000000..b93ed5f --- /dev/null +++ b/recommender/app/recommendations/weights.py @@ -0,0 +1,9 @@ +# How strongly each interaction type influences training. +PURCHASE_WEIGHT = 5.0 +VIEW_WEIGHT = 1.0 + +# Saved model file paths (relative to artifacts root). +ALS_FILE = "current/als_model.npz" +SIMILAR_PRODUCTS_FILE = "current/similar_products.npz" +POPULAR_PRODUCTS_FILE = "current/popular_products.json" +METADATA_FILE = "current/metadata.json" diff --git a/recommender/app/services/__init__.py b/recommender/app/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/recommender/app/services/recommender.py b/recommender/app/services/recommender.py deleted file mode 100644 index dcf69bb..0000000 --- a/recommender/app/services/recommender.py +++ /dev/null @@ -1,62 +0,0 @@ -from surprise import SVD, Dataset, Reader -from app.db.session import ReplicaSession -from shared.repo import ( - fetch_interactions, - get_all_product_ids, - get_interacted_ids_for_user, - get_interacted_ids_for_viewed -) - -class Recommender: - def __init__(self): - self.model = SVD(n_factors=50, random_state=42) - self.trainset = None - self.product_ids = set() - - def train(self): - df = fetch_interactions() - self.product_ids = set(df["product_id"].unique()) - reader = Reader(rating_scale=(1, 3)) - data = Dataset.load_from_df(df[["user_id", "product_id", "rating"]], reader) - self.trainset = data.build_full_trainset() - self.model.fit(self.trainset) - - def _predict_and_sort(self, user_id: str, candidates: list[str]) -> list: - """ - Predict ratings for each candidate and return a list - of (product_id, est_prediction), sorted descending by est_prediction. - """ - predictions = [self.model.predict(user_id, pid) for pid in candidates] - sorted_predictions = sorted(predictions, key=lambda x: x.est, reverse=True) - return sorted_predictions - - def recommend_on_user_id(self, user_id: str, skip: int = 0, take: int = 5) -> list[str]: - """Recommend based on user interactions.""" - with ReplicaSession() as session: - all_product_ids = get_all_product_ids(session) - interacted_ids = get_interacted_ids_for_user(session, user_id) - - candidates = [pid for pid in all_product_ids if pid not in interacted_ids] - - sorted_predictions = self._predict_and_sort(user_id, candidates) - - sliced = sorted_predictions[skip : skip + take] - - return [pred.iid for pred in sliced] - - def recommend_on_viewed_ids(self, viewed_ids: list[str], skip: int = 0, take: int = 5) -> list[str]: - """Recommend based on a set of viewed product IDs.""" - with ReplicaSession() as session: - all_product_ids = get_all_product_ids(session) - interacted_ids = get_interacted_ids_for_viewed(session, viewed_ids) - - candidates = [pid for pid in all_product_ids if pid not in interacted_ids] - - sorted_predictions = self._predict_and_sort("anonymous_user", candidates) - - sliced = sorted_predictions[skip : skip + take] - - return [pred.iid for pred in sliced] - - -recommender = Recommender() # Singleton for simplicity diff --git a/recommender/pyproject.toml b/recommender/pyproject.toml index 29e5e17..f2e3c5c 100644 --- a/recommender/pyproject.toml +++ b/recommender/pyproject.toml @@ -5,12 +5,14 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.11,<3.12" dependencies = [ + "boto3>=1.35.0", "certifi==2025.1.31", "charset-normalizer==3.4.1", "greenlet==3.1.1", "grpcio==1.71.0", "grpcio-tools==1.71.0", "idna==3.10", + "implicit>=0.7.2", "joblib==1.4.2", "kafka-python==2.1.4", "loguru>=0.7.3", @@ -22,11 +24,9 @@ dependencies = [ "python-dotenv==1.1.0", "pytz==2025.2", "requests==2.32.3", - "scikit-surprise==1.1.4", "scipy==1.15.2", "six==1.17.0", "sqlalchemy==2.0.40", - "surprise==0.1", "typing-extensions==4.13.0", "tzdata==2025.2", "urllib3==2.3.0", diff --git a/recommender/uv.lock b/recommender/uv.lock index 16c2da8..d551122 100644 --- a/recommender/uv.lock +++ b/recommender/uv.lock @@ -2,6 +2,34 @@ version = 1 revision = 3 requires-python = "==3.11.*" +[[package]] +name = "boto3" +version = "1.43.36" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/9f/897287e955db0f50b12fd69ef45956e4fd2c7ddb48c736872f7ea2314443/boto3-1.43.36.tar.gz", hash = "sha256:587d7ee92a12e440ad12b0e7f11f3358f0c4d65b19f64726efc94aaf194aff28", size = 112690, upload-time = "2026-06-23T02:47:14.561Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/f1/274303f52483ecf199eae6f8d9b6f5951670397ee4d72c06cfd4eb644612/boto3-1.43.36-py3-none-any.whl", hash = "sha256:42942dde254673abcbc9e6e60017c88341a4f49d99d24e1f2e290fb38138c26f", size = 140031, upload-time = "2026-06-23T02:47:13.178Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.36" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/37/da9e7f6ca73ac73afd7f0bb7f238aa5daba35c081e98d7f48a7c399599c0/botocore-1.43.36.tar.gz", hash = "sha256:4cae47d1b2d426316b85a0087d9e69e048f13bc003b5177d74639fe9dfd28205", size = 15625488, upload-time = "2026-06-23T02:47:03.192Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/19/934f81592527a3f7f9b943c893e334c721a4644948642bc33885d584e9ec/botocore-1.43.36-py3-none-any.whl", hash = "sha256:3c65fdc39ed01d8dfde1e961b34038aed03c459f8ddf80717a12ac006475e49d", size = 15313630, upload-time = "2026-06-23T02:46:59.327Z" }, +] + [[package]] name = "certifi" version = "2025.1.31" @@ -109,6 +137,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "implicit" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/24/bdaa4be7fcfb568d63e0ddff948c486515c6b728055ee060f3f131803476/implicit-0.7.3.tar.gz", hash = "sha256:9571a52d70b3667e99a6e6aae6681526dd3286c765cf0ea10e1073d068a98057", size = 443981, upload-time = "2026-05-14T04:36:05.331Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/e4/5d033371a1aca288bba68ed65a74307cb1ec3f6bb03339b3df6b8c050d37/implicit-0.7.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a059c926f23813d288b0ad68e436c84ed24c7049a26dc1b4cced70c9b8d1cf40", size = 1248365, upload-time = "2026-05-08T18:37:09.808Z" }, + { url = "https://files.pythonhosted.org/packages/06/7f/16e15852849f99ccbaf9f4e0f8ccbab08f20ba37b09a60ad5b84e8d2e415/implicit-0.7.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7a6035c0fbe89eef9d1789367c8097eb3145af3cfffd0dc0e92b2570ab67a6fc", size = 671535, upload-time = "2026-05-08T18:37:11.632Z" }, + { url = "https://files.pythonhosted.org/packages/e7/30/0f855b8b74dfd441e07b26db2ec230d110fdb9276cef75b2911d29606d32/implicit-0.7.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77b9b93ea7840be5367c0b49ff90a371d5f5e86bc4743776ee491dfb62854c2b", size = 641824, upload-time = "2026-05-08T18:37:12.851Z" }, + { url = "https://files.pythonhosted.org/packages/9c/e2/f4102085c133c0e824976a55d3b59f244029aad50e5305fa4c4d38654bc2/implicit-0.7.3-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:e4d56a2d9db92e9cf1d0c9af740581d40423c216b0e207d7d72d59a0ca5519c2", size = 874812, upload-time = "2026-05-08T18:37:13.998Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/9d0e5a300bd20847202e071030147ec9db0bcd9e3e2d2ba44ba0153836b8/implicit-0.7.3-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:bad1d7263ce295674562cf74482a83ae6847579579f8e2f5e70da840386c2c14", size = 7672415, upload-time = "2026-05-08T18:37:15.311Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/a0cb666d80520bf6a5be4370cd7f572b7a4057fc7a3779249f10d1dec840/implicit-0.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:02cf46507f62514415374511cc79b2cba8021d9d432abfca4407276d4d41bc4d", size = 692921, upload-time = "2026-05-08T18:37:17.053Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "joblib" version = "1.4.2" @@ -246,12 +303,14 @@ name = "recommender" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "boto3" }, { name = "certifi" }, { name = "charset-normalizer" }, { name = "greenlet" }, { name = "grpcio" }, { name = "grpcio-tools" }, { name = "idna" }, + { name = "implicit" }, { name = "joblib" }, { name = "kafka-python" }, { name = "loguru" }, @@ -263,11 +322,9 @@ dependencies = [ { name = "python-dotenv" }, { name = "pytz" }, { name = "requests" }, - { name = "scikit-surprise" }, { name = "scipy" }, { name = "six" }, { name = "sqlalchemy" }, - { name = "surprise" }, { name = "typing-extensions" }, { name = "tzdata" }, { name = "urllib3" }, @@ -275,12 +332,14 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "boto3", specifier = ">=1.35.0" }, { name = "certifi", specifier = "==2025.1.31" }, { name = "charset-normalizer", specifier = "==3.4.1" }, { name = "greenlet", specifier = "==3.1.1" }, { name = "grpcio", specifier = "==1.71.0" }, { name = "grpcio-tools", specifier = "==1.71.0" }, { name = "idna", specifier = "==3.10" }, + { name = "implicit", specifier = ">=0.7.2" }, { name = "joblib", specifier = "==1.4.2" }, { name = "kafka-python", specifier = "==2.1.4" }, { name = "loguru", specifier = ">=0.7.3" }, @@ -292,11 +351,9 @@ requires-dist = [ { name = "python-dotenv", specifier = "==1.1.0" }, { name = "pytz", specifier = "==2025.2" }, { name = "requests", specifier = "==2.32.3" }, - { name = "scikit-surprise", specifier = "==1.1.4" }, { name = "scipy", specifier = "==1.15.2" }, { name = "six", specifier = "==1.17.0" }, { name = "sqlalchemy", specifier = "==2.0.40" }, - { name = "surprise", specifier = "==0.1" }, { name = "typing-extensions", specifier = "==4.13.0" }, { name = "tzdata", specifier = "==2025.2" }, { name = "urllib3", specifier = "==2.3.0" }, @@ -318,15 +375,16 @@ wheels = [ ] [[package]] -name = "scikit-surprise" -version = "1.1.4" +name = "s3transfer" +version = "0.19.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "joblib" }, - { name = "numpy" }, - { name = "scipy" }, + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/94/dcdaeb1713cab9c84def276cfac7388b17c7d9855bbcfe88d77e4dbafd44/s3transfer-0.19.0.tar.gz", hash = "sha256:ce436931687addc4c1712d52d40b32f53e88315723f107ffa20ba82b05a0f685", size = 165171, upload-time = "2026-06-16T19:44:51.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/5f/4c174edad94f82de888ac00a5ddd8d07b35609b6c94f0bdf4d74af57703e/s3transfer-0.19.0-py3-none-any.whl", hash = "sha256:777cc2415536f1debadb5c2ef7779275d0fc0fe0e042411cdd6caebeb2685262", size = 90101, upload-time = "2026-06-16T19:44:50.439Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d9/8d/40ac32e703f3808159f9e2b33760cfbd6224cc7783eb663091eddc9581c2/scikit_surprise-1.1.4.tar.gz", hash = "sha256:130c45feaee9de4b8cba0aff413ad9b51b2d5c5c90d41aee4759e00059913752", size = 154434, upload-time = "2024-05-19T14:25:59.527Z" } [[package]] name = "scipy" @@ -388,14 +446,24 @@ wheels = [ ] [[package]] -name = "surprise" -version = "0.1" +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + +[[package]] +name = "tqdm" +version = "4.68.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "scikit-surprise" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/87/d7/0535a28b1f5f24f6612fb3ff1e89fb1a8d160fee0f976e0aa6803862134b/tqdm-4.68.3.tar.gz", hash = "sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482", size = 170596, upload-time = "2026-06-17T07:36:52.105Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/61/de/e5cba8682201fcf9c3719a6fdda95693468ed061945493dea2dd37c5618b/surprise-0.1-py2.py3-none-any.whl", hash = "sha256:0b5ad9b2e1c07ad74b93ece2e74beca83a05e32c7bd2ff2dcff7261e4cad0305", size = 1821, upload-time = "2017-01-14T13:39:47.865Z" }, + { url = "https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl", hash = "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03", size = 78337, upload-time = "2026-06-17T07:36:50.132Z" }, ] [[package]] From 440cc832e534066041db80263cc6f33a371433c9 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Wed, 24 Jun 2026 22:24:11 +0400 Subject: [PATCH 06/12] updated entries --- recommender/app/entry/grpc_server.py | 96 ++++++++++--------------- recommender/app/entry/kafka_consumer.py | 93 +++++++++++++++--------- recommender/app/main.py | 28 ++++++++ 3 files changed, 124 insertions(+), 93 deletions(-) create mode 100644 recommender/app/main.py diff --git a/recommender/app/entry/grpc_server.py b/recommender/app/entry/grpc_server.py index 50974e3..1af1cfa 100644 --- a/recommender/app/entry/grpc_server.py +++ b/recommender/app/entry/grpc_server.py @@ -1,82 +1,62 @@ import grpc from concurrent import futures +from loguru import logger from generated.pb import recommender_pb2, recommender_pb2_grpc -from app.services.recommender import recommender -from recommender.app.shared.db.repo import get_products_by_ids -from recommender.app.shared.db.session import get_session -from recommender.app.shared.product.utils import fetch_product_by_id - - -def _handle_exception(context, error_message): - """Helper method to set gRPC status code and details.""" - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details(error_message) +from recommendations.load import load_service +from shared.db.repo import get_products_by_ids +from shared.db.session import get_session class RecommenderServiceServicer(recommender_pb2_grpc.RecommenderServiceServicer): + def __init__(self, service): + self._service = service def GetRecommendations(self, request, context): - user_id = request.user_id - skip = request.skip or 0 # default to 0 if not set - take = request.take or 5 # default to 5 if not set - try: - # Get recommended product IDs - recommended_product_ids = recommender.recommend_on_user_id( - user_id=user_id, - skip=skip, - take=take + product_ids = self._service.for_user( + user_id=request.user_id, + skip=request.skip or 0, + take=request.take or 5, ) - - # Fetch product details as gRPC objects - recommended_products = [] - with get_session() as session: - products = get_products_by_ids(session, recommended_product_ids) - recommended_products = [product.to_grpc_model() for product in products] - - return recommender_pb2.RecommendationResponse( - recommended_products=recommended_products - ) - - except Exception as e: - _handle_exception(context, f"Failed to get recommendations: {str(e)}") + return _to_response(product_ids) + except Exception as exc: + logger.exception("Failed to get recommendations for user {}", request.user_id) + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(exc)) return recommender_pb2.RecommendationResponse() def GetRecommendationsBasedOnViewed(self, request, context): - viewed_product_ids = request.viewed_product_ids - skip = request.skip or 0 - take = request.take or 5 - try: - recommended_product_ids = recommender.recommend_on_viewed_ids( - viewed_ids=viewed_product_ids, - skip=skip, - take=take + product_ids = self._service.for_viewed_products( + product_ids=list(request.ids), + skip=request.skip or 0, + take=request.take or 5, ) + return _to_response(product_ids) + except Exception as exc: + logger.exception("Failed to get viewed-based recommendations") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(exc)) + return recommender_pb2.RecommendationResponse() - # Fetch product details as gRPC objects - recommended_products = [] - with get_session() as session: - products = get_products_by_ids(session, recommended_product_ids) - recommended_products = [product.to_grpc_model() for product in products] - return recommender_pb2.RecommendationResponse( - recommended_products=recommended_products - ) - except Exception as e: - _handle_exception(context, f"Failed to get recommendations: {str(e)}") - return recommender_pb2.RecommendationResponse() +def _to_response(product_ids: list[str]) -> recommender_pb2.RecommendationResponse: + with get_session() as session: + products = get_products_by_ids(session, product_ids) + return recommender_pb2.RecommendationResponse( + recommended_products=[p.to_grpc_model() for p in products] + ) + -def serve(): +def serve() -> None: + service = load_service() server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) recommender_pb2_grpc.add_RecommenderServiceServicer_to_server( - RecommenderServiceServicer(), server + RecommenderServiceServicer(service), + server, ) - server.add_insecure_port('[::]:50051') - print("gRPC server started on port 50051") + server.add_insecure_port("[::]:50051") + logger.info("gRPC server started on port 50051") server.start() server.wait_for_termination() - -if __name__ == '__main__': - serve() \ No newline at end of file diff --git a/recommender/app/entry/kafka_consumer.py b/recommender/app/entry/kafka_consumer.py index dc69a71..daa5294 100644 --- a/recommender/app/entry/kafka_consumer.py +++ b/recommender/app/entry/kafka_consumer.py @@ -4,49 +4,72 @@ import requests from loguru import logger -from recommender.app.shared.product.utils import fetch_product_by_id -from shared.db.session import get_session +from shared.config.settings import ( + INTERACTION_EVENTS_TOPIC, + KAFKA_SERVER, + PRODUCT_EVENTS_TOPIC, +) from shared.db.repo import ( - get_product_by_id, create_or_update_product, create_product, - record_interaction, delete_product_by_id, - ) -from shared.kafka.utils import ( - product_is_created_or_updated, - product_is_deleted, - is_interaction_event) -from shared.config.settings import KAFKA_SERVER + get_product_by_id, + record_interaction, +) +from shared.db.session import get_session +from shared.kafka.utils import product_is_created_or_updated, product_is_deleted +from shared.product.utils import fetch_product_by_id + -def start_kafka_consumer(): +def start_kafka_consumer() -> None: consumer = KafkaConsumer( - "product_events", - "interaction_events", - bootstrap_servers=KAFKA_SERVER) + PRODUCT_EVENTS_TOPIC, + INTERACTION_EVENTS_TOPIC, + bootstrap_servers=KAFKA_SERVER, + group_id="recommender-sync", + ) for message in consumer: event = json.loads(message.value) - with get_session() as session: - if product_is_created_or_updated(event): - product_data = event["data"] - logger.info("Processing product event {} for product ID: {}", event['type'], product_data['product_id']) - create_or_update_product(session, product_data) - - elif product_is_deleted(event): - delete_product_by_id(session, event["data"]["product_id"]) - - elif is_interaction_event(event): - record_interaction(session, event) - product = get_product_by_id(session, event["data"]["product_id"]) - if not product: - try: - product = fetch_product_by_id(event["data"]["product_id"]) - create_product(session, product) - session.commit() - except requests.RequestException as e: - logger.error("Failed to fetch product {} for interaction event {}: {}", event["data"]["product_id"], event["type"], e) - session.rollback() + + if message.topic == PRODUCT_EVENTS_TOPIC: + _handle_product_event(event) + elif message.topic == INTERACTION_EVENTS_TOPIC: + _handle_interaction_event(event) + + +def _handle_product_event(event: dict) -> None: + with get_session() as session: + if product_is_created_or_updated(event): + product_data = event["data"] + logger.info( + "Processing product event {} for product ID: {}", + event["type"], + product_data["product_id"], + ) + create_or_update_product(session, product_data) + elif product_is_deleted(event): + delete_product_by_id(session, event["data"]["product_id"]) + + +def _handle_interaction_event(event: dict) -> None: + with get_session() as session: + record_interaction(session, event) + product = get_product_by_id(session, event["data"]["product_id"]) + if not product: + try: + product_data = fetch_product_by_id(event["data"]["product_id"]) + create_product(session, product_data) + session.commit() + except requests.RequestException as exc: + logger.error( + "Failed to fetch product {} for interaction event {}: {}", + event["data"]["product_id"], + event["type"], + exc, + ) + session.rollback() + if __name__ == "__main__": - start_kafka_consumer() \ No newline at end of file + start_kafka_consumer() diff --git a/recommender/app/main.py b/recommender/app/main.py new file mode 100644 index 0000000..3be0e66 --- /dev/null +++ b/recommender/app/main.py @@ -0,0 +1,28 @@ +import sys + +from loguru import logger + +from entry.grpc_server import serve +from entry.kafka_consumer import start_kafka_consumer +from recommendations.train import train_and_save + + +def main() -> None: + if len(sys.argv) < 2: + logger.error("Usage: python main.py [consumer|server|train]") + sys.exit(1) + + command = sys.argv[1] + if command == "consumer": + start_kafka_consumer() + elif command == "server": + serve() + elif command == "train": + train_and_save() + else: + logger.error("Unknown command: {}", command) + sys.exit(1) + + +if __name__ == "__main__": + main() From 22717db0720b599a789e788bafa75a06ad471f93 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Wed, 24 Jun 2026 22:25:01 +0400 Subject: [PATCH 07/12] updated utility functions --- recommender/app/shared/config/settings.py | 5 +- recommender/app/shared/db/models.py | 7 -- recommender/app/shared/db/repo.py | 105 ++++++++++++---------- recommender/app/shared/kafka/utils.py | 11 ++- recommender/app/shared/product/utils.py | 16 +++- 5 files changed, 78 insertions(+), 66 deletions(-) diff --git a/recommender/app/shared/config/settings.py b/recommender/app/shared/config/settings.py index ad38ce4..432dcee 100644 --- a/recommender/app/shared/config/settings.py +++ b/recommender/app/shared/config/settings.py @@ -4,7 +4,10 @@ load_dotenv() PRODUCT_API = os.getenv("PRODUCT_API") -KAFKA_SERVER = os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'kafka:9092') +KAFKA_SERVER = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092") + +PRODUCT_EVENTS_TOPIC = os.getenv("PRODUCT_EVENTS_TOPIC", "product_events") +INTERACTION_EVENTS_TOPIC = os.getenv("INTERACTION_EVENTS_TOPIC", "interaction_events") DATABASE_URL = os.getenv("DATABASE_URL") diff --git a/recommender/app/shared/db/models.py b/recommender/app/shared/db/models.py index da8dfe6..51c8070 100644 --- a/recommender/app/shared/db/models.py +++ b/recommender/app/shared/db/models.py @@ -34,10 +34,3 @@ class Interaction(Base): product: Mapped["Product"] = relationship( "Product", back_populates="interactions") - - def to_grpc_model(self) -> recommender_pb2.Interaction: - return recommender_pb2.Interaction( - user_id=self.user_id, - product_id=self.product_id, - interaction_type=self.interaction_type - ) \ No newline at end of file diff --git a/recommender/app/shared/db/repo.py b/recommender/app/shared/db/repo.py index 4cf0dc9..ea600bf 100644 --- a/recommender/app/shared/db/repo.py +++ b/recommender/app/shared/db/repo.py @@ -1,85 +1,92 @@ import pandas as pd -from shared.db.session import ReplicaSession +from recommendations.weights import PURCHASE_WEIGHT, VIEW_WEIGHT from shared.db.models import Interaction, Product +from shared.db.session import ReplicaSession, replica_engine + def fetch_interactions() -> pd.DataFrame: with ReplicaSession() as session: - interactions = session.query(Interaction).all() - data = [ - { - "user_id": i.user_id, - "product_id": i.product_id, - "rating": 3.0 if i.interaction_type == "purchase" else 1.0 - } - for i in interactions - ] - return pd.DataFrame(data) - -def get_all_product_ids(session): - """Fetch all product IDs from the database.""" + query = session.query(Interaction) + df = pd.read_sql(query.statement, con=replica_engine) + + df["confidence"] = df["interaction_type"].apply( + lambda t: PURCHASE_WEIGHT if t == "purchase" else VIEW_WEIGHT + ) + return df + + +def get_all_product_ids(session) -> set[str]: return {p.id for p in session.query(Product.id).all()} + def get_product_by_id(session, product_id: str): return session.query(Product).filter(Product.id == product_id).first() + def get_products_by_ids(session, product_ids: list[str]): - return session.query(Product).filter(Product.id.in_(product_ids)).all() + if not product_ids: + return [] + products = session.query(Product).filter(Product.id.in_(product_ids)).all() + by_id = {product.id: product for product in products} + return [by_id[product_id] for product_id in product_ids if product_id in by_id] -def get_interacted_ids_for_user(session, user_id: str): - """Get set of product IDs that the user has interacted with.""" - return { - i.product_id - for i in session.query(Interaction.product_id) - .filter(Interaction.user_id == user_id) - .all() - } -def get_interacted_ids_for_viewed(session, viewed_ids: list[str]): - """Get set of product IDs among 'viewed_ids' that have existing interactions.""" +def get_interacted_ids_for_user(session, user_id: str) -> set[str]: return { i.product_id for i in session.query(Interaction.product_id) - .filter(Interaction.product_id.in_(viewed_ids)) - .all() + .filter(Interaction.user_id == user_id) + .all() } -def record_interaction(session, event: dict): - interaction = Interaction( - user_id=event["data"]["user_id"], - product_id=event["data"]["product_id"], - interaction_type=event["type"] + +def record_interaction(session, event: dict) -> None: + data = event["data"] + user_id = data.get("user_id", data.get("accountID", data.get("account_id"))) + if user_id is None: + raise ValueError(f"Interaction event missing user identifier: {event}") + + session.add( + Interaction( + user_id=str(user_id), + product_id=data["product_id"], + interaction_type=event["type"], + ) ) - session.add(interaction) - -def create_product(session, product_data: dict): - product = Product( - id=product_data["product_id"], - name=product_data["name"], - description=product_data["description"], - price=product_data["price"], - account_id=product_data["accountID"] + session.commit() + + +def create_product(session, product_data: dict) -> None: + session.add( + Product( + id=product_data["product_id"], + name=product_data["name"], + description=product_data["description"], + price=product_data["price"], + account_id=product_data["accountID"], + ) ) - session.add(product) -def update_product(session, product_data: dict): + +def update_product(session, product_data: dict) -> None: product = get_product_by_id(session, product_data["product_id"]) product.name = product_data["name"] product.description = product_data["description"] product.price = product_data["price"] - product.account_id = product_data["account_id"] + product.account_id = product_data["accountID"] -def create_or_update_product(session, product_data: dict): - product = get_product_by_id(session, product_data["product_id"]) - if product: + +def create_or_update_product(session, product_data: dict) -> None: + if get_product_by_id(session, product_data["product_id"]): update_product(session, product_data) else: create_product(session, product_data) session.commit() -def delete_product_by_id(session, product_id: str): +def delete_product_by_id(session, product_id: str) -> None: product = get_product_by_id(session, product_id) if product: session.delete(product) - session.commit() \ No newline at end of file + session.commit() diff --git a/recommender/app/shared/kafka/utils.py b/recommender/app/shared/kafka/utils.py index ae126f9..f13a0a3 100644 --- a/recommender/app/shared/kafka/utils.py +++ b/recommender/app/shared/kafka/utils.py @@ -1,11 +1,10 @@ -from recommender.app.shared.kafka.models import EventType +from shared.kafka.models import EventType def product_is_created_or_updated(event: dict) -> bool: - return event["type"] == EventType.PRODUCT_CREATED or event["type"] == EventType.PRODUCT_UPDATED + event_type = event["type"] + return event_type in (EventType.PRODUCT_CREATED.value, EventType.PRODUCT_UPDATED.value) -def product_is_deleted(event: dict) -> bool: - return event["type"] == EventType.PRODUCT_DELETED -def is_interaction_event(event: dict) -> bool: - return event["type"] == EventType.INTERACTION_CREATED or event["type"] == EventType.INTERACTION_UPDATED \ No newline at end of file +def product_is_deleted(event: dict) -> bool: + return event["type"] == EventType.PRODUCT_DELETED.value diff --git a/recommender/app/shared/product/utils.py b/recommender/app/shared/product/utils.py index 3910b7f..728592e 100644 --- a/recommender/app/shared/product/utils.py +++ b/recommender/app/shared/product/utils.py @@ -3,9 +3,19 @@ from shared.config.settings import PRODUCT_API +def normalize_product_data(product_data: dict) -> dict: + product_id = product_data.get("product_id") or product_data.get("id") + account_id = product_data.get("accountID") or product_data.get("account_id") + return { + "product_id": product_id, + "name": product_data["name"], + "description": product_data["description"], + "price": product_data["price"], + "accountID": account_id, + } + + def fetch_product_by_id(product_id: str) -> dict: - """Fetch a product data by its ID from the product API.""" response = requests.get(f"{PRODUCT_API}/{product_id}") response.raise_for_status() - product_data = response.json() - return product_data + return normalize_product_data(response.json()) From f80809895fd16212c5d3034931cb2178a20a8db2 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Sun, 12 Jul 2026 21:42:42 +0400 Subject: [PATCH 08/12] fix(recommender): Initialzie schema safely on startup --- recommender/app/shared/db/schema.py | 15 +++++++++++++++ recommender/app/shared/db/session.py | 4 +--- 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 recommender/app/shared/db/schema.py diff --git a/recommender/app/shared/db/schema.py b/recommender/app/shared/db/schema.py new file mode 100644 index 0000000..8e44068 --- /dev/null +++ b/recommender/app/shared/db/schema.py @@ -0,0 +1,15 @@ +from loguru import logger +from sqlalchemy.exc import IntegrityError, OperationalError + +from shared.db.models import Base +from shared.db.session import replica_engine + + +def ensure_schema() -> None: + """Create tables if missing. Tolerates concurrent startup across containers.""" + try: + Base.metadata.create_all(replica_engine, checkfirst=True) + except (IntegrityError, OperationalError) as exc: + logger.info("Database schema already initialized by another process") + logger.debug("Schema init detail: {}", exc) + replica_engine.dispose() diff --git a/recommender/app/shared/db/session.py b/recommender/app/shared/db/session.py index 196e75c..90783a7 100644 --- a/recommender/app/shared/db/session.py +++ b/recommender/app/shared/db/session.py @@ -1,13 +1,11 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from shared.db.models import Base from shared.config.settings import DATABASE_URL - replica_engine = create_engine(DATABASE_URL, pool_pre_ping=True) ReplicaSession = sessionmaker(bind=replica_engine) -Base.metadata.create_all(replica_engine) + def get_session(): return ReplicaSession() \ No newline at end of file From 288ae1135318f32a94d26f4ece7f1b879d925876 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Sun, 12 Jul 2026 21:43:10 +0400 Subject: [PATCH 09/12] refactor(recommender): consolidate CLI into root main.py --- recommender/app/main.py | 28 ---------------------------- recommender/main.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 28 deletions(-) delete mode 100644 recommender/app/main.py diff --git a/recommender/app/main.py b/recommender/app/main.py deleted file mode 100644 index 3be0e66..0000000 --- a/recommender/app/main.py +++ /dev/null @@ -1,28 +0,0 @@ -import sys - -from loguru import logger - -from entry.grpc_server import serve -from entry.kafka_consumer import start_kafka_consumer -from recommendations.train import train_and_save - - -def main() -> None: - if len(sys.argv) < 2: - logger.error("Usage: python main.py [consumer|server|train]") - sys.exit(1) - - command = sys.argv[1] - if command == "consumer": - start_kafka_consumer() - elif command == "server": - serve() - elif command == "train": - train_and_save() - else: - logger.error("Unknown command: {}", command) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/recommender/main.py b/recommender/main.py index e69de29..c3fa21e 100644 --- a/recommender/main.py +++ b/recommender/main.py @@ -0,0 +1,36 @@ +import os +import sys + +APP_DIR = os.path.join(os.path.dirname(__file__), "app") +if APP_DIR not in sys.path: + sys.path.insert(0, APP_DIR) + +from loguru import logger + +from entry.grpc_server import serve +from entry.kafka_consumer import start_kafka_consumer +from recommendations.train import train_and_save +from shared.db.schema import ensure_schema + + +def main() -> None: + if len(sys.argv) < 2: + logger.error("Usage: python main.py [consumer|server|train]") + sys.exit(1) + + ensure_schema() + + command = sys.argv[1] + if command == "consumer": + start_kafka_consumer() + elif command == "server": + serve() + elif command == "train": + train_and_save() + else: + logger.error("Unknown command: {}", command) + sys.exit(1) + + +if __name__ == "__main__": + main() From 1d1d537144171e50c3469fb577ade3f4f9bd5385 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Sun, 12 Jul 2026 21:43:27 +0400 Subject: [PATCH 10/12] fix(recommender): compare Kafka event types using str Enum --- recommender/app/entry/kafka_consumer.py | 1 - recommender/app/shared/kafka/models.py | 3 ++- recommender/app/shared/kafka/utils.py | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/recommender/app/entry/kafka_consumer.py b/recommender/app/entry/kafka_consumer.py index daa5294..dd2ec2e 100644 --- a/recommender/app/entry/kafka_consumer.py +++ b/recommender/app/entry/kafka_consumer.py @@ -70,6 +70,5 @@ def _handle_interaction_event(event: dict) -> None: ) session.rollback() - if __name__ == "__main__": start_kafka_consumer() diff --git a/recommender/app/shared/kafka/models.py b/recommender/app/shared/kafka/models.py index 6ee0ba9..cb9b365 100644 --- a/recommender/app/shared/kafka/models.py +++ b/recommender/app/shared/kafka/models.py @@ -1,6 +1,7 @@ from enum import Enum -class EventType(Enum): + +class EventType(str, Enum): PRODUCT_CREATED = "product_created" PRODUCT_UPDATED = "product_updated" PRODUCT_DELETED = "product_deleted" diff --git a/recommender/app/shared/kafka/utils.py b/recommender/app/shared/kafka/utils.py index f13a0a3..4d03a9f 100644 --- a/recommender/app/shared/kafka/utils.py +++ b/recommender/app/shared/kafka/utils.py @@ -2,9 +2,8 @@ def product_is_created_or_updated(event: dict) -> bool: - event_type = event["type"] - return event_type in (EventType.PRODUCT_CREATED.value, EventType.PRODUCT_UPDATED.value) + return event["type"] in (EventType.PRODUCT_CREATED, EventType.PRODUCT_UPDATED) def product_is_deleted(event: dict) -> bool: - return event["type"] == EventType.PRODUCT_DELETED.value + return event["type"] == EventType.PRODUCT_DELETED From 2f096bddd72546f18444b9dd93898375a983d8a4 Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Sun, 12 Jul 2026 21:43:56 +0400 Subject: [PATCH 11/12] fix(recommender): listen on port 8080 for GraphQL integration --- docker/services/recommender.dockerfile | 24 +++++++++++++++++++---- recommender/app/entry/grpc_server.py | 5 +++-- recommender/app/shared/config/settings.py | 2 ++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docker/services/recommender.dockerfile b/docker/services/recommender.dockerfile index c5f1d42..57d2d99 100644 --- a/docker/services/recommender.dockerfile +++ b/docker/services/recommender.dockerfile @@ -1,12 +1,28 @@ FROM python:3.11-slim +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + WORKDIR /app -RUN apt-get update && apt-get install -y build-essential libpq-dev && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install dependencies first so this layer is cached when only app code changes. +COPY recommender/pyproject.toml recommender/uv.lock ./ +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never -COPY recommender/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN uv sync --frozen --no-install-project --no-dev COPY recommender /app +RUN uv sync --frozen --no-dev + +ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONPATH="/app/app:${PYTHONPATH}" -ENV PYTHONPATH="/app:${PYTHONPATH}" \ No newline at end of file +EXPOSE 8080 + +WORKDIR /app diff --git a/recommender/app/entry/grpc_server.py b/recommender/app/entry/grpc_server.py index 1af1cfa..e6abe3f 100644 --- a/recommender/app/entry/grpc_server.py +++ b/recommender/app/entry/grpc_server.py @@ -4,6 +4,7 @@ from generated.pb import recommender_pb2, recommender_pb2_grpc from recommendations.load import load_service +from shared.config.settings import GRPC_PORT from shared.db.repo import get_products_by_ids from shared.db.session import get_session @@ -56,7 +57,7 @@ def serve() -> None: RecommenderServiceServicer(service), server, ) - server.add_insecure_port("[::]:50051") - logger.info("gRPC server started on port 50051") + server.add_insecure_port(f"[::]:{GRPC_PORT}") + logger.info("gRPC server started on port {}", GRPC_PORT) server.start() server.wait_for_termination() diff --git a/recommender/app/shared/config/settings.py b/recommender/app/shared/config/settings.py index 432dcee..f4ddaf6 100644 --- a/recommender/app/shared/config/settings.py +++ b/recommender/app/shared/config/settings.py @@ -19,3 +19,5 @@ S3_ACCESS_KEY_ID = os.getenv("S3_ACCESS_KEY_ID") S3_SECRET_ACCESS_KEY = os.getenv("S3_SECRET_ACCESS_KEY") S3_REGION = os.getenv("S3_REGION", "auto") + +GRPC_PORT = int(os.getenv("GRPC_PORT", "8080")) From 2258ea1e4e9f78e31d1db0cb86f880dc6005cb5f Mon Sep 17 00:00:00 2001 From: Rauf Asadov Date: Sun, 12 Jul 2026 21:44:29 +0400 Subject: [PATCH 12/12] chore(docker): wire recommender train job and uv-based image --- docker-compose.yaml | 28 ++++++++++++++++++++++++++-- recommender/requirements.txt | Bin 908 -> 0 bytes 2 files changed, 26 insertions(+), 2 deletions(-) delete mode 100644 recommender/requirements.txt diff --git a/docker-compose.yaml b/docker-compose.yaml index a1242b3..b55a007 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -162,13 +162,17 @@ services: build: context: . dockerfile: docker/services/recommender.dockerfile - command: python app/entry/main.py + command: python main.py server depends_on: - recommender_db - kafka environment: DATABASE_URL: postgresql://rasadov:123456@recommender_db/rasadov?sslmode=disable KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + ARTIFACTS_LOCAL_PATH: /app/artifacts + GRPC_PORT: 8080 + volumes: + - recommender_artifacts:/app/artifacts restart: on-failure recommender-sync: @@ -177,13 +181,32 @@ services: build: context: . dockerfile: docker/services/recommender.dockerfile - command: python app/entry/sync.py + command: python main.py consumer depends_on: - recommender_db - kafka environment: DATABASE_URL: postgresql://rasadov:123456@recommender_db/rasadov?sslmode=disable KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + PRODUCT_API: http://product:8080/products + PRODUCT_EVENTS_TOPIC: product_events + INTERACTION_EVENTS_TOPIC: interaction_events + restart: on-failure + + recommender-train: + container_name: "recommender-train" + image: "rasadov/recommender-train:latest" + build: + context: . + dockerfile: docker/services/recommender.dockerfile + command: sh -c "while true; do python main.py train; sleep 43200; done" + depends_on: + - recommender_db + environment: + DATABASE_URL: postgresql://rasadov:123456@recommender_db/rasadov?sslmode=disable + ARTIFACTS_LOCAL_PATH: /app/artifacts + volumes: + - recommender_artifacts:/app/artifacts restart: on-failure graphql: @@ -215,5 +238,6 @@ volumes: order_db_data: payment_db_data: recommender_db_data: + recommender_artifacts: kafka-volume: zookeeper-volume: \ No newline at end of file diff --git a/recommender/requirements.txt b/recommender/requirements.txt deleted file mode 100644 index 7cc70857ff30e83fbbb24f9461737dcff45d4c05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 908 zcmZvb%TB^j5Qb-M;-l0+ixL+u+`BP8fT38VQcI=9AP=v8|4fTDAhsaTl7J+&JUaoeC!lVH)O6IUm5ZbO}0&!3~Ras~A`wcC+G zi}n&+L+4WR_tdU^Etpo!N=ej#Szph&1iFu%>|ix`)J)+Sl|$bd`D!zw#@KT0d-SN| YU!duCoap@jKB?xa-@>d%Xgdb}FRQPGNB{r;