insert batch

docker-compose.yml

@@ -11,9 +11,9 @@ services:
       - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=false
       # volumes:
       #   - ./kafka:/bitnami/kafka:rw
-    ports:
-      - 9094:9094
-      - 9092:9092
+    expose:
+      - 9094
+      - 9092
     healthcheck:
       test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "localhost:9092"]
       interval: 30s
@@ -30,7 +30,7 @@ services:
       - JVM_OPTS=-Xms32M -Xmx64M
       - SERVER_SERVLET_CONTEXTPATH=/
     ports:
-      - 9000:9000
+      - 9999:9000
     restart: on-failure
     networks:
       - botdetector-network
@@ -62,8 +62,10 @@ services:
     volumes:
       - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
       # - ./mysql/mount:/var/lib/mysql # creates persistence
-    ports:
-      - 3306:3306
+    # ports:
+    #   - 3306:3306
+    expose:
+      - 3306
     networks:
       - botdetector-network
     healthcheck:

kafka_setup/setup_kafka.py

@@ -1,7 +1,6 @@
 # setup_kafka.py
 import json
 import os
-import time
 import zipfile
 from queue import Queue
 

src/app/schemas/highscores.py

@@ -1,108 +1,107 @@
-from datetime import date, datetime
-from typing import Optional
-
-from pydantic import BaseModel, ConfigDict
-
-
-class playerHiscoreData(BaseModel):
-    model_config = ConfigDict(from_attributes=True)
-
-    # id: Optional[int] = None
-    timestamp: datetime = datetime.utcnow()
-    # ts_date: Optional[date] = None
-    Player_id: int
-    total: int
-    attack: int
-    defence: int
-    strength: int
-    hitpoints: int
-    ranged: int
-    prayer: int
-    magic: int
-    cooking: int
-    woodcutting: int
-    fletching: int
-    fishing: int
-    firemaking: int
-    crafting: int
-    smithing: int
-    mining: int
-    herblore: int
-    agility: int
-    thieving: int
-    slayer: int
-    farming: int
-    runecraft: int
-    hunter: int
-    construction: int
-    league: int
-    bounty_hunter_hunter: int
-    bounty_hunter_rogue: int
-    cs_all: int
-    cs_beginner: int
-    cs_easy: int
-    cs_medium: int
-    cs_hard: int
-    cs_elite: int
-    cs_master: int
-    lms_rank: int
-    soul_wars_zeal: int
-    abyssal_sire: int
-    alchemical_hydra: int
-    barrows_chests: int
-    bryophyta: int
-    callisto: int
-    cerberus: int
-    chambers_of_xeric: int
-    chambers_of_xeric_challenge_mode: int
-    chaos_elemental: int
-    chaos_fanatic: int
-    commander_zilyana: int
-    corporeal_beast: int
-    crazy_archaeologist: int
-    dagannoth_prime: int
-    dagannoth_rex: int
-    dagannoth_supreme: int
-    deranged_archaeologist: int
-    general_graardor: int
-    giant_mole: int
-    grotesque_guardians: int
-    hespori: int
-    kalphite_queen: int
-    king_black_dragon: int
-    kraken: int
-    kreearra: int
-    kril_tsutsaroth: int
-    mimic: int
-    nightmare: int
-    nex: int = 0
-    phosanis_nightmare: int
-    obor: int
-    phantom_muspah: int = 0
-    sarachnis: int
-    scorpia: int
-    skotizo: int
-    tempoross: int = 0
-    the_gauntlet: int
-    the_corrupted_gauntlet: int
-    theatre_of_blood: int
-    theatre_of_blood_hard: int = 0
-    thermonuclear_smoke_devil: int
-    tombs_of_amascut: int = 0
-    tombs_of_amascut_expert: int = 0
-    tzkal_zuk: int
-    tztok_jad: int
-    venenatis: int
-    vetion: int
-    vorkath: int
-    wintertodt: int
-    zalcano: int
-    zulrah: int
-    rifts_closed: int = 0
-    artio: int = 0
-    calvarion: int = 0
-    duke_sucellus: int = 0
-    spindel: int = 0
-    the_leviathan: int = 0
-    the_whisperer: int = 0
-    vardorvis: int = 0
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict
+
+
+class playerHiscoreData(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    # id: Optional[int] = None
+    timestamp: datetime = datetime.utcnow()
+    # ts_date: Optional[date] = None
+    Player_id: int
+    total: int
+    attack: int
+    defence: int
+    strength: int
+    hitpoints: int
+    ranged: int
+    prayer: int
+    magic: int
+    cooking: int
+    woodcutting: int
+    fletching: int
+    fishing: int
+    firemaking: int
+    crafting: int
+    smithing: int
+    mining: int
+    herblore: int
+    agility: int
+    thieving: int
+    slayer: int
+    farming: int
+    runecraft: int
+    hunter: int
+    construction: int
+    league: int
+    bounty_hunter_hunter: int
+    bounty_hunter_rogue: int
+    cs_all: int
+    cs_beginner: int
+    cs_easy: int
+    cs_medium: int
+    cs_hard: int
+    cs_elite: int
+    cs_master: int
+    lms_rank: int
+    soul_wars_zeal: int
+    abyssal_sire: int
+    alchemical_hydra: int
+    barrows_chests: int
+    bryophyta: int
+    callisto: int
+    cerberus: int
+    chambers_of_xeric: int
+    chambers_of_xeric_challenge_mode: int
+    chaos_elemental: int
+    chaos_fanatic: int
+    commander_zilyana: int
+    corporeal_beast: int
+    crazy_archaeologist: int
+    dagannoth_prime: int
+    dagannoth_rex: int
+    dagannoth_supreme: int
+    deranged_archaeologist: int
+    general_graardor: int
+    giant_mole: int
+    grotesque_guardians: int
+    hespori: int
+    kalphite_queen: int
+    king_black_dragon: int
+    kraken: int
+    kreearra: int
+    kril_tsutsaroth: int
+    mimic: int
+    nightmare: int
+    nex: int = 0
+    phosanis_nightmare: int
+    obor: int
+    phantom_muspah: int = 0
+    sarachnis: int
+    scorpia: int
+    skotizo: int
+    tempoross: int = 0
+    the_gauntlet: int
+    the_corrupted_gauntlet: int
+    theatre_of_blood: int
+    theatre_of_blood_hard: int = 0
+    thermonuclear_smoke_devil: int
+    tombs_of_amascut: int = 0
+    tombs_of_amascut_expert: int = 0
+    tzkal_zuk: int
+    tztok_jad: int
+    venenatis: int
+    vetion: int
+    vorkath: int
+    wintertodt: int
+    zalcano: int
+    zulrah: int
+    rifts_closed: int = 0
+    artio: int = 0
+    calvarion: int = 0
+    duke_sucellus: int = 0
+    spindel: int = 0
+    the_leviathan: int = 0
+    the_whisperer: int = 0
+    vardorvis: int = 0

src/core/logging.py

@@ -1,7 +1,6 @@
 import json
 import logging
 import sys
-import warnings
 
 from core.config import settings
 

src/database/database.py

@@ -1,4 +1,3 @@
-import sqlalchemy
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker