WIP: Mode to update the current season. (#64)

* Adding default_mode and current_season_mode switches

* live refresh of shot_chart_detail working

* current season ignore tables

* update current season for player_game_log and game table

requirements.txt

@@ -1,10 +1,10 @@
-requests==2.25.1
+requests==2.26.0
 certifi==2020.12.5
 cffi==1.14.5
 chardet==4.0.0
-cryptography==3.3.2
+cryptography
 idna==2.10
-peewee==3.14.4
+peewee==3.14.8
 pycparser==2.20
 PyMySQL==1.0.0
 python-dotenv==0.15.0
@@ -13,4 +13,4 @@ urllib3==1.26.5
 psycopg2==2.9.1
 psycopg2-binary==2.9.1
 gooey==1.0.8.1
-pyinstaller
+pyinstaller

scripts/create_mysql.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --create-schema
+DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --default_mode --database="mysql" --create-schema

scripts/create_postgres.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --database="postgres" --create-schema
+DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --default_mode --database="postgres" --skip-tables play_by_play pgtt

scripts/create_sqlite.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --database="sqlite" --create-schema --skip-tables play_by_play
+DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --default_mode --skip-tables play_by_play pgtt

scripts/refresh_postgres.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --current_season_mode --database="postgres"

scripts/refresh_sqlite.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+DB_NAME="nba" DB_HOST="localhost" DB_USER=nba_sql DB_PASSWORD=nba_sql python stats/nba_sql.py --current_season_mode

stats/args.py

@@ -0,0 +1,101 @@
+from gooey import GooeyParser
+from constants import season_list
+
+
+"""
+Creates a parser.
+"""
+
+
+def create_parser():
+    """
+    Creates and returns a Gooey parser.
+    """
+
+    parser = GooeyParser(description="nba-sql")
+
+    mode_parser = parser.add_mutually_exclusive_group(
+        required=True,
+        gooey_options={
+            'initial_selection': 0
+        })
+    mode_parser.add_argument(
+        '--default_mode',
+        help='Mode to create the database and load historic data. Use this mode when creating a new database or when trying to load a specific season or a range of seasons.',
+        action='store_true')
+    mode_parser.add_argument(
+        '--current_season_mode',
+        help='Mode to refresh the current season. Use this mode on an existing database to update it with the latest data.',
+        action='store_true')
+
+    parser.add_argument(
+        '--database',
+        dest='database_type',
+        default='sqlite',
+        choices=['mysql', 'postgres', 'sqlite'],
+        help='The database flag specifies which database protocol to use.  Defaults to "sqlite", but also accepts "postgres" and "mysql".')
+
+    parser.add_argument(
+        '--database_name', 
+        help="Database Name (Not Needed For SQLite)",
+        default='nba')
+
+    parser.add_argument(
+        '--database_host', 
+        help="Database Hostname (Not Needed For SQLite)",
+        default=None)
+
+    parser.add_argument(
+        '--username',
+        help="Database Username (Not Needed For SQLite)",
+        default=None)
+
+    parser.add_argument(
+        '--password',
+        help="Database Password (Not Needed For SQLite)",
+        widget='PasswordField',
+        default=None)
+
+    last_loadable_season = season_list[-1]
+
+    parser.add_argument(
+        '--seasons',
+        dest='seasons',
+        default=[last_loadable_season],
+        choices=season_list,
+        widget='Listbox',
+        nargs="*",
+        help='The seasons flag loads the database with the specified season.  The format of the season should be in the form "YYYY-YY".  The default behavior is loading the current season.')
+
+    parser.add_argument(
+        '--create-schema',
+        dest='create_schema',
+        action="store_true",
+        default=True,
+        help='Flag to initialize the database schema before loading data. If the schema already exists then nothing will happen.')
+
+    parser.add_argument(
+        '--time-between-requests',
+        dest='request_gap',
+        default='.7',
+        help='This flag exists to prevent rate limiting, and injects the desired amount of time inbetween requesting resources.')
+
+    parser.add_argument(
+        '--skip-tables',
+        action='store',
+        nargs="*",
+        default='',
+        choices=['player_season', 'player_game_log', 'play_by_play', 'pgtt', 'shot_chart_detail', 'game', 'event_message_type', 'team', 'player', ''],
+        widget='Listbox',
+        help='Use this option to skip loading certain tables.')
+
+    #To fix issue https://github.com/mpope9/nba-sql/issues/56
+    parser.add_argument(
+        '--batch_size',
+        default='10000',
+        type=int,
+        help="Inserts BATCH_SIZE chunks of rows to the database. This value is ignored when selecting database 'sqlite'.")
+
+    return parser
+
+

stats/game.py

@@ -1,7 +1,7 @@
 from models import Game
 from constants import team_abbrev_mapping
 from collections import namedtuple
-from db_utils import insert_many
+from db_utils import insert_many, insert_many_on_conflict_ignore
 
 
 GameEntry = namedtuple("GameEntry", "season_id, game_id, game_date, matchup_in, winner, loser")
@@ -19,11 +19,18 @@ def create_ddl(self):
         """
         self.settings.db.create_tables([Game], safe=True)
 
-    def populate_table(self, game_set):
+    def game_id_predicate(self):
+        """
+        Returns a selection of the game id.
+        """
+        return Game.select(Game.game_id)
+
+    def populate_table(self, game_set, ignore_dups = False):
         """
         Takes a set of tuples and builds the game table.
         @params:
         game_set   - Required  : Set of GameEntry namedtuple entries (Set)
+        ignore_dups - Optional : Will ignore duplicate entries if present.
         """
         rows = []
 
@@ -55,4 +62,7 @@ def populate_table(self, game_set):
 
             rows.append(new_row)
 
-        insert_many(self.settings, Game, rows)
+        if ignore_dups:
+            insert_many_on_conflict_ignore(self.settings, Game, rows)
+        else:
+            insert_many(self.settings, Game, rows)

stats/models/PlayerGameLogTemp.py

@@ -0,0 +1,57 @@
+from peewee import (
+    IntegerField,
+    FloatField,
+    Model,
+    CompositeKey,
+    FixedCharField
+)
+from . import Player
+from . import Team
+from . import Game
+
+
+class PlayerGameLogTemp(Model):
+
+    # Composite PK Fields
+    player_id = IntegerField(index=True)
+    game_id = IntegerField(null=True)
+
+    team_id = IntegerField(index=True)
+
+    # Indexes
+    season_id = IntegerField(index=True)
+
+    wl = FixedCharField(null=True, max_length=1)
+    min = FloatField(null=True)
+    fgm = FloatField(null=True)
+    fga = FloatField(null=True)
+    fg_pct = FloatField(null=True)
+    fg3m = FloatField(null=True)
+    fg3a = FloatField(null=True)
+    fg3_pct = FloatField(null=True)
+    ftm = FloatField(null=True)
+    fta = FloatField(null=True)
+    ft_pct = FloatField(null=True)
+    oreb = FloatField(null=True)
+    dreb = FloatField(null=True)
+    reb = FloatField(null=True)
+    ast = FloatField(null=True)
+    tov = FloatField(null=True)
+    stl = FloatField(null=True)
+    blk = FloatField(null=True)
+    blka = FloatField(null=True)
+    pf = FloatField(null=True)
+    pfd = FloatField(null=True)
+    pts = FloatField(null=True)
+    plus_minus = FloatField(null=True)
+    nba_fantasy_pts = FloatField(null=True)
+    dd2 = FloatField(null=True)
+    td3 = FloatField(null=True)
+
+    class Meta:
+        db_table = 'player_game_log_temp'
+        primary_key = CompositeKey(
+            'player_id',
+            'game_id'
+        )
+        temporary = True

stats/models/__init__.py

@@ -13,6 +13,7 @@
 # Player Tables
 from .PlayerSeason import PlayerSeason
 from .PlayerGameLog import PlayerGameLog
+from .PlayerGameLogTemp import PlayerGameLogTemp
 from .PlayerGeneralTraditionalTotal import PlayerGeneralTraditionalTotal
 from .PlayByPlay import PlayByPlay
 from .ShotChartDetail import ShotChartDetail