Assignment_2 (1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment_2.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "6x0hBa1tgNZj"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import math\n",
        "import random\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.optimizers import Adam\n",
        "num_actions=3  #0 for hold, 1 for buy, 2 for sell\n",
        "def create_model():\n",
        "        model = Sequential()\n",
        "        model.add(Dense(units=32, input_shape=(20,) , activation=\"relu\"))\n",
        "        model.add(Dense(units=16, activation=\"relu\"))\n",
        "        model.add(Dense(units=8, activation=\"relu\"))\n",
        "        model.add(Dense(num_actions, activation=\"linear\"))\n",
        "        model.compile(loss=\"mse\", optimizer=Adam(learning_rate=0.0025),metrics=['accuracy'])\n",
        "        return model\n",
        "class EpsilonGreedyStrategy():\n",
        "    def __init__(self,epsilon,epo_min, decay):\n",
        "        self.epsilon = epsilon\n",
        "        self.epo_min = epo_min\n",
        "        self.decay = decay  \n",
        "    def get_exploration_rate(self, current_step):\n",
        "        return self.epo_min+(self.epsilon-self.epo_min)*math.exp(-1*current_step*self.decay) \n",
        "def getStockData(key):\n",
        "    stocks = []\n",
        "    lines = open(key+\".csv\",\"r\").read().splitlines()\n",
        "    for line in lines[1:]:\n",
        "        stocks.append(float(line.split(\",\")[4])) \n",
        "    return stocks    \n",
        "def sigmoid(x):\n",
        "    return 1/(1+math.exp(-x))    \n",
        "def getState(data, t, n):\n",
        "    d =t-n+1\n",
        "    block=data[d:t+1]\n",
        "    return np.array(block)\n",
        "    # res=[]\n",
        "    # for i in range(n-1):\n",
        "    #   res.append((block[i+1]-block[i]))  \n",
        "    # return np.array(res)\n",
        "    \n",
        "\n",
        "        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "stY2q7W0gimb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e7a6d630-59ca-4b13-aab4-ad1bbc13fb97"
      },
      "source": [
        "#random.seed(2)\n",
        "gamma = 0.9  # Discount factor for past rewards\n",
        "epsilon = 1.0  # Epsilon greedy parameter\n",
        "epsilon_min = 0.1  # Minimum epsilon greedy parameter\n",
        "decay_rate = 0.001  #epsilon decay rate  \n",
        "memory_size = 100000\n",
        "num_episodes = 50\n",
        "num_space=[i for i in range(num_actions)]\n",
        "action_data = []\n",
        "state_data = []\n",
        "next_state_data = []\n",
        "rewards_data = []\n",
        "done_data = []\n",
        "stock_name = input(\"Enter stock_name : \")\n",
        "stock_name = str(stock_name)\n",
        "window_size = 20\n",
        "data=getStockData(stock_name)\n",
        "l=len(data)-1\n",
        "batch_size = 32\n",
        "i=window_size-1\n",
        "update_target_network = 30 #update target network weights after 30 days \n",
        "strategy=EpsilonGreedyStrategy(epsilon,epsilon_min,decay_rate)\n",
        "# The first model makes the predictions for Q-values which are used to\n",
        "# make a action.\n",
        "model = create_model()\n",
        "# Target model used to predict optimal Q-values\n",
        "target_model= create_model()\n",
        "loss_function = keras.losses.Huber()\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=0.0005)\n",
        "MAX_TRANSACTIONS =50   #maximum number of time we can do a buy after a selling\n",
        "TOTAL_MONEY = int(input())    #total money that we have in cash(virtual )\n",
        "money=TOTAL_MONEY\n",
        "for epi in range(100):\n",
        "    TOTAL_MONEY=money\n",
        "    CURRENT_STOCKS_BOUGHT = 0      #the number of stocks we have bought currently\n",
        "    CURRENT_TRANSACTION_COUNT = 0\n",
        "    state = getState(data, i, window_size)\n",
        "    total_profit = 0\n",
        "    for day in range(i,l):\n",
        "        action=0 \n",
        "        done = True if day == l - 1 else False \n",
        "        epsilon=strategy.get_exploration_rate(day-i+1)\n",
        "        if random.random()<= epsilon:\n",
        "            action= np.random.choice(num_space)\n",
        "        else: \n",
        "            state_tensor = tf.convert_to_tensor(state)\n",
        "            state_tensor = tf.expand_dims(state_tensor, 0)\n",
        "            action= tf.argmax((model(state_tensor,training=False))[0]).numpy()\n",
        "            \n",
        "        next_state =getState(data, i+1, window_size) \n",
        "        \n",
        "        reward=0\n",
        "        x=0\n",
        "        if CURRENT_TRANSACTION_COUNT<MAX_TRANSACTIONS:\n",
        "            x=TOTAL_MONEY/(MAX_TRANSACTIONS- CURRENT_TRANSACTION_COUNT) \n",
        "            x=round(x*100)/100      \n",
        "        if( action==1 and x>=data[day]):\n",
        "            y=int(x/data[day])\n",
        "            TOTAL_MONEY-=x\n",
        "            reward=(data[day+1]-data[day])*y\n",
        "            CURRENT_STOCKS_BOUGHT+=y\n",
        "            CURRENT_TRANSACTION_COUNT+=1  \n",
        "        elif action ==2 and CURRENT_STOCKS_BOUGHT>0:\n",
        "            reward=data[day]*CURRENT_STOCKS_BOUGHT + TOTAL_MONEY - money    \n",
        "            TOTAL_MONEY+= data[day]*CURRENT_STOCKS_BOUGHT\n",
        "            CURRENT_TRANSACTION_COUNT =0\n",
        "            CURRENT_STOCKS_BOUGHT=0\n",
        "        #elif action ==0:\n",
        "            # if(data[day]>data[day+1] and CURRENT_STOCKS_BOUGHT):\n",
        "            #     reward=-(data[day]*CURRENT_STOCKS_BOUGHT + TOTAL_MONEY - money)\n",
        "            # elif data[day]<data[day+1] and x>=data[day]:\n",
        "            #     stocks=int(x/data[day])                \n",
        "            #     reward=-(data[day+1]-data[day])*stocks\n",
        "        else:\n",
        "            action=0                  \n",
        "                           \n",
        "        reward=round(reward*100)/100\n",
        "        TOTAL_MONEY=round(TOTAL_MONEY*100)/100;\n",
        "        \n",
        "        action_data.append(action)\n",
        "        state_data.append(state)\n",
        "        next_state_data.append(next_state)\n",
        "        done_data.append(done)\n",
        "        rewards_data.append(reward)\n",
        "        state = next_state    \n",
        "        \n",
        "        #Experience replay\n",
        "        if len(done_data) > batch_size:\n",
        "            sample = np.random.choice(range(len(done_data)), size=batch_size)\n",
        "\n",
        "            state_sample = np.array([state_data[i] for i in sample])\n",
        "            state_next_sample = np.array([next_state_data[i] for i in sample])\n",
        "            rewards_sample = [rewards_data[i] for i in sample]\n",
        "            action_sample = [action_data[i] for i in sample]  \n",
        "            done_sample=[done_data[i] for i in sample]\n",
        "          \n",
        "            future_q_val= target_model.predict(state_next_sample)\n",
        "            target_q_val= np.array(rewards_sample) + gamma* tf.reduce_max(future_q_val, axis=1)\n",
        "            masks = tf.one_hot(action_sample, num_actions)\n",
        "            \n",
        "\n",
        "            with tf.GradientTape() as tape:\n",
        "               \n",
        "                cur_q_values = model(state_sample)\n",
        "\n",
        "                # Apply the masks to the Q-values to get the Q-value for action taken\n",
        "                cur_q_action = tf.reduce_sum(tf.multiply(cur_q_values, masks), axis=1)\n",
        "                # Calculate loss between Target Q-value and current Q-value\n",
        "                loss = loss_function(target_q_val, cur_q_action)\n",
        "\n",
        "            grad = tape.gradient(loss, model.trainable_variables)\n",
        "            optimizer.apply_gradients(zip(grad, model.trainable_variables))\n",
        "               \n",
        "                                         \n",
        "        if  day-i+1 % update_target_network == 0:\n",
        "            # update the the target network with new weights\n",
        "            target_model.set_weights(model.get_weights()) \n",
        "             \n",
        "\n",
        "        if done:\n",
        "            break\n",
        "        if len(rewards_data) >memory_size:\n",
        "            rewards_data[len(rewards_data)%memory_size]=reward\n",
        "            state_data[len(rewards_data)%memory_size]=state\n",
        "            next_state_data[len(rewards_data)%memory_size]=next_state\n",
        "            action_data[len(rewards_data)%memory_size]=action\n",
        "            done_data[len(rewards_data)%memory_size]=done                \n",
        "\n",
        "model.save(\"mymodel\")  \n",
        "\n",
        "            \n",
        "    \n",
        "            \n",
        "\n",
        "          \n",
        "        \n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter stock_name : CSCO\n",
            "20000\n",
            "INFO:tensorflow:Assets written to: mymodel/assets\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BkEAtJ83S00s"
      },
      "source": [
        "Test evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MRTgvXemiBy2",
        "outputId": "159f107f-8081-4521-d517-eaa085e48c0c"
      },
      "source": [
        "stock_name = input(\"Enter Stock_name: \")\n",
        "data = getStockData(stock_name)\n",
        "l = len(data) - 1\n",
        "print(l)\n",
        "total_profit = 0\n",
        "window_size=20\n",
        "i=window_size-1\n",
        "MAX_TRANSACTIONS = 50   #maximum number of time we can do a buy after a selling\n",
        "TOTAL_MONEY = int(input())    #total money that we have in cash(virtual )\n",
        "CURRENT_STOCKS_BOUGHT = 0      #the number of stocks we have bought currently\n",
        "CURRENT_TRANSACTION_COUNT = 0\n",
        "money=TOTAL_MONEY\n",
        "state = getState(data, i, window_size)\n",
        "model=keras.models.load_model(\"mymodel\")\n",
        "total_days=l-i+1\n",
        "for day in range(i,l):\n",
        "     \n",
        "    action =0  \n",
        "    done = True if day == l - 1 else False \n",
        "  \n",
        "    state_tensor = tf.convert_to_tensor(state)\n",
        "    state_tensor = tf.expand_dims(state_tensor, 0)\n",
        "    action= tf.argmax((model(state_tensor))[0]).numpy()\n",
        "   \n",
        "    next_state = getState(data, day + 1, window_size)\n",
        "        \n",
        "    if day == l-1:\n",
        "       action =2\n",
        "\n",
        "    if(action==1 and TOTAL_MONEY>=data[day] and CURRENT_TRANSACTION_COUNT<MAX_TRANSACTIONS):\n",
        "        x=TOTAL_MONEY/(MAX_TRANSACTIONS- CURRENT_TRANSACTION_COUNT) \n",
        "        x=round(x*100)/100\n",
        "        y=int(x/data[day])\n",
        "        TOTAL_MONEY-=data[day]*y;\n",
        "        CURRENT_STOCKS_BOUGHT+=y\n",
        "        CURRENT_TRANSACTION_COUNT+=1  \n",
        "    elif action ==2 and CURRENT_STOCKS_BOUGHT>0:\n",
        "        TOTAL_MONEY+= data[day]*CURRENT_STOCKS_BOUGHT\n",
        "        total_profit+=TOTAL_MONEY - money     \n",
        "        CURRENT_TRANSACTION_COUNT =0\n",
        "        CURRENT_STOCKS_BOUGHT=0\n",
        "    else:\n",
        "        action =0    \n",
        "      \n",
        "    TOTAL_MONEY=round(TOTAL_MONEY*100)/100\n",
        "    total_profit=round(total_profit*100)/100\n",
        "    \n",
        "    if done:\n",
        "        print(\"Rs.\"+ \"{0:.2f}\".format(total_profit))\n",
        "        break       "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter Stock_name: CSCO_TESTDATA\n",
            "501\n",
            "20000\n",
            "Rs.2300.69\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}