RPS_regretmatching.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Returns the adjusted strategy after an iteration\n",
    "def getStrategy(regretSum,strategySum):\n",
    "    ''' \n",
    "    Define each variable and give example values:\n",
    "    - regretSum = [-2, -5, 2] ---> cumulative regret\n",
    "    - strategy = [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] or [0.0, 0.0, 1.0] or [0.3, 0.4, 0.3] ---> policy yielded by regret matching for next RPS showdown\n",
    "    - strategySum = [1.0, 0.3333333333333333, 3.6666666666666665] ---> sum of probabilities in strategy array over all iterations\n",
    "    - normalizingSum = 37 ---> used to get strategy array probabilities ---> strategy[i] = strategy[i]/normalizingSum ---> sum of positive regrets only\n",
    "    '''\n",
    "    actions = 3\n",
    "    normalizingSum = 0\n",
    "    strategy = [0,0,0]\n",
    "    #Normalizingsum is the sum of positive regrets. \n",
    "    #This ensures do not 'over-adjust' and converge to equilibrium\n",
    "    for i in range(0,actions):\n",
    "        if regretSum[i] > 0:\n",
    "            strategy[i] = regretSum[i]\n",
    "        else:\n",
    "            strategy[i] = 0\n",
    "        normalizingSum += strategy[i]\n",
    "    ##This loop normalizes our updated strategy\n",
    "    for i in range(0,actions):\n",
    "        if normalizingSum > 0:\n",
    "            strategy[i] = strategy[i]/normalizingSum\n",
    "        else:\n",
    "            #Default to 33%\n",
    "            strategy[i] = 1.0 / actions\n",
    "        strategySum[i] += strategy[i]\n",
    "    return (strategy, strategySum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Returns a random action according to the strategy\n",
    "def getAction(strategy):\n",
    "    ''' \n",
    "    As an example, use strategy = [0.3, 0.4, 0.3]\n",
    "    This function returns 'ROCK' (0) if a random uniform generated value falls between 0-0.3\n",
    "    This function returns 'PAPER' (2) if a random uniform generated value falls between 0.3-0.7 \n",
    "    This function returns 'SCISSORS' (2) if a random uniform generated value falls between 0.7-1 \n",
    "    '''\n",
    "    r = random.uniform(0,1)\n",
    "    if r >= 0 and r < strategy[0]:\n",
    "        return 0\n",
    "    elif r >= strategy[0] and r < strategy[0] + strategy[1]:\n",
    "        return 1\n",
    "    elif r >= strategy[0] + strategy[1] and r < sum(strategy):\n",
    "        return 2\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(iterations,regretSum,oppStrategy):\n",
    "    ''' \n",
    "    Define each variable and give example values:\n",
    "    - regretSum = [-2, -5, 2] ---> cumulative regret\n",
    "    - oppStrategy = [0.3, 0.4, 0.3]\n",
    "    - strategy = [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] or [0.0, 0.0, 1.0] or [0.3, 0.4, 0.3] ---> policy yielded by regret matching for next RPS showdown\n",
    "    - strategySum = [1.0, 0.3333333333333333, 3.6666666666666665] ---> sum of probabilities in strategy array over all iterations\n",
    "    - myaction = 0, 1 or 2 ---> 'ROCK' 'PAPER' or 'SCISSORS'\n",
    "    - otherAction = 0, 1 or 2 ---> 'ROCK' 'PAPER' or 'SCISSORS'\n",
    "    - actionUtility = [-1, 1, -1] or [-1, 0, 1] ---> Hapiness you would have gotten from each of RPS had you chosen them against a fixed opponent choice like 'ROCK'. \n",
    "                      First the 3 actionUtilities are calculated for the opponent, then the action utility for myaction and opponent action pair is calculated. \n",
    "                      Then they're all used to get the regret and then regretSum\n",
    "    '''\n",
    "    actionUtility = [0,0,0]\n",
    "    strategySum = [0,0,0]\n",
    "    actions = 3\n",
    "    for i in range(0,iterations):\n",
    "        ##Retrieve Actions\n",
    "        t = getStrategy(regretSum,strategySum)\n",
    "        strategy = t[0]\n",
    "        strategySum = t[1]\n",
    "        myaction = getAction(strategy)\n",
    "        #Define an arbitrary opponent strategy from which to adjust\n",
    "        otherAction = getAction(oppStrategy)   \n",
    "        #Opponent Chooses scissors\n",
    "        if otherAction == actions - 1: # FALSE: 3-1 = 2, WE HAVE otherAction == 1 \n",
    "            #Utility(Rock) = 1\n",
    "            actionUtility[0] = 1\n",
    "            #Utility(Paper) = -1\n",
    "            actionUtility[1] = -1\n",
    "        #Opponent Chooses Rock\n",
    "        elif otherAction == 0: # FALSE: WE HAVE otherAction == 1 \n",
    "            #Utility(Scissors) = -1\n",
    "            actionUtility[actions - 1] = -1\n",
    "            #Utility(Paper) = 1\n",
    "            actionUtility[1] = 1\n",
    "        #Opopnent Chooses Paper\n",
    "        else: # TRUE: WE HAVE otherAction == 1 \n",
    "            #Utility(Rock) = -1\n",
    "            actionUtility[0] = -1\n",
    "            #Utility(Scissors) = 1\n",
    "            actionUtility[2] = 1\n",
    "                    \n",
    "        #Add the regrets from this decision\n",
    "        for i in range(0,actions):\n",
    "            regretSum[i] += actionUtility[i] - actionUtility[myaction]\n",
    "    return strategySum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAverageStrategy(iterations,oppStrategy):\n",
    "    ''' \n",
    "    Define each variable and give example values:\n",
    "    - avgStrategy = [0, 0, 0] ---> a.k.a.: 'Maximally Exploitative Strategy' ---> Strategy to use in next round, considering info from all previous rounds put together\n",
    "    - strategySum = [1.0, 0.3333333333333333, 3.6666666666666665] ---> sum of probabilities in strategy array over all iterations\n",
    "    - oppStrategy = [0.3, 0.4, 0.3]\n",
    "    - normalizingSum = 37 ---> used to get strategy array probabilities ---> strategy[i] = strategy[i]/normalizingSum ---> sum of positive regrets only\n",
    "    '''\n",
    "    actions = 3\n",
    "    strategySum = train(iterations,[0,0,0],oppStrategy)\n",
    "    avgStrategy = [0,0,0]\n",
    "    normalizingSum = 0\n",
    "    for i in range(0,actions):\n",
    "        normalizingSum += strategySum[i]\n",
    "    for i in range(0,actions):\n",
    "        if normalizingSum > 0:\n",
    "            avgStrategy[i] = strategySum[i] / normalizingSum\n",
    "        else:\n",
    "            avgStrategy[i] = 1.0 / actions\n",
    "    print('avgStrategy: ', avgStrategy)\n",
    "    return avgStrategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opponent's Strategy [0.34, 0.3, 0.3]\n",
      "avgStrategy:  [0.004613333333333333, 0.9951866666666667, 0.0002]\n",
      "Maximally Exploitative Strat [0.004613333333333333, 0.9951866666666667, 0.0002]\n"
     ]
    }
   ],
   "source": [
    "oppStrategy = [.34,.3,.3]\n",
    "print(\"Opponent's Strategy\",oppStrategy)\n",
    "print(\"Maximally Exploitative Strat\", getAverageStrategy(5000, oppStrategy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "e62dcc45d4cf21a563dc8c58b679171a51000e7c290e6a36376a6644d2c9a33b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}