Copy from frcfr_data repo.

.gitignore

@@ -0,0 +1,51 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+.eggs/
+*.egg-info/
+
+ # C extensions
+*.so
+
+ # Jupyter Notebook
+.ipynb_checkpoints
+
+ # virtualenv
+.venv
+venv/
+ENV/
+env/
+
+ # OSX specific
+.DS_Store
+
+ # Build products
+build/
+build.*/
+build
+
+ # External modules
+open_spiel/
+open_spiel/abseil-cpp/
+open_spiel/games/bridge/double_dummy_solver/
+pybind11/
+eigen/
+
+# Install artifacts
+get-pip.py
+
+# IDE
+.idea/
+compile_commands.json
+.clangd
+
+# Singularity images
+*.sif
+
+# Misc.
+saved_results/
+manifest.db
+*.out
+runs_remaining.gen.sh
+results/

Makefile

@@ -0,0 +1,86 @@
+FRCFR =f-rcfr
+EXE :=$(FRCFR)/build/bin/run_cfr
+
+default: results/Error.pdf results/Plateau.pdf results/Exploitability_Num_Part.pdf
+	@true
+
+SEEDS_LEDUC :=$(shell seq 1 5)
+NUM_PARTITIONS_LEDUC :=10 20 30 40 50
+ALGS_LEDUC :=hedge-t0_01 hedge-t0_05 hedge-t0_1 hedge-t0_5 hedge-t1 \
+ poly-p1_1 poly-p1_5 poly-p2_5 poly-p3 rm
+
+NUM_PARTITIONS_GOOFSPIEL :=20 40 50 60
+ALGS_GOOFSPIEL :=hedge-t0_1 hedge-t0_5 hedge-t1 hedge-t5 hedge-t10 \
+	poly-p1_1 poly-p1_5 poly-p2_5 poly-p3 rm
+
+NUM_PARTITIONS_RANDOM_GOOFSPIEL :=30 60 90 120
+ALGS_RANDOM_GOOFSPIEL :=hedge-t0_01 hedge-t0_05 hedge-t0_1 hedge-t0_5 hedge-t1 \
+	poly-p1_1 poly-p1_5 poly-p2_5 poly-p3 rm
+
+data:
+	mkdir data
+
+results:
+	mkdir results
+
+results/Error.pdf: \
+			data/goofspiel.null.rm.gen.txt \
+			$(foreach i,$(SEEDS_LEDUC),\
+			$(foreach n,$(NUM_PARTITIONS_GOOFSPIEL),\
+			$(foreach alg,$(ALGS_GOOFSPIEL),data/goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) \
+			data/leduc.null.rm.gen.txt \
+			$(foreach i,$(SEEDS_LEDUC),\
+			$(foreach n,$(NUM_PARTITIONS_LEDUC),\
+			$(foreach alg,$(ALGS_LEDUC),data/leduc.null.$(alg)-s10-n$(n).$(i).gen.txt))) \
+			data/random_goofspiel.null.rm.gen.txt \
+			$(foreach i,$(SEEDS_LEDUC),\
+			$(foreach n,$(NUM_PARTITIONS_RANDOM_GOOFSPIEL),\
+			$(foreach alg,$(ALGS_RANDOM_GOOFSPIEL),data/random_goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) | results
+	python3 bin/plot_aamas_figures.py
+
+results/Exploitability_Num_Part.pdf: \
+		data/goofspiel.null.rm.gen.txt \
+		$(foreach i,$(SEEDS_LEDUC),\
+		$(foreach n,$(NUM_PARTITIONS_GOOFSPIEL),\
+		$(foreach alg,$(ALGS_GOOFSPIEL),data/goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) \
+		data/leduc.null.rm.gen.txt \
+		$(foreach i,$(SEEDS_LEDUC),\
+		$(foreach n,$(NUM_PARTITIONS_LEDUC),\
+		$(foreach alg,$(ALGS_LEDUC),data/leduc.null.$(alg)-s10-n$(n).$(i).gen.txt))) \
+		data/random_goofspiel.null.rm.gen.txt \
+		$(foreach i,$(SEEDS_LEDUC),\
+		$(foreach n,$(NUM_PARTITIONS_RANDOM_GOOFSPIEL),\
+		$(foreach alg,$(ALGS_RANDOM_GOOFSPIEL),data/random_goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) | results
+	python3 bin/plot_aamas_figures.py
+
+results/Plateau.pdf: \
+		data/goofspiel.null.rm.gen.txt \
+		$(foreach i,$(SEEDS_LEDUC),\
+		$(foreach n,$(NUM_PARTITIONS_GOOFSPIEL),\
+		$(foreach alg,$(ALGS_GOOFSPIEL),data/goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) \
+		data/leduc.null.rm.gen.txt \
+		$(foreach i,$(SEEDS_LEDUC),\
+		$(foreach n,$(NUM_PARTITIONS_LEDUC),\
+		$(foreach alg,$(ALGS_LEDUC),data/leduc.null.$(alg)-s10-n$(n).$(i).gen.txt))) \
+		data/random_goofspiel.null.rm.gen.txt \
+		$(foreach i,$(SEEDS_LEDUC),\
+		$(foreach n,$(NUM_PARTITIONS_RANDOM_GOOFSPIEL),\
+		$(foreach alg,$(ALGS_RANDOM_GOOFSPIEL),data/random_goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) | results
+	python3 bin/plot_aamas_figures.py
+
+
+.PHONY: aamas
+
+aamas: results/Error.pdf results/Plateau.pdf results/Exploitability_Num_Part.pdf
+
+$(EXE):
+	cd $(FRCFR) && $(MAKE)
+
+data/%.gen.txt: | data
+	python3 bin/run_experiment.py --exe $(EXE) -a $* > $@
+
+runs_remaining.gen.sh: bin/list_runs_remaining.sh Makefile
+	$< > $@
+
+print-%:
+	@echo $* = $($*)

README.md

@@ -0,0 +1,15 @@
+# Generalized RCFR Data
+
+Generalized RCFR experiment data for AAMAS 2020 submission.
+
+
+## Installation
+
+Experiments and plotting requires Python3 and some associated libraries. Run `pip install -r requirements.txt` to install the libraries.
+
+
+## Managing Experiments and Analysis
+
+To generate the desired file, simply run the corresponding make command. E.g., `make data/leduc.null.rm.gen.txt`.
+Generating the figures in the paper can be done with the following command `make aamas`. Alternatively,
+once all the appropriate data files have been created, the figures in the paper may be generated by running `python bin/plot_aamas_figures.py`.

bin/list_runs_remaining.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+make -n | grep run_experiment.py

bin/run_experiment.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+import os
+import sys
+from absl import app
+from absl import flags
+import frcfr_data
+
+flags.DEFINE_string("exe", None, "The experiment executable to run.")
+flags.mark_flag_as_required('exe')
+flags.DEFINE_string("a", None, "The experiment parameters.")
+flags.mark_flag_as_required('a')
+
+
+def run_experiment(argv):
+    del argv  # Unused.
+
+    x_params = frcfr_data.ExperimentParameters(flags.FLAGS.a)
+    command = x_params.command(flags.FLAGS.exe)
+    print(command, file=sys.stderr, flush=True)
+    os.system(command)
+
+
+if __name__ == '__main__':
+    app.run(run_experiment)

bin/save_data.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+from os import path
+import glob
+
+from absl import app
+from absl import flags
+import numpy as np
+
+flags.DEFINE_list(
+    "i", [],
+    "The data files to compile together into a numpy data file. "
+    "Defaults to all of them."
+)
+flags.DEFINE_string("o", None, "The name of the output numpy data file.")
+flags.mark_flag_as_required('o')
+
+
+ALG_NAME_MAP = {
+    'rm': 'RM',
+    'hedge_s': 'S-Hedge',
+    'hedge_m': 'M-Hedge',
+    'hedge_h': 'H-Hedge',
+    'ada_normal_hedge': 'AdaNormalHedge',
+    'elm_mu': 'MU-ELM',
+    'simple_ada_normal_hedge': 'SimpleANH',
+    'max_simple_ada_normal_hedge': 'MaxSANH',
+    'hedge_k': 'K-Hedge',
+}
+
+GAME_NAME_MAP = {
+    'leduc': 'Leduc Poker',
+    'liars_dice': "Liar's Dice",
+    'goofspiel3': 'Goofspiel(3)',
+    'goofspiel4': 'Goofspiel(4)'
+}
+
+SAMPLER_NAME_MAP = {
+    'null': 'No',
+    'external': "External",
+    'outcome': 'Outcome(0.6)'
+}
+
+
+def read_data_file(file_name):
+    file_name_components = file_name.split('.')
+    try:
+        seed = int(file_name_components[3])
+    except:
+        seed = None
+
+    final_data = {
+        'game':
+        file_name_components[0],
+        'sampler':
+        file_name_components[1],
+        '_alg':
+        file_name_components[2],
+        '_seed': seed,
+    }
+
+    num_iterations_completed = []
+    exploitability_cur = []
+    exploitability_avg = []
+    nodes_touched = []
+    with open(file_name) as file:
+        for line in file:
+            if line[0] == '#': continue
+            t, data = line.split(':')
+            num_iterations_completed.append(int(t))
+
+            data_components = [datum.strip() for datum in data.split(',')]
+            exploitability_cur.append(float(data_components[0]))
+            exploitability_avg.append(float(data_components[1]))
+
+            if len(data_components) > 2:
+                nodes_touched.append(int(data_components[2]))
+
+    final_data['num_iterations_completed'] = np.array(num_iterations_completed)
+    final_data['exploitability_cur'] = np.array(exploitability_cur) * 1000
+    final_data['exploitability_avg'] = np.array(exploitability_avg) * 1000
+    final_data['nodes_touched'] = np.array(nodes_touched)
+    final_data['label'] = ALG_NAME_MAP[final_data['_alg']]
+    final_data['is_one_of_many_reps'] = final_data['_seed'] is not None
+    return final_data
+
+
+def save_data(argv):
+    del argv  # Unused.
+
+    if len(flags.FLAGS.i) < 1:
+        results_files = glob.glob('*.gen.txt')
+    else:
+        results_files = flags.FLAGS.i
+
+    data = sorted(
+        [read_data_file(file) for file in results_files if path.isfile(file)],
+        key=lambda results: results['label'])
+
+    np.save(flags.FLAGS.o, data)
+    print('Saved data to {}'.format(flags.FLAGS.o))
+
+
+if __name__ == '__main__':
+    app.run(save_data)

bin/sbatch_run_experiments.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+make runs_remaining.gen.sh
+sbatch bin/sbatch_config.sh

mal_cfr_data/__init__.py

@@ -0,0 +1,86 @@
+class ExperimentParameters():
+    GAME_MAP = {
+        'leduc':
+        'leduc_poker',
+        'goofspiel':
+        'goofspiel(imp_info=True,num_cards=5,points_order=descending)',
+        'random_goofspiel':
+        'goofspiel(imp_info=True,num_cards=4,points_order=random)',
+    }
+    NUM_ITERATIONS_MAP = {
+        'leduc': 100000,
+        'goofspiel': 100000,
+        'random_goofspiel': 100000,
+    }
+    REPORT_GAP_FACTOR_MAP = {
+        'leduc': 2,
+        'goofspiel': 2,
+        'random_goofspiel': 2,
+    }
+    ALG_MAP = {
+        'rm': 'LsRcfrRm',
+        'hedge': 'LsRcfrHedge',
+        'poly': 'LsRcfrPoly',
+    }
+
+    def __init__(self, args):
+        args = args.split('.')
+        self.game = args[0].replace('data/', '')
+        alg_and_args = args[2]
+
+        if len(args) > 3:
+            self.seed = args[3]
+
+        alg_and_args = alg_and_args.split('-')
+        self.alg = alg_and_args[0]
+
+        self.alg_args = {}
+        if len(alg_and_args) > 1:
+            self.alg_args = alg_and_args[1:]
+            self.alg_args = {
+                a[0]: a[1:].replace('_', '.')
+                for a in self.alg_args
+            }
+
+    def is_tabular(self):
+        return 'n' not in self.alg_args
+
+    def num_iterations(self):
+        return self.NUM_ITERATIONS_MAP[self.game]
+
+    def report_gap_factor(self):
+        return self.REPORT_GAP_FACTOR_MAP[self.game]
+
+    def command(self, exe):
+        if self.is_tabular():
+            return ('time {exe} '
+                    "--game '{game}' "
+                    '-t {t} '
+                    '--alg CfrRm '
+                    '--report_gap_factor {report_gap_factor} ').format(
+                        exe=exe,
+                        game=self.GAME_MAP[self.game],
+                        t=self.num_iterations(),
+                        report_gap_factor=self.report_gap_factor())
+        command = ('time {exe} '
+                   "--game '{game}' "
+                   '-t {t} '
+                   '--alg {alg} '
+                   '--fractional_partition_size "{s:0.3f}" '
+                   '--num_partitions {n} '
+                   '--random_seed {random_seed} '
+                   '--report_gap_factor {report_gap_factor} ').format(
+                       exe=exe,
+                       game=self.GAME_MAP[self.game],
+                       t=self.num_iterations(),
+                       alg=self.ALG_MAP[self.alg],
+                       s=float(self.alg_args['s']) / 100.0,
+                       n=self.alg_args['n'],
+                       random_seed=self.seed,
+                       report_gap_factor=self.report_gap_factor())
+
+        if 't' in self.alg_args:
+            command += '--temperature {}'.format(self.alg_args['t'])
+        if 'p' in self.alg_args:
+            command += '--p_exponent {}'.format(self.alg_args['p'])
+        return command

requirements.txt

@@ -0,0 +1 @@
+-e .

setup.cfg

@@ -0,0 +1,4 @@
+[aliases]
+test=pytest
+[flake8]
+ignore = E701, F403, E704, E301, E402, E501, E241, E201, W503, W504