diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e90fd6a --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +.eggs/ +*.egg-info/ + + # C extensions +*.so + + # Jupyter Notebook +.ipynb_checkpoints + + # virtualenv +.venv +venv/ +ENV/ +env/ + + # OSX specific +.DS_Store + + # Build products +build/ +build.*/ +build + + # External modules +open_spiel/ +open_spiel/abseil-cpp/ +open_spiel/games/bridge/double_dummy_solver/ +pybind11/ +eigen/ + +# Install artifacts +get-pip.py + +# IDE +.idea/ +compile_commands.json +.clangd + +# Singularity images +*.sif + +# Misc. +saved_results/ +manifest.db +*.out +runs_remaining.gen.sh +results/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1d34b1e --- /dev/null +++ b/Makefile @@ -0,0 +1,86 @@ +FRCFR =f-rcfr +EXE :=$(FRCFR)/build/bin/run_cfr + +default: results/Error.pdf results/Plateau.pdf results/Exploitability_Num_Part.pdf + @true + +SEEDS_LEDUC :=$(shell seq 1 5) +NUM_PARTITIONS_LEDUC :=10 20 30 40 50 +ALGS_LEDUC :=hedge-t0_01 hedge-t0_05 hedge-t0_1 hedge-t0_5 hedge-t1 \ + poly-p1_1 poly-p1_5 poly-p2_5 poly-p3 rm + +NUM_PARTITIONS_GOOFSPIEL :=20 40 50 60 +ALGS_GOOFSPIEL :=hedge-t0_1 hedge-t0_5 hedge-t1 hedge-t5 hedge-t10 \ + poly-p1_1 poly-p1_5 poly-p2_5 poly-p3 rm + +NUM_PARTITIONS_RANDOM_GOOFSPIEL :=30 60 90 120 +ALGS_RANDOM_GOOFSPIEL :=hedge-t0_01 hedge-t0_05 hedge-t0_1 hedge-t0_5 hedge-t1 \ + poly-p1_1 poly-p1_5 poly-p2_5 poly-p3 rm + +data: + mkdir data + +results: + mkdir results + +results/Error.pdf: \ + data/goofspiel.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_GOOFSPIEL),\ + $(foreach alg,$(ALGS_GOOFSPIEL),data/goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) \ + data/leduc.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_LEDUC),\ + $(foreach alg,$(ALGS_LEDUC),data/leduc.null.$(alg)-s10-n$(n).$(i).gen.txt))) \ + data/random_goofspiel.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_RANDOM_GOOFSPIEL),\ + $(foreach alg,$(ALGS_RANDOM_GOOFSPIEL),data/random_goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) | results + python3 bin/plot_aamas_figures.py + +results/Exploitability_Num_Part.pdf: \ + data/goofspiel.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_GOOFSPIEL),\ + $(foreach alg,$(ALGS_GOOFSPIEL),data/goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) \ + data/leduc.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_LEDUC),\ + $(foreach alg,$(ALGS_LEDUC),data/leduc.null.$(alg)-s10-n$(n).$(i).gen.txt))) \ + data/random_goofspiel.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_RANDOM_GOOFSPIEL),\ + $(foreach alg,$(ALGS_RANDOM_GOOFSPIEL),data/random_goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) | results + python3 bin/plot_aamas_figures.py + +results/Plateau.pdf: \ + data/goofspiel.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_GOOFSPIEL),\ + $(foreach alg,$(ALGS_GOOFSPIEL),data/goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) \ + data/leduc.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_LEDUC),\ + $(foreach alg,$(ALGS_LEDUC),data/leduc.null.$(alg)-s10-n$(n).$(i).gen.txt))) \ + data/random_goofspiel.null.rm.gen.txt \ + $(foreach i,$(SEEDS_LEDUC),\ + $(foreach n,$(NUM_PARTITIONS_RANDOM_GOOFSPIEL),\ + $(foreach alg,$(ALGS_RANDOM_GOOFSPIEL),data/random_goofspiel.null.$(alg)-s10-n$(n).$(i).gen.txt))) | results + python3 bin/plot_aamas_figures.py + + +.PHONY: aamas + +aamas: results/Error.pdf results/Plateau.pdf results/Exploitability_Num_Part.pdf + +$(EXE): + cd $(FRCFR) && $(MAKE) + +data/%.gen.txt: | data + python3 bin/run_experiment.py --exe $(EXE) -a $* > $@ + +runs_remaining.gen.sh: bin/list_runs_remaining.sh Makefile + $< > $@ + +print-%: + @echo $* = $($*) diff --git a/README.md b/README.md new file mode 100644 index 0000000..3552584 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# Generalized RCFR Data + +Generalized RCFR experiment data for AAMAS 2020 submission. + + +## Installation + +Experiments and plotting requires Python3 and some associated libraries. Run `pip install -r requirements.txt` to install the libraries. + + +## Managing Experiments and Analysis + +To generate the desired file, simply run the corresponding make command. E.g., `make data/leduc.null.rm.gen.txt`. +Generating the figures in the paper can be done with the following command `make aamas`. Alternatively, +once all the appropriate data files have been created, the figures in the paper may be generated by running `python bin/plot_aamas_figures.py`. diff --git a/bin/list_runs_remaining.sh b/bin/list_runs_remaining.sh new file mode 100755 index 0000000..1f6b556 --- /dev/null +++ b/bin/list_runs_remaining.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +make -n | grep run_experiment.py diff --git a/bin/run_experiment.py b/bin/run_experiment.py new file mode 100755 index 0000000..45c636d --- /dev/null +++ b/bin/run_experiment.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import os +import sys +from absl import app +from absl import flags +import frcfr_data + +flags.DEFINE_string("exe", None, "The experiment executable to run.") +flags.mark_flag_as_required('exe') +flags.DEFINE_string("a", None, "The experiment parameters.") +flags.mark_flag_as_required('a') + + +def run_experiment(argv): + del argv # Unused. + + x_params = frcfr_data.ExperimentParameters(flags.FLAGS.a) + command = x_params.command(flags.FLAGS.exe) + print(command, file=sys.stderr, flush=True) + os.system(command) + + +if __name__ == '__main__': + app.run(run_experiment) diff --git a/bin/save_data.py b/bin/save_data.py new file mode 100755 index 0000000..6f32b06 --- /dev/null +++ b/bin/save_data.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +from os import path +import glob + +from absl import app +from absl import flags +import numpy as np + +flags.DEFINE_list( + "i", [], + "The data files to compile together into a numpy data file. " + "Defaults to all of them." +) +flags.DEFINE_string("o", None, "The name of the output numpy data file.") +flags.mark_flag_as_required('o') + + +ALG_NAME_MAP = { + 'rm': 'RM', + 'hedge_s': 'S-Hedge', + 'hedge_m': 'M-Hedge', + 'hedge_h': 'H-Hedge', + 'ada_normal_hedge': 'AdaNormalHedge', + 'elm_mu': 'MU-ELM', + 'simple_ada_normal_hedge': 'SimpleANH', + 'max_simple_ada_normal_hedge': 'MaxSANH', + 'hedge_k': 'K-Hedge', +} + +GAME_NAME_MAP = { + 'leduc': 'Leduc Poker', + 'liars_dice': "Liar's Dice", + 'goofspiel3': 'Goofspiel(3)', + 'goofspiel4': 'Goofspiel(4)' +} + +SAMPLER_NAME_MAP = { + 'null': 'No', + 'external': "External", + 'outcome': 'Outcome(0.6)' +} + + +def read_data_file(file_name): + file_name_components = file_name.split('.') + try: + seed = int(file_name_components[3]) + except: + seed = None + + final_data = { + 'game': + file_name_components[0], + 'sampler': + file_name_components[1], + '_alg': + file_name_components[2], + '_seed': seed, + } + + num_iterations_completed = [] + exploitability_cur = [] + exploitability_avg = [] + nodes_touched = [] + with open(file_name) as file: + for line in file: + if line[0] == '#': continue + t, data = line.split(':') + num_iterations_completed.append(int(t)) + + data_components = [datum.strip() for datum in data.split(',')] + exploitability_cur.append(float(data_components[0])) + exploitability_avg.append(float(data_components[1])) + + if len(data_components) > 2: + nodes_touched.append(int(data_components[2])) + + final_data['num_iterations_completed'] = np.array(num_iterations_completed) + final_data['exploitability_cur'] = np.array(exploitability_cur) * 1000 + final_data['exploitability_avg'] = np.array(exploitability_avg) * 1000 + final_data['nodes_touched'] = np.array(nodes_touched) + final_data['label'] = ALG_NAME_MAP[final_data['_alg']] + final_data['is_one_of_many_reps'] = final_data['_seed'] is not None + return final_data + + +def save_data(argv): + del argv # Unused. + + if len(flags.FLAGS.i) < 1: + results_files = glob.glob('*.gen.txt') + else: + results_files = flags.FLAGS.i + + data = sorted( + [read_data_file(file) for file in results_files if path.isfile(file)], + key=lambda results: results['label']) + + np.save(flags.FLAGS.o, data) + print('Saved data to {}'.format(flags.FLAGS.o)) + + +if __name__ == '__main__': + app.run(save_data) diff --git a/bin/sbatch_run_experiments.sh b/bin/sbatch_run_experiments.sh new file mode 100755 index 0000000..8009093 --- /dev/null +++ b/bin/sbatch_run_experiments.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +make runs_remaining.gen.sh +sbatch bin/sbatch_config.sh diff --git a/mal_cfr_data/__init__.py b/mal_cfr_data/__init__.py new file mode 100644 index 0000000..defbc33 --- /dev/null +++ b/mal_cfr_data/__init__.py @@ -0,0 +1,86 @@ +class ExperimentParameters(): + GAME_MAP = { + 'leduc': + 'leduc_poker', + 'goofspiel': + 'goofspiel(imp_info=True,num_cards=5,points_order=descending)', + 'random_goofspiel': + 'goofspiel(imp_info=True,num_cards=4,points_order=random)', + } + NUM_ITERATIONS_MAP = { + 'leduc': 100000, + 'goofspiel': 100000, + 'random_goofspiel': 100000, + } + REPORT_GAP_FACTOR_MAP = { + 'leduc': 2, + 'goofspiel': 2, + 'random_goofspiel': 2, + } + ALG_MAP = { + 'rm': 'LsRcfrRm', + 'hedge': 'LsRcfrHedge', + 'poly': 'LsRcfrPoly', + } + + def __init__(self, args): + args = args.split('.') + self.game = args[0].replace('data/', '') + alg_and_args = args[2] + + if len(args) > 3: + self.seed = args[3] + + alg_and_args = alg_and_args.split('-') + self.alg = alg_and_args[0] + + self.alg_args = {} + if len(alg_and_args) > 1: + self.alg_args = alg_and_args[1:] + self.alg_args = { + a[0]: a[1:].replace('_', '.') + for a in self.alg_args + } + + def is_tabular(self): + return 'n' not in self.alg_args + + def num_iterations(self): + return self.NUM_ITERATIONS_MAP[self.game] + + def report_gap_factor(self): + return self.REPORT_GAP_FACTOR_MAP[self.game] + + def command(self, exe): + if self.is_tabular(): + return ('time {exe} ' + "--game '{game}' " + '-t {t} ' + '--alg CfrRm ' + '--report_gap_factor {report_gap_factor} ').format( + exe=exe, + game=self.GAME_MAP[self.game], + t=self.num_iterations(), + report_gap_factor=self.report_gap_factor()) + command = ('time {exe} ' + "--game '{game}' " + '-t {t} ' + '--alg {alg} ' + '--fractional_partition_size "{s:0.3f}" ' + '--num_partitions {n} ' + '--random_seed {random_seed} ' + '--report_gap_factor {report_gap_factor} ').format( + exe=exe, + game=self.GAME_MAP[self.game], + t=self.num_iterations(), + alg=self.ALG_MAP[self.alg], + s=float(self.alg_args['s']) / 100.0, + n=self.alg_args['n'], + random_seed=self.seed, + report_gap_factor=self.report_gap_factor()) + + if 't' in self.alg_args: + command += '--temperature {}'.format(self.alg_args['t']) + if 'p' in self.alg_args: + command += '--p_exponent {}'.format(self.alg_args['p']) + return command diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d6e1198 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5ac9670 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[aliases] +test=pytest +[flake8] +ignore = E701, F403, E704, E301, E402, E501, E241, E201, W503, W504 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9c7891d --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup, find_packages + +setup( + name='frcfr_data', + version='0.0.1', + license='', + packages=find_packages(), + install_requires=[ + 'setuptools >= 20.2.2', + 'absl-py', + 'matplotlib', + 'seaborn', + 'numpy', + 'pandas', + ], + tests_require=['pytest', 'pytest-cov'], + setup_requires=['pytest-runner'], +)