From a90222263c367941b856a2f98a90b865335384af Mon Sep 17 00:00:00 2001
From: Jordan Laser <jordan.science.laser@gmail.com>
Date: Thu, 14 Nov 2024 09:14:06 -0700
Subject: [PATCH 1/3] add timeout

---
 research_datastream/terraform/GETTING_STARTED.md |   5 +++--
 .../executions/execution_datastream_example.json |   3 ++-
 .../executions/execution_gp_example.json         |   5 +++--
 .../execution_template_datastream.json           |   3 ++-
 .../lambda_functions/checker_lambda.zip          | Bin 623 -> 0 bytes
 .../lambda_functions/commander_lambda.zip        | Bin 1446 -> 0 bytes
 .../lambda_functions/poller/lambda_function.py   |   7 ++++++-
 .../terraform/lambda_functions/poller_lambda.zip | Bin 686 -> 0 bytes
 .../start_ami/lambda_function.py                 |   6 ++++++
 .../lambda_functions/starter_lambda.zip          | Bin 982 -> 0 bytes
 .../lambda_functions/stopper_lambda.zip          | Bin 606 -> 0 bytes
 11 files changed, 22 insertions(+), 7 deletions(-)
 delete mode 100644 research_datastream/terraform/lambda_functions/checker_lambda.zip
 delete mode 100644 research_datastream/terraform/lambda_functions/commander_lambda.zip
 delete mode 100644 research_datastream/terraform/lambda_functions/poller_lambda.zip
 delete mode 100644 research_datastream/terraform/lambda_functions/starter_lambda.zip
 delete mode 100644 research_datastream/terraform/lambda_functions/stopper_lambda.zip

diff --git a/research_datastream/terraform/GETTING_STARTED.md b/research_datastream/terraform/GETTING_STARTED.md
index f72757e72..ee4de37c2 100644
--- a/research_datastream/terraform/GETTING_STARTED.md
+++ b/research_datastream/terraform/GETTING_STARTED.md
@@ -100,11 +100,12 @@ Starting from execution_template_general_purpose. Make sure to wrap commands in
 ```
 
 ### Edit Run Options
-The state machine is capable of confirming a complete execution by checking for the existence output data in the form of an s3 object. Set booleans here. If `s3_bucket` and `s3_prefix` are provided in `datastream_command_options`, `ngen-datastream` will create a `ngen-run.tar.gz` file that can be found at `s3://<s3_bucket>/<s3_prefix>/ngen-run.tar.gz`
+The state machine is capable of confirming a complete execution by checking for the existence output data in the form of an s3 object. Set booleans here. If `s3_bucket` and `s3_prefix` are provided in `datastream_command_options`, `ngen-datastream` will create a `ngen-run.tar.gz` file that can be found at `s3://<s3_bucket>/<s3_prefix>/ngen-run.tar.gz`. `timeout_s` is a timeout for the commands issued during execution. This is valuable for shutting down hanging instances that may become unresponsive due to memory overflow, etc. Default is 3600.
 ```
   "run_options":{
     "ii_delete_volume" : false,
-    "ii_check_s3"      : true
+    "ii_check_s3"      : true,
+    "timeout_s"        : 3600
 },
 ```
 
diff --git a/research_datastream/terraform/executions/execution_datastream_example.json b/research_datastream/terraform/executions/execution_datastream_example.json
index 773f7103d..72d04d904 100644
--- a/research_datastream/terraform/executions/execution_datastream_example.json
+++ b/research_datastream/terraform/executions/execution_datastream_example.json
@@ -13,7 +13,8 @@
 },
   "run_options":{
     "ii_delete_volume" : true,
-    "ii_check_s3"      : true
+    "ii_check_s3"      : true,
+    "timeout_s"        : 3600
 },
 "instance_parameters": {
   "ImageId": "ami-062bdcbb454b8d833",
diff --git a/research_datastream/terraform/executions/execution_gp_example.json b/research_datastream/terraform/executions/execution_gp_example.json
index 1db212df8..3e6c7e86c 100644
--- a/research_datastream/terraform/executions/execution_gp_example.json
+++ b/research_datastream/terraform/executions/execution_gp_example.json
@@ -4,11 +4,12 @@
 ],
 "run_options":{
   "ii_delete_volume" : false,
-  "ii_check_s3"      : false
+  "ii_check_s3"      : false,
+  "timeout_s"        : 3600
 },
 "instance_parameters" :
 {
-  "ImageId"            : "ami-03b72f226b125860d",
+  "ImageId"            : "ami-07161bb3f4b6e5b6d",
   "InstanceType"       : "t4g.large",
   "KeyName"            : "jlaser_west2",
   "SecurityGroupIds"   : ["sg-04365a4248fe126bc"],
diff --git a/research_datastream/terraform/executions/execution_template_datastream.json b/research_datastream/terraform/executions/execution_template_datastream.json
index 5b5cc5e51..080cbdabc 100644
--- a/research_datastream/terraform/executions/execution_template_datastream.json
+++ b/research_datastream/terraform/executions/execution_template_datastream.json
@@ -13,7 +13,8 @@
 },
   "run_options":{
     "ii_delete_volume" : true,
-    "ii_check_s3"      : true
+    "ii_check_s3"      : true,
+    "timeout_s"        : 3600
 },
 "instance_parameters": {
   "ImageId": "",
diff --git a/research_datastream/terraform/lambda_functions/checker_lambda.zip b/research_datastream/terraform/lambda_functions/checker_lambda.zip
deleted file mode 100644
index 8b154961880d9a70f87c8943a283c09418c3546f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 623
zcmWIWW@Zs#-~d7f2E{HQ0S7`rR!(AWQc7ZcT4`Q#NoIbYUO{EgWdHuz1_DRl*K}p-
zsbzVE7IshZUj1!TkbiZz(&<LU9gT*Y+J&T^?hcn|Kalrdr`@B%=KbG$J{L-pbq-(L
z;JNt2p<Tx`PHFWXxwMpF_r`fW?i&kMMz{;i(>Ts#wKP)fm{;+iMoI5I8OJ>&j~zF&
zR`w1u&2#$L^S4U6l5uT2U*wEwMK=zbhg!0HSi0_tD3fGd%q_otIlqse{@`=Kvr~&-
z<zxE${j-w~2!GoxdD>~|##t76Vg5Jr9b@FL=P}tF+<kS^L)Vt{+mAwyd$;lG8@9hx
zTj!MX$ba!9y}08ntG77+c*VM9;f{z64tW=T>e**rV7VT7M0!t5z|vRA>rz%~tiN#V
zhj6F)1j(d5AF5RL9RBgBE;V}6vDv$?ChBjQ7TI|-a#x6q+(f25Cq%W6i-qbM2Vd{{
zRJ~c^#hz#LgKVcuJU9?=_3%ue=L<|^H!$XN=(QSJi5xq0zjwjqm?rP17MuZ7+TCtO
z=zL+7`IKYuQ0k4-qxw{fH%C=;_$D9Ou)+9S{?v#I9n5>!KTh=Xkk9ev)Sf-3uG7={
zkMPmyi5#>14zE~$@PpaAyJyAfXRH3c|9r>J=6>A>%RBo%{mo08EN^~Mlb!e0p&L@4
zRV04Nci*|S;n~M8<-myk|3AQ+ox{O!#`T+w3=B~$3=9F@j7%cTh@^om2TB?+u%r>h
W!b~Uu-mGjO8Ac#<0Me37ARYj3w+9XY

diff --git a/research_datastream/terraform/lambda_functions/commander_lambda.zip b/research_datastream/terraform/lambda_functions/commander_lambda.zip
deleted file mode 100644
index 7c2fcd7034809d2be7346c34bae8809ab9770209..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1446
zcmV;X1zGw~O9KQH00;mG0000XiU0rr000000000001^NI0Bm7xVq{@oW_503bZKvH
zE^v9Q7h7+mxca@n;ydVQNY0d`m!tDgiIIBdqOG*E8nvU{hirr#^O1N9HnL6HF3Nvj
z8DlVz%uH9y3;4U={lSV-NDUjI#r0Wx(5%3-Oo{?hqS2h7#klj?S&A7v5vG$&$b^cb
zAUsVZs#0);Gxu@MxYC4EOjtU<IRgM-8l{35E}Bz8o)ZVTw4kRQ%MrfH3N3?qw0!9g
z=5ZQX3NbistMI)AhwJY1E)CZM&Qdb2XFdOXfEAT4D-AKwoFUhVQpIA6y7Q3ny`V%h
z!Dj<L=+wo%=<^{70Nl0VxJzR%GGn^=5rebvzV#BsMia&{h3FmB*5E@p<k?6hj~xV`
zDVF-?s8o|f#<!3O>4MtDbFo;A<eJ<sRE}8AE{7PK#J4eA4#8QAdDo}<1}1TF)_l6X
z`}yOCky7X!YN{?L#U>@mj_@=`Im112eGXJ`jnBF*vLFbW_fM7xvWioaG|&Q=QWZL1
zm4OfrNUs`(0rxf+8<KmaaPGd5UUk$SD7JmHu%KCp^eQ};xMhMToD>*0#;!sO3fJ>S
zh;56$93Y04t^<}tk^)WeOlkL36s}wD41$zsqO?R(bk!<K<FpEZaVxT~fmAtcr|w%4
z`W|XcbMXt;5W}iR8%xx6vZ?4J>PB&|X;^`l1Y(b1<*|;t7BludA{LuJkm|(ZMhp`(
z7o;qar?Ws-yizD3%1zcI`%NJ%9#bYEDq*=31uh+K$+yU(J~T`7xhzD*br{TlOlNjl
zsgV<@YpAW;hrg$tmr_tQ3XcVPaZ#GcNx%&(F@5dln5QE$<ms3w6OuB%O_Zo4#S#4w
z>2XOUIs3Djpd%I!<I(T3WRFr!$#*ZQ6dBn_MjeIkYPC_QZ8Htrb?_;zeksRZbkq&f
zV{z@Fw1gN=r<bPR$tS#d1B<1il9gI5Eo`B75Gkme)Epo}sF9}!h~7gzjOziSJE%J=
z4-m2LXe>TJw1>J6;Q^u){yi(7(zz`p&yeUIYA*!`h&a@?gbu-vwfAtY`!RZJf&|gS
zKzUQe5DBPl2YYHpyy5R*czOW2F7ufD!O9Z4L;9FVk(>OQ#iBvjJ!63W#=>#>=5x1Y
zzce7dTBK-_y1~TFa5Y*O2HSv&R}(A@Z-ohdIPZ>5qP@tg!Z0^ex?hv-QK8&7c{Ufy
zg?Fd8^Z#=}A3xxpQT$BGlJTv>eB3mb(65c`6YG^zHQEuf*V*u(iAJOA7gC^cAqLZF
zobhdNGl_@WA)FLo3R>!OXmV`!oV(l4U{ZZ#$6%yL`Wsh@j5P)G3X{#MI1BGc;lN+4
zlyJTZZzT`c3svS!&%$LmUqzSe*Z2TWOz#@gp+08mcm*^MQ3|C#;n;V#jHw%4DyFd;
zb=fD9D?IJq?Ps&FWlR>Bo;Mqr@vv=R&yjCzu0a<g=TRz9@lZoSG~N9RzRDLPlxzv(
z!Q2jmewCZ-YDyKYVrS+11_P)x+2l;^jHwEFIyIHC#30<(bacx21pm;SF+V51*zZgj
zZ<ws7za}39tqSDYBsdOle|w1TEB>&<Jb!3zr*O*hrK^{??&tB%bV5&A{U^X@q?PHa
zudF~(>EH%dSx{g9@8aS=SN}Z(({l5l!vs5;I+NrKi__MaA5#l4UB0k2@KH)3fhfRZ
zrKq%=;k>P|SN;<Z3a<7r2h;jFsW{!CDYJ%f7+5>kf8hVOy9*`iO18fOXMY6%0RR6`
zO9u!9eUe%b1pom04*&pAO928D0~7!V00;m803nJ5eUe%b1pom04*&oX0000000000
z0001_fdBvi0Bm7xVq{@oW_503bZKvHE^v8JO9ci1000010096%0000`1poj50JZ>^
A=>Px#

diff --git a/research_datastream/terraform/lambda_functions/poller/lambda_function.py b/research_datastream/terraform/lambda_functions/poller/lambda_function.py
index fdc7bcde6..fc39800e6 100644
--- a/research_datastream/terraform/lambda_functions/poller/lambda_function.py
+++ b/research_datastream/terraform/lambda_functions/poller/lambda_function.py
@@ -20,6 +20,7 @@ def lambda_handler(event, context):
     Generic Poller funcion    
     """
     t0 = time.perf_counter()
+    timeout_s = event['run_options']['timeout_s']
 
     global client_ssm, client_ec2
     client_ssm = boto3.client('ssm',region_name=event['region'])
@@ -40,9 +41,13 @@ def lambda_handler(event, context):
         elif output['Status'] == 'InProgress':
             ii_pass = False   
             print(f'Commands are still in progress. Waiting 5 seconds and checking again')
-            if (time.perf_counter() - t0) > 850: 
+            if (time.perf_counter() - t0) > 800: 
                 print(f'Cycling...')
                 ii_time = True
+            duration = time.time() - event['t0']
+            if duration >= timeout_s:
+                print(f'Duration -> {duration}\nTimeout -> {timeout_s}')
+                raise Exception(f'Commands duration have exceed the timeout specified in the execution')
             time.sleep(5)                            
         else:
             raise Exception(f'Command failed {output}')         
diff --git a/research_datastream/terraform/lambda_functions/poller_lambda.zip b/research_datastream/terraform/lambda_functions/poller_lambda.zip
deleted file mode 100644
index 54256bf28b0a6e561244a6cc0f3545534db2ac20..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 686
zcmWIWW@Zs#-~d7f2E{HQ0S7`rR!(AWQc7ZcT4`Q#NoIbYUP0xQ5Z`=ZLxH`|!+G{r
zD7SK)T6?KzrN$}NP%WW$jVQ}+DaNa=$KBk#l;h+5iEsbrcr(fiRX_WA_H&Jn>Dtdc
z?#JI9US)4!x+5yn#<4d>^Sa5USErI+@Q5Ft_@yl;AyYJL$@Oq;hK4VP|GC>8eQK3w
z-6k|eaZ60=g`DSWQsn1M@;N8=c<zpMo9maKjy%Acblv^_%ObUu&d7dstFJ-d9`&h~
zJIXZZyg3@XT%=9GOd(H2@=9U3?9E623bsDTwvE*DTNKWoF6ppyndylY58t}~nA!YC
z$kND8E9YlxPEDa$_{ldDt@Eq>&+VFcHt_T-owa5!xN96vO9z}YvMZYZvb($c%YRAT
zs-S|`J&yd#&uJBjwMDr^+`4j1cInK}MB_Jsv1&)2G#PzAY&M%q&`H@$F1u*krcDQp
zJa2X92(^Em!*KPfF;mF&^};F#XQt|yJ8;_O8Fe{3oz=2i=$5R#H)p~{_QD(8D+7O>
zP8R*QckgDiMXRPNI&hj_V)5DbKHZ4p+Q$dA3vc}wOyBW4X#X_Tp0~fg@8c2dl=>>Y
zF|J{Gc+FQ?y%_iYPjy=s{G8l1ckkDA-_Ez%oV;+M?Qqp1<AyIBjVEVV^yHr3*_uCT
zMh>r#)FxTwBkn7WSN%>|t9nztzw7>*T!W6>kN@sU#YJv@&Kk4o`y{4A%T2a?>^4#T
zGR2N}^T)VLPZQtoHdwBZmgRB0=7O+R{4ZGshX4NqyxBQ={#|@0!NkB&!^*%A;LXS+
l!i-2+$a0{R1p`YOK`hL47U0dw29jX}LI)rn0W_C^0RRvuCRhLf

diff --git a/research_datastream/terraform/lambda_functions/start_ami/lambda_function.py b/research_datastream/terraform/lambda_functions/start_ami/lambda_function.py
index 0958ea543..9ac4fc57b 100644
--- a/research_datastream/terraform/lambda_functions/start_ami/lambda_function.py
+++ b/research_datastream/terraform/lambda_functions/start_ami/lambda_function.py
@@ -34,6 +34,12 @@ def replace_in_dict(d, pattern, replacement):
     
 def lambda_handler(event, context):
 
+    t0 = time.time()
+    event['t0'] = t0
+    if not "timeout_s" in event['run_options']:
+        print(f'Setting timeout_s to default 3600 seconds')
+        event['run_options']['timeout_s'] = 3600
+
     event['region'] = os.environ['AWS_REGION']
     global client_ec2
     client_ec2 = boto3.client('ec2',region_name=event['region'])
diff --git a/research_datastream/terraform/lambda_functions/starter_lambda.zip b/research_datastream/terraform/lambda_functions/starter_lambda.zip
deleted file mode 100644
index bda47efd5c58ba5f6ffe5f81226af61487ec60cd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 982
zcmWIWW@Zs#-~d7f2E{HQ0S7`rR!(AWQc7ZcT4`Q#NoIbYUP0xYu=m-I%_RQK+y6*)
z&53=Im&@LG&$OQsm&da0-VU~7f=z*IuJ?vT8%?@nCi8tizvnic^vLjK{uh6L{;2%2
z^6+gxE#~UXC;r`u$ras_9(z|z&1np5zQ)vCen(%7C%a<dohe&GRK;%v1SYQJHQnoX
zXx?U?J-W@)_D$pJeYB!`wfF8zvsoDqB(yN;-Tn9UfHcpw_(v}DA51P$Z*|jHkY&LY
zoK!m3_{!8r)jS;~LPiV;wh`BT#Bbk73}>+_dwWQYcYlTKul$qZS60kh`7kJB)9tu-
z+fv>bzmw-N^(c!j@{d-r*5Ha*7000`H{IM=X4;(?^N!e0#a5p+O!f+G2()FhHj{e(
zbMB)<DqlZce*0_tRriXA&kO>;=KlFwYk9TLDsHLZ-g$rJxpzdlytTjR#=m_1hU^tf
zy{Ghqt0hX7T#*P-lCIU|GL|XQ6}5S2*qnar3r7sY>756|)L;6Z(Z2rJQbo^*Iof8g
z<-??VRUJ0B?UY{UROW_og={dq?Nq$3(tBI&p~rk&?@zb$o&2A5v!M2u>$G|MHyr#H
zf9tUQjlzrRHzluYdOIJxJ>hsMSFMNL@oB|#+XW(u4c{l6y}PDuTeaJ1X3;f{Qy&;}
zO?FgmJsAFbqu!;+?Jf_Mryts<Il;>P<C&=<(M7+XJO51m`uf*r+mHM|HcqU*<r4L9
z&1EsJ_02oYRasVNPx|?4=>lCbb>7R8JqmKNp<7&=PwlJyWL>*=@?HbJ`&X-lc3Aip
zFjl!`e@qkQKjUR8;<&L`F@BOs#i?DU&F0tLu9~SRipT5U;&!|A@aXyV{l_Qj*8iy3
z9jUuH;F<lR>)-ij-~PQjbmz}s?Y^J;JeAFl^D)nMT=giw_e<<orAgoT<!cQ44|v~?
zICxvhcm6BiOSP*mU#^je{IdB(*SnH~y;kS)t62i1SNX6MniVRU?7kBBZO4f?#oAd@
z9&bH#ZqC)RO@T6!JfE5OxrXjf4+y@ea5`~u-!*S}O>xVa``_<;QoSfk;L@JrMU&4j
zddTFt#Ja1fcSF&Nog$ulWA4VX|9Y;_Somz0=8~(5A;t9~1(Nb<zRx<@S0(mqELgaA
z1>cD|BD_8m%J-i5(eXC5WBI=V*1HReL;vwHF#P`?;LXlq9dPKW12ZtcaWXIjcr!AI
lFe7pxvK%M}!oZS75DPOu26(fwfn*qg&;dv{0L^7!005%Rv+Mu>

diff --git a/research_datastream/terraform/lambda_functions/stopper_lambda.zip b/research_datastream/terraform/lambda_functions/stopper_lambda.zip
deleted file mode 100644
index 01ba661c4211cbc78c8554c2adb9e1e2df8bc07e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 606
zcmWIWW@Zs#-~d7f2E{HQ0S7`rR!(AWQc7ZcT4`Q#NoIbYUP0xYV8`w`1|s`@Ys<wg
zGMIb#o7sb<TcV!@d5J$$QkwZ>CtH%z$>Qy&4gT-VFwtVxz2beS`u^{IHSvpHRlVj|
zH|@=H6~U;53#E<<i!3bTS=zI(BB^qcM~=hicUqNqi<C`w@!Y>~fw9H3WxuAqDg#62
zEWg*~CNHKm%n!>hxxnM5k$uds#aBAjrcil9m=M<t4R5h-+wwimPR}DlH+_z=^A2l$
zc~8?h@^IX(Mvs}{6TaPbc)!Ns{o3Z+ZU4@F%1rxO^(EzZRpXDGr{7(7&Rp|p!MAgw
z3!9U?KX!$*S8bE4e)2Z(T+_l@nd$XjI&ETi?lEq;b6E6ZgZz<<jgnjbHAKF%cy0M*
zuc_R3_NgyggTvR{Un6Jr<z|cQ{`95CTe^+H@(RuC-mt%)Ft?%gq`%ju!*T@=s~pbz
z1)LL?n6P!ynpI(MY;#sS@B6rGj@gw*2_H|0-19cJpLB(H**06v?3$>lD}%PWJX$oD
zciB}<F8d`_GZoS~x1XqI&t+Y1>h{BXiuBwZ@uOAyk3M_!tx)%o>y>-o_wVYDZ`Pc*
z^P7b9&*qu=@o~>~{Chre!M7g*0aY`9nys{Wv%^TvqIX#!Q{)1%$NkklpH+SeGcf%B
zAK=Z-k=MX9=L91IgC7e6Lx49UlL#{+B_PXzQUVMtX#}w_(?x(cD;r3L5eOZCGzSxi
F2LQT20_p$&


From ffc5214b8cf785ce5fff6fb581b4329a326053f6 Mon Sep 17 00:00:00 2001
From: Jordan Laser <jordan.science.laser@gmail.com>
Date: Thu, 14 Nov 2024 10:08:18 -0700
Subject: [PATCH 2/3] add terminate instance option

---
 .../terraform/GETTING_STARTED.md              | 13 +++--
 .../execution_datastream_example.json         |  7 +--
 .../executions/execution_gp_example.json      |  7 +--
 .../execution_template_datastream.json        |  7 +--
 .../stopper/lambda_function.py                | 50 +++++++++++++------
 research_datastream/terraform/main.tf         |  1 +
 6 files changed, 58 insertions(+), 27 deletions(-)

diff --git a/research_datastream/terraform/GETTING_STARTED.md b/research_datastream/terraform/GETTING_STARTED.md
index ee4de37c2..68789abd3 100644
--- a/research_datastream/terraform/GETTING_STARTED.md
+++ b/research_datastream/terraform/GETTING_STARTED.md
@@ -100,14 +100,19 @@ Starting from execution_template_general_purpose. Make sure to wrap commands in
 ```
 
 ### Edit Run Options
-The state machine is capable of confirming a complete execution by checking for the existence output data in the form of an s3 object. Set booleans here. If `s3_bucket` and `s3_prefix` are provided in `datastream_command_options`, `ngen-datastream` will create a `ngen-run.tar.gz` file that can be found at `s3://<s3_bucket>/<s3_prefix>/ngen-run.tar.gz`. `timeout_s` is a timeout for the commands issued during execution. This is valuable for shutting down hanging instances that may become unresponsive due to memory overflow, etc. Default is 3600.
 ```
   "run_options":{
-    "ii_delete_volume" : false,
-    "ii_check_s3"      : true,
-    "timeout_s"        : 3600
+    "ii_terminate_instance" : true,
+    "ii_delete_volume"      : false,
+    "ii_check_s3"           : true,
+    "timeout_s"             : 3600
 },
 ```
+If `s3_bucket` and `s3_prefix` are provided in `datastream_command_options` and `ii_check_s3` is set to `true` , the state machine will confirm that at least one object exists at `s3://<s3_bucket>/<s3_prefix>/`. 
+
+`ii_terminate_instance` and `ii_delete_volume` allow the user to clean up AWS resources to avoid needless costs. While stopped instances do not incur costs, detached volumes do incur costs until deleted. 
+
+`timeout_s` is a timeout for the commands issued during execution. This is valuable for shutting down hanging instances that may become unresponsive due to memory overflow, etc. Default is 3600.
 
 ### Edit Instance Options
 4) Define the AMI ID. 
diff --git a/research_datastream/terraform/executions/execution_datastream_example.json b/research_datastream/terraform/executions/execution_datastream_example.json
index 72d04d904..48b84f0d1 100644
--- a/research_datastream/terraform/executions/execution_datastream_example.json
+++ b/research_datastream/terraform/executions/execution_datastream_example.json
@@ -12,9 +12,10 @@
     "s3_prefix"           : "test_directory"
 },
   "run_options":{
-    "ii_delete_volume" : true,
-    "ii_check_s3"      : true,
-    "timeout_s"        : 3600
+    "ii_delete_volume"      : true,
+    "ii_check_s3"           : true,
+    "ii_terminate_instance" : false,
+    "timeout_s"             : 3600
 },
 "instance_parameters": {
   "ImageId": "ami-062bdcbb454b8d833",
diff --git a/research_datastream/terraform/executions/execution_gp_example.json b/research_datastream/terraform/executions/execution_gp_example.json
index 3e6c7e86c..065dd0601 100644
--- a/research_datastream/terraform/executions/execution_gp_example.json
+++ b/research_datastream/terraform/executions/execution_gp_example.json
@@ -3,9 +3,10 @@
     "runuser -l ec2-user -c 'ls -la'"
 ],
 "run_options":{
-  "ii_delete_volume" : false,
-  "ii_check_s3"      : false,
-  "timeout_s"        : 3600
+  "ii_delete_volume"      : false,
+  "ii_terminate_instance" : true,
+  "ii_check_s3"           : false,
+  "timeout_s"             : 3600
 },
 "instance_parameters" :
 {
diff --git a/research_datastream/terraform/executions/execution_template_datastream.json b/research_datastream/terraform/executions/execution_template_datastream.json
index 080cbdabc..c5cb31595 100644
--- a/research_datastream/terraform/executions/execution_template_datastream.json
+++ b/research_datastream/terraform/executions/execution_template_datastream.json
@@ -12,9 +12,10 @@
     "s3_prefix"       : ""
 },
   "run_options":{
-    "ii_delete_volume" : true,
-    "ii_check_s3"      : true,
-    "timeout_s"        : 3600
+    "ii_terminate_instance" : true,
+    "ii_delete_volume"      : true,
+    "ii_check_s3"           : true,
+    "timeout_s"             : 3600
 },
 "instance_parameters": {
   "ImageId": "",
diff --git a/research_datastream/terraform/lambda_functions/stopper/lambda_function.py b/research_datastream/terraform/lambda_functions/stopper/lambda_function.py
index 7a0281ad7..afa554761 100644
--- a/research_datastream/terraform/lambda_functions/stopper/lambda_function.py
+++ b/research_datastream/terraform/lambda_functions/stopper/lambda_function.py
@@ -17,6 +17,20 @@ def confirm_detach(volume_id):
             time.sleep(1)
         else:
             return
+        
+def confirm_instance_termination(instance_id):
+    while True:
+        response = client_ec2.describe_instances(
+            InstanceIds=[
+                instance_id
+            ]
+        )
+        if response['Reservations'][0]['Instances'][0]['State']['Name'] != 'terminated':
+            print(f'Instance not yet terminated')
+            time.sleep(1)
+        else:
+            print(f'Instance {instance_id} terminated')
+            return        
 
 def lambda_handler(event, context):
     """
@@ -41,20 +55,28 @@ def lambda_handler(event, context):
     ],)
     print(response)
     volume_id=event['volume_id']
-    if event["run_options"]["ii_delete_volume"]:
-        print(f'Instance VolumeId {volume_id} located.')
-        response = client_ec2.detach_volume(
-            InstanceId=instance_id,
-            VolumeId=volume_id,
-            DryRun=False
+    if event["run_options"]["ii_terminate_instance"]:
+        response = client_ec2.terminate_instances(
+            InstanceIds=[
+                instance_id,
+            ],
         )
-        confirm_detach(volume_id)
-        print(f'EBS volume {instance_id} has been successfully detached.')
-        response = client_ec2.delete_volume(
-            VolumeId=volume_id,
-            DryRun=False
-        )   
-        print(f'EBS volume {instance_id} has been successfully deleted.')     
+        confirm_instance_termination(instance_id)
     else:
-        print(f"Volume {volume_id} remains attached or available and is still incurring costs.")
+        if event["run_options"]["ii_delete_volume"]:
+            print(f'Instance VolumeId {volume_id} located.')
+            response = client_ec2.detach_volume(
+                InstanceId=instance_id,
+                VolumeId=volume_id,
+                DryRun=False
+            )
+            confirm_detach(volume_id)
+            print(f'EBS volume {instance_id} has been successfully detached.')
+            response = client_ec2.delete_volume(
+                VolumeId=volume_id,
+                DryRun=False
+            )   
+            print(f'EBS volume {volume_id} has been successfully deleted.')     
+        else:
+            print(f"Volume {volume_id} remains attached or available and is still incurring costs.")
 
diff --git a/research_datastream/terraform/main.tf b/research_datastream/terraform/main.tf
index 5142988e6..a9a662e9a 100644
--- a/research_datastream/terraform/main.tf
+++ b/research_datastream/terraform/main.tf
@@ -124,6 +124,7 @@ resource "aws_iam_policy" "datastreamlambda_policy" {
           "ec2:StartInstances",
           "ec2:StopInstances",
           "ec2:DescribeInstances",
+          "ec2:TerminateInstances",
           "ec2:DescribeVolumes",
           "ec2:DeleteVolume",
           "ec2:DetachVolume",

From 245a010a3e02dc3b85abcf04fd520d2bdede5812 Mon Sep 17 00:00:00 2001
From: Jordan Laser <jordan.science.laser@gmail.com>
Date: Thu, 14 Nov 2024 11:38:26 -0700
Subject: [PATCH 3/3] update options

---
 .../test/execution_gp_arm_docker_buildNtester.json        | 8 +++++---
 research_datastream/terraform/test/execution_gp_test.json | 8 +++++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json b/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json
index b0c7d6580..490c1d879 100644
--- a/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json
+++ b/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json
@@ -9,12 +9,14 @@
     "runuser -l ec2-user -c 'aws s3 cp /home/ec2-user/ngen-datastream/docker_build_log.txt s3://ciroh-community-ngen-datastream/docker_build_log.txt'"
 ],
 "run_options":{
-  "ii_delete_volume" : true,
-  "check_s3"         : false
+  "ii_terminate_instance" : true,
+  "ii_delete_volume"      : true,
+  "ii_check_s3"           : false,
+  "timeout_s"             : 3600
 },
 "instance_parameters" :
 {
-  "ImageId"            : "ami-03b72f226b125860d",
+  "ImageId"            : "ami-07161bb3f4b6e5b6d",
   "InstanceType"       : "t4g.large",
   "KeyName"            : "actions_key_arm",
   "SecurityGroupIds"   : ["sg-0fcbe0c6d6faa0117"],
diff --git a/research_datastream/terraform/test/execution_gp_test.json b/research_datastream/terraform/test/execution_gp_test.json
index c088dc03e..24d45b30a 100644
--- a/research_datastream/terraform/test/execution_gp_test.json
+++ b/research_datastream/terraform/test/execution_gp_test.json
@@ -3,12 +3,14 @@
     "runuser -l ec2-user -c 'ls -la'"
 ],
 "run_options":{
-  "ii_delete_volume" : true,
-  "check_s3"         : false
+  "ii_terminate_instance" : true,
+  "ii_delete_volume"      : true,
+  "ii_check_s3"           : false,
+  "timeout_s"             : 3600
 },
 "instance_parameters" :
 {
-  "ImageId"            : "ami-03b72f226b125860d",
+  "ImageId"            : "ami-07161bb3f4b6e5b6d",
   "InstanceType"       : "t4g.nano",
   "KeyName"            : "actions_key",
   "SecurityGroupIds"   : ["sg-06f57f883e902d7bc"],