Merge pull request #267 from worldbank/v6.4

Merge v6.4 to Master

.gitignore

@@ -61,3 +61,6 @@ _config.yml
 
 #the folder test/ is used for peoples individual testing, this is different from test_scripts/
 test/
+
+#Outputs in the run/outputs folder
+run/output

admin/checklist-submitting-SSC.md

@@ -8,11 +8,11 @@
 	- [ ] 3.1 **Test in different operative systems** - This step is not necessary every time, but testing the commands in Stata on each of the PC, Mac and Linux operative systems should be done from time to time. A particularly good time to do this is after writing or editing code that depends on file paths, the console, special settings etc. If small updates are needed, then do them in the _version_ branch, otherwise do them in branches of the `develop` branch, merge those to `develop` and then re-merge `develop` to the version branch and test again.
 	- [ ] 3.2 **Update version and date** - In the _version_ branch, update the version number and date in all ado-files and all dates in all help files. See section below for details.
 	- [ ] 3.3 **Update version locals in ietoolkit** - In the _ietoolkit.ado_ file in the _version_ branch, update the _version_ and _versionDate_ locals at the top of the file.
-	- [ ] 3.4 **Update version in .pkg and .toc** - This has nothing to do with SSC but should be kept up to date to. This is for when people install directly through GitHub using `net install`
-	- [ ] 3.5 **Create a .zip file** - Create a .zip file with all ado-files and help files in the folder you just created in the archive folder. Then remove all files but the .zip file from the archive folder.
-- [ ] 4. **Email Prof. Baum** - Email the .zip file created in step 3.4 to **[email protected]**.
+	- [ ] 3.4 **Update version in .pkg and .toc** - This has nothing to do with SSC but should be kept up to date to. This is for when people install directly through GitHub using `net install`. If any new command has been added, remember to add the files for that command to the `.pkg` file.
+	- [ ] 3.5 **Create a .zip file** - Create a .zip file with all ado-files and help files only. These files are not allowed to be in a sub-folder in this .zip file. No other files should be in this folder. Make a copy of this file in the archive folder of this package.
+- [ ] 4. **Email Prof. Baum** - Email the .zip file created in step 3.5 to **[email protected]**.
 	- [ ] 4.1 - If any commands are added or deleted, make note of that in the email.
-	- [ ] 4.2 - If any of the meta info (title, description, keywords, version or authour/contact) has changed then include those updates in your email.
+	- [ ] 4.2 - If any of the meta info (title, description, keywords, version or author/contact) has changed then include those updates in your email.
 - [ ] 5. **Draft release note** - Go to the [release notes](https://github.com/worldbank/ietoolkit/releases) and draft a new release note for the new version. Follow the format from previous releases with links to [issues](https://github.com/worldbank/ietoolkit/issues) solved.
 - [ ] 6. **Wait for publication confirmation** - Do not proceed pass this step until Prof. Baum has confirmed that the new version is uploaded to the servers.
 - [ ] 7. **Merge version branch to *master*** - If step 2 and 3 was done correctly, then there should not be any merge conflicts in this step. Once merged, delete the `version` branch.

run/ie_recurse_rmdir.do

@@ -0,0 +1,67 @@
+* This file can delete all your folders on your computer if used incorrectly.
+
+cap program drop ie_recurse_rmdir
+	program define ie_recurse_rmdir
+
+qui {
+	syntax , folder(string) [DRYrun okifnotexist]
+
+	/*
+		folder - full path to folder which should be deleted with all its content
+		dryrun - just list and do not delete all files that would have been deleted without this option
+		okifnotexist - it is ok that the top folder does not exist, do not throw erroe
+
+	*/
+
+	*Test that folder exist
+	mata : st_numscalar("r(dirExist)", direxists("`folder'"))
+
+	if (`r(dirExist)' == 0) {
+		if missing("`okifnotexist'") {
+			noi di as error `"{phang}The folder used in [folder(`folder')] does not exist.{p_end}"'
+			error 693
+			exit
+		}
+		else {
+			*Folder is missin and that is ok, just output a confirmation that it does not exists
+			noi di as result `"{phang}The folder used in [folder(`folder')] is already deleted.{p_end}"'
+		}
+	}
+	else {
+
+		 if (length("`folder'")<=10) {
+			noi di as error `"{phang}The folder used in [folder(`folder')] does not exist or you have not entered the full path.{p_end}"'
+			error 693
+			exit
+		}
+
+		* File paths can have both forward and/or back slash. We'll standardize them so they're easier to handle
+		local folderStd			= subinstr(`"`folder'"',"\","/",.)
+
+		*List files, directories and other files
+		local flist : dir `"`folderStd'"' files "*"	, respectcase
+		local dlist : dir `"`folderStd'"' dirs  "*" , respectcase
+		local olist : dir `"`folderStd'"' other "*"	, respectcase
+
+		*Use the command on each subfolder to this folder (if any)
+		foreach dir of local dlist {
+			*Recursive call on each subfolder
+			noi ie_recurse_rmdir , folder(`"`folderStd'/`dir'"') `automatic' `dryrun'
+		}
+
+		*REmove all files
+		local files `"`flist' `olist'"'
+		foreach file of local files {
+			*If dryrun then list otherwise delete
+			if !missing("`dryrun'") noi di as result "{pstd}DRY RUN! Without option {bf:dryrun} file [`folderStd'/`file'] would have been deleted.{p_end}"
+			else rm `"`folderStd'/`file'"'
+		}
+
+		* Remove this folder as it is now empty
+		if missing("`dryrun'") {
+			rmdir `"`folderStd'"'
+			noi di as result "{pstd}Folder [`folderStd'] and all its content were deleted.{p_end}"
+		}
+	}
+}
+end

run/ieddtab.do

@@ -0,0 +1,108 @@
+
+/*******************************************************************************
+	Set up
+*******************************************************************************/
+
+	* Set versson and seed
+	ieboilstart , version(13.1)
+	`r(version)'
+	set seed 313833
+
+
+	* Add the path to your local clone of the [ietoolkit] repo
+
+	global 	ietoolkit "C:\Users\wb462869\Documents\GitHub\ietoolkit"
+	qui		do 		  "${ietoolkit}/src/ado_files/ieddtab.ado"
+
+
+	* Load data blood pressure patient data
+	sysuse	bplong, clear
+
+	*Rename and recode time var
+	rename when time
+	recode time (1=0) (2=1)
+
+	* Sort patient and time and randomize number on baseline value
+	bys patient : gen rand = runiform() if time == 0
+
+	*Sort rand and assign half of baseline to tmt == 1. 
+	*Baseline is half of all obs, so half bseline = .25
+	sort rand 
+	gen tmt = (_n / _N > .25) if !missing(rand)
+
+	*Sort on patient and time and copy baselin value to endline obs
+	sort patient time
+	by patient : replace tmt = tmt[_n-1] if time == 1
+
+	*Label vars. USe suffix -test to see what is dynamc and what is hard coded
+	label define treatment 0 "Control-test" 1 "Treatment-test"
+	label define time 0 "Baseline-test" 1 "Endline-test"
+	label value time time
+	label value tmt treatment
+
+
+	*Tidy up required vars
+	local orderkeep patient tmt time bp
+	keep `orderkeep'
+	order `orderkeep'
+
+
+	**************************************
+
+	*Utility function to test result matrix
+	program define test_matrix 
+		syntax ,  rmatrix(string) row(string) col(string) expected_val(string)	
+		local value = `rmatrix'["`row'","`col'"]
+		di "{pstd}Row: `row', Col: `col', Expected value: `expected_val', Actual value `value'{p_end}"
+		assert `value' == `expected_val'
+	end
+
+
+
+	**************************************
+
+	*regular run, test all values
+	ieddtab bp, time(time) treat(tmt)
+
+	mat result1 = r(ieddtabResults)
+	mat list result1
+
+	*Test second difference
+	test_matrix , rmatrix(result1) row("bp") col("2D") 			expected_val(-1.649999999999993)
+	test_matrix , rmatrix(result1) row("bp") col("2D_err") 		expected_val(3.332609384661584)
+	test_matrix , rmatrix(result1) row("bp") col("2D_stars") 	expected_val(0)
+	test_matrix , rmatrix(result1) row("bp") col("2D_N") 		expected_val(240)
+
+	*Test control first difference
+	test_matrix , rmatrix(result1) row("bp") col("1DC") 		expected_val(-4.266666666666667)
+	test_matrix , rmatrix(result1) row("bp") col("1DC_err") 	expected_val(2.39864211429447)
+	test_matrix , rmatrix(result1) row("bp") col("1DC_stars") 	expected_val(1)
+	test_matrix , rmatrix(result1) row("bp") col("1DC_N") 		expected_val(120)
+
+	*Test treatment first difference
+	test_matrix , rmatrix(result1) row("bp") col("1DT") 		expected_val(-5.916666666666667)
+	test_matrix , rmatrix(result1) row("bp") col("1DT_err") 	expected_val(2.313612179745651)
+	test_matrix , rmatrix(result1) row("bp") col("1DT_stars") 	expected_val(2)
+	test_matrix , rmatrix(result1) row("bp") col("1DT_N") 		expected_val(120)
+
+	*Test control baseline summary stats
+	test_matrix , rmatrix(result1) row("bp") col("C0_mean") 	expected_val(156.0666666666667)
+	test_matrix , rmatrix(result1) row("bp") col("C0_err") 		expected_val(1.594458388227513)
+	test_matrix , rmatrix(result1) row("bp") col("C0_N") 		expected_val(60)
+
+	*Test treatment baseline summary stats
+	test_matrix , rmatrix(result1) row("bp") col("T0_mean") 	expected_val(156.8333333333333)
+	test_matrix , rmatrix(result1) row("bp") col("T0_err") 		expected_val(1.346719526847542)
+	test_matrix , rmatrix(result1) row("bp") col("T0_N") 		expected_val(60)
+
+	*Test control endline summary stats
+	test_matrix , rmatrix(result1) row("bp") col("C1_mean") 	expected_val(151.8)
+	test_matrix , rmatrix(result1) row("bp") col("C1_err") 		expected_val(1.791978359433497)
+	test_matrix , rmatrix(result1) row("bp") col("C1_N") 		expected_val(60)
+
+	*Test treatment endline summary stats
+	test_matrix , rmatrix(result1) row("bp") col("T1_mean") 	expected_val(150.9166666666667)
+	test_matrix , rmatrix(result1) row("bp") col("T1_err") 		expected_val(1.881262298105969)
+	test_matrix , rmatrix(result1) row("bp") col("T1_N") 		expected_val(60)
+
+

run/iegitaddmd.do

@@ -1,11 +1,21 @@
 
 	* Set this local to a folder where folders can be creaetd for this experiment
-	global test_folder ""
+	global ietoolkit_clone "C:\Users\wb462869\Documents\GitHub\ietoolkit"
+	global test_folder "${ietoolkit_clone}\run\output\iegitaddmd"
+
+	qui do "${ietoolkit_clone}/run/ie_recurse_rmdir.do" 
+
+	* Make sure this folder is created
+	cap mkdir "${ietoolkit_clone}\run\output\iegitaddmd"
 
-	* Create a folder clone and a folder dropbox (see use case 2 in help file for why both these folders are needed)
-	mkdir "${test_folder}/clone"
-	mkdir "${test_folder}/dropbox"
 
+	* Create a folder clone and a folder dropbox (see use case 2 in help file for why both these folders are needed)
+	local highlevel_folders clone dropbox gitfilter1 gitfilter2
+	foreach folder of local highlevel_folders {
+		ie_recurse_rmdir, folder("${test_folder}/`folder'") okifnotexist
+		mkdir "${test_folder}/`folder'"
+	}
+
 	* USe iefolder to create identical project folders 
 	iefolder new project , projectfolder("${test_folder}/clone")
 	iefolder new project , projectfolder("${test_folder}/dropbox")
@@ -23,18 +33,46 @@
 	rmdir "${test_folder}/clone/DataWork/Baseline/DataSets"
 	rmdir "${test_folder}/clone/DataWork/Baseline/Questionnaire/Questionnaire Documentation"
 
+	* Create the git folder
+	local gitfolders gitfilter1 gitfilter2
+	foreach gitfolder of local gitfolders {
+		mkdir "${test_folder}/`gitfolder'/.git"
+		mkdir "${test_folder}/`gitfolder'/includeme"
+		mkdir "${test_folder}/`gitfolder'/skipmein2"
+		mkdir "${test_folder}/`gitfolder'/skipalsomein2"
+		mkdir "${test_folder}/`gitfolder'/sub"
+		mkdir "${test_folder}/`gitfolder'/sub/.git"
+		mkdir "${test_folder}/`gitfolder'/sub/includealsome"
+
+		mkdir "${test_folder}/`gitfolder'/ado"
+		mkdir "${test_folder}/`gitfolder'/asdfasd"
+		mkdir "${test_folder}/`gitfolder'/asdfasd/ado"
+	}
+
+
 	*Set global to ietoolkit clone
-	global ietoolkit_clone ""
 	qui do "${ietoolkit_clone}/src/ado_files/iegitaddmd.ado"
 
 	*Use case 1 - see helpfile for description of use cases
 	iegitaddmd , folder("${test_folder}/clone/DataWork") auto dryrun
 	iegitaddmd , folder("${test_folder}/clone/DataWork") auto 
 
 	*Use case 2 - see helpfile for description of use cases
-	iegitaddmd , folder("${test_folder}/clone/DataWork") comparefolder("${test_folder}/dropbox/DataWork") auto dryrun
-	iegitaddmd , folder("${test_folder}/clone/DataWork") comparefolder("${test_folder}/dropbox/DataWork") auto
+	iegitaddmd , folder("${test_folder}/clone\DataWork") comparefolder("${test_folder}/dropbox\DataWork") auto dryrun
+	iegitaddmd , folder("${test_folder}/clone\DataWork") comparefolder("${test_folder}/dropbox\DataWork") auto
 
 	*Test prompt
 	mkdir "${test_folder}/clone/DataWork/Baseline/DataSets/Final/Publishable"
-	iegitaddmd , folder("${test_folder}/clone/DataWork") 
+	*iegitaddmd , folder("${test_folder}/clone/DataWork") 
+
+	*Test skip git folders
+	iegitaddmd , folder("${test_folder}/gitfilter1") auto 
+
+	*Test skip custom
+	iegitaddmd , folder("${test_folder}/gitfilter2") auto dry skipfolders(skipmein2 skipalsomein2 folderthatnotexist ado)
+	iegitaddmd , folder("${test_folder}/gitfilter2") auto skipfolders(skipmein2 skipalsomein2 folderthatnotexist ado)
+
+	* Test that folders are not used
+	cap iegitaddmd , folder("${test_folder}/gitfilter2") auto skipfolders(skipmein2 skipalsomein2 folderthatnotexist asdfasd/ado)
+	assert _rc == 198
+

src/ado_files/iebaltab.ado

@@ -1,4 +1,4 @@
-*! version 6.3 5NOV2019 DIME Analytics [email protected]
+*! version 6.4 11JAN2022 DIME Analytics [email protected]
 
 	capture program drop iebaltab
 	program iebaltab

src/ado_files/ieboilsave.ado

@@ -1,4 +1,4 @@
-*! version 6.3 5NOV2019 DIME Analytics [email protected]
+*! version 6.4 11JAN2022 DIME Analytics [email protected]
 
 	capture program drop ieboilsave
 	program ieboilsave , rclass

src/ado_files/ieboilstart.ado

@@ -1,4 +1,4 @@
-*! version 6.3 5NOV2019 DIME Analytics [email protected]
+*! version 6.4 11JAN2022 DIME Analytics [email protected]
 
 	capture program drop ieboilstart
 	program ieboilstart , rclass
@@ -16,7 +16,8 @@
 
 		*********************************/
 
-		local stata_versions "11 11.0 11.1 11.2 12 12.0 12.1 13 13.0 13.1 14 14.0 14.1 14.2 15 15.0 15.1"
+		* Based on versions listed here: https://www.stata.com/support/faqs/resources/history-of-stata/
+		local stata_versions "11 11.0 11.1 11.2 12 12.0 12.1 13 13.0 13.1 14 14.0 14.1 14.2 15 15.0 15.1 16.0 16.1 17.0"
 
 		if `:list versionnumber in stata_versions' == 0 {
 

src/ado_files/ieddtab.ado

@@ -1,4 +1,4 @@
-*! version 6.3 5NOV2019 DIME Analytics [email protected]
+*! version 6.4 11JAN2022 DIME Analytics [email protected]
 
 cap program drop 	ieddtab
 	program define	ieddtab, rclass
@@ -676,7 +676,7 @@ cap program drop 	templateResultMatrix
 	local basicMean_C1_cols		C1_mean C1_err C1_N C1_clus
 	local basicMean_T1_cols		T1_mean T1_err T1_N T1_clus
 
-	local colnames `2ndDiff_cols' `1stDiff_C_cols' `1stDiff_T_cols' `basicMean_C0_cols' `basicMean_T0_cols' `basicMean_C1_cols' `basicMean_T0_cols'
+	local colnames `2ndDiff_cols' `1stDiff_C_cols' `1stDiff_T_cols' `basicMean_C0_cols' `basicMean_T0_cols' `basicMean_C1_cols' `basicMean_T1_cols'
 
 	*Define default row here. The results for each var will be one row that starts with all missing vlaues
 	mat startRow = (.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.)

src/ado_files/iedropone.ado

@@ -1,4 +1,4 @@
-*! version 6.3 5NOV2019 DIME Analytics [email protected]
+*! version 6.4 11JAN2022 DIME Analytics [email protected]
 
 	capture program drop 	iedropone
 			program define 	iedropone ,

src/ado_files/iefolder.ado

@@ -1,4 +1,4 @@
-*! version 6.3 5NOV2019 DIME Analytics [email protected]
+*! version 6.4 11JAN2022 DIME Analytics [email protected]
 
 cap program drop 	iefolder
 	program define	iefolder
@@ -899,7 +899,7 @@ cap program drop 	mdofle_p0
 		file write  `subHandle' 	///
 			_col(4)"*Install all packages that this project requires:" _n ///
 			_col(4)"*(Note that this never updates outdated versions of already installed commands, to update commands use adoupdate)" _n ///
-			_col(4)"local user_commands ietoolkit" _col(40) "//Fill this list will all user-written commands this project requires" _n ///
+			_col(4)"local user_commands ietoolkit iefieldkit" _col(40) "//Fill this list will all user-written commands this project requires" _n ///
 			_col(4)"foreach command of local user_commands {" _n ///
 			_col(8)		"cap which " _char(96) "command'" _n ///
 			_col(8)		"if _rc == 111 {" _n ///