prog def scset  , rclass
*!	version 1.2 by Mead Over  4Jun2024
*	Data in memory must have Stata's -long- structure defined by both a panel variable and a time variable
*	which have been -xtset- or -tsset-. As usual the -time- variable must be an integer.  Let T be the number of time periods.
*	The -panelvar- in the input data defines the units of analysis, including both the treatment unit and the donor units.
*	Let J+1 be the number of units of analysis, with one treatment unit and J donor units.
*	The analysis variables include 1 outcome variable and K independent covariates.
*	Thus the input data contains (J+1) x T rows, each of shich contains the values of the K+1 analysis variables.

*	-scset- reshapes and then stacks the data to a structure which is amenable to Synthetic Control methods.

*	The SC structure will have J+3 Stata variables or columns, of which J represent the donor units, one the treatment unit,
*	one the time variable and the J+3rd is a newly created variable named "analvars".
*	The variable "analvars" is a categorical variable with value labels defined by the names of the K+1 analytical input variables.
*	The created SC dataset is -tsset- with the same -time- variable defining the input data and "analvars" as its new panel variable.
*	The new data set has T x (K+1) rows (i.e. Stata observations) and J+3 columns (i.e. Stata variables).

*	Dependencies:
*		egen ... rank() which requires _grank.ado

syntax namelist(min=1 name=dpvlist  ///  Variables to be reshaped and stacked.
	id="outcome, predictor variable names")  ///  First listed variable is dependent variable, others are predictors
	[using/]          ///  Optionally read the input data from disk
	[if] [in],        ///
	TReated(string)   ///  The unit or units that are treated. (required)
	[time(varname)    ///  Variable whose unique values denote a period in time such as a year, day, week, etc.
	unit(varname numeric min=1)   ///  Numeric variable with a value label whose unique values and labels denote a subobservation.  Includes donors and treatment units
	rank              ///  Add the variable "rank" to measure the rank of the treatment unit among the donor units.  Requires Cox's _grank.
	float             ///  Recast type -double- analytical variables to type -float-.  Use with caution.
	double            ///  Recast type -float- analytical variables to type -double-.
	std               ///  Convert all variables to standardized variables with mean zero and standard deviation of 1.0
	savewide(string asis)  ///  If specified, save the intermediate reshaped wide file with this name. Subopts: replace, orphans
	saving(string asis)    ///  Saves the output data file in Synthetic Control format with T x (K+1) rows and J+3 columns.  Subopts: replace, orphans
	clear            ///  Allows -scset- to use data in memory even if it has not been cleared
	verbose  ]

	*###############################################
	****  Step 0: Setup and initialization
	*###############################################
	version 15

	if "`using'"=="" {
		qui des, sh
		if `r(N)'==0 & `r(k)'==0 {
			di as err "No input data. An input dataset must either be in memory or specified by the option -using filename-."
			exit 3
		}
		else if `r(changed)'>0 & "`clear'"==""  {
			di as err "The dataset in memory, `c(filename)' has been changed since it was last saved."  ///
				_n "Either save it before executing {cmd:scset} or specify the option {cmd:clear}."
			exit 198
		}
	}
	else {
		use `"`using'"'
	}
	local 0 `dpvlist'
	syntax varlist

	local inputdatalabel : data label
	if "`verbose'"=="verbose" {
		di as txt `"This data in memory from `c(filename)' will be "rotated" to the synthetic control sturucture:"'
		des, sh
	}
	local changed `r(changed)'
	local dtafileinmem `c(filename)'

	preserve

	local shhh quietly
	if "`verbose'"=="verbose"  {
		local shhh
		local xeq Xeq
	}

	//  Count the total number of variables including the outcome and the covariates
	local Nanalvars = wordcount("`varlist'")

	gettoken outcome indepvars : varlist
	local varlist `outcome' `indepvars'

	local i = 0
	foreach v in `varlist' {
		local i = `i' + 1
		local varlabels     `varlabels' `i' `v'
		local varlabellist  `varlabellist' `comma' `v'
		local comma ", "
	}

	cap tsset
	if _rc > 0 {
		di as err "Input data file must be {help xtset} or {help tsset}"
		exit 198
	}
	if "`time'"~="" & "`time'"~="`r(timevar)'" {
		di as err "Either omit the {cmd:time()} option or assure that it matches the -timevar- in the input data as previoulsy set by {help tsset}"  ///
			_n "Here the time option specifies {cmd:time(`time')}, while the -timevar- dimension in the input data is {cmd:`r(timevar)'}"
		exit 198
	}
	else if "`time'"=="" {
		local time `r(timevar)'
	}
	if "`unit'"~="" & "`unit'"~="`r(panelvar)'"  {
		di as err "Either omit the {cmd:unit} option or assure that it matches the -panelvar- in the input data as previoulsy set by {help tsset}"  ///
			_n "Here the unit option specifies {cmd:unit(`unit')}, while the -panelvar- dimension in the input data is {cmd:`r(panelvar)'}"
		exit 198
	}
	else if "`unit'"=="" {
		local unit `r(panelvar)'
	}

	//  Assure that neither the -unit- nor the -time- variable is in the list of variables to be reshaped
	local timeinvarlist : list time in varlist
	if `timeinvarlist' {
		di as err "The variable {cmd:`time'} must not appear among the outcome and predictor variables in the command's -varlist-."
		exit 198
	}
	local unitinvarlist : list unit in varlist
	if `unitinvarlist' {
		di as err "The variable {cmd:`unit'} must not appear among the outcome and predictor variables in the command's -varlist-."
		exit 198
	}

	//  To work with -tsset- or -xtset-, panelvar must be an integer.
	//  But in Synthetic Control data, each unit's name becomes a variable name.
	//  So the -panelvar-'s integer identifiers must be mapped into legitimate
	//  Stata variable names.

	local ulab : value label `unit'

	//  Convert panel variable to a string and assure that unit names have no spaces
	local unittype : type `unit'
	if inlist("`unittype'","byte","int","long") & `unit' > 0  {
		decode `unit', gen(`unit'_str) maxlength(32)  //  Maximum length of variable names constructed from value labels
		qui replace  `unit'_str = subinstr(`unit'_str," ","_",.)
		drop `unit'
	}
	else {
		di as err "The variable identifying the units, `unit', must be a positive integer of type -byte-, -int- or -long-"  ///
			_n "Here it is:"
			codebook `unit'
		exit 198
	}

	marksample touse , novarlist strok
	markout `touse' `time' `unit'_str, strok

	if "`verbose'"=="verbose" {
		di _n as txt "Before reshaping, the following variables and observations are dropped from the input data."  ///
			_n "    Dropped variables:"
		qui des , varlist
		local allvars `r(varlist)'
		local droppedvrbls : list allvars - varlist
		local droppedvrbls : list droppedvrbls - touse
		local droppedvrbls : list droppedvrbls - time
		local droppedvrbls : list droppedvrbls - unitstr
		if "`droppedvrbls'"~=""  {
			des `droppedvrbls'
			sum `droppedvrbls'
		}
		di _n as txt "    Dropped observations:"
		sum `varlist' if `touse'==0
	}

	//  Drop variables and observations not to be reshaped
	keep `time' `unit'_str `varlist' `touse'
	qui keep if `touse'
		drop `touse'

	//  Optionally standardize retained variables over retained sample
	foreach v of varlist `varlist' {
		local vlbl : variable label `v'
			qui sum `v'
			local mean `r(mean)'
			local sd   `r(sd)'
			local Nobs `r(N)'
		qui egen `double' std`v' = std(`v')
		qui replace `v' = std`v'
		lab var `v' `"`vlbl'"'
			local char_mean_`v' `mean'
			local char_sd_`v'   `sd'
			local char_N_`v'    `Nobs'
			local note_`v' "Variable \`analvstr' has been standardized by its former mean of `mean' and standard deviation of `sd' over `Nobs' observations, across all treated and donor units and all years.  See -char analvar[\`analvstr'_mean]-, -char analvar[\`analvstr'_sd]- and -char analvar[\`analvstr'_N]- for this variable."
		drop std`v'
	}

	*#################################################################
	****  Step 1: reshape all variables in -varlist- from long to wide
	*#################################################################

	//  Identify donor units
	qui levelsof `unit'_str, local(unitlist) clean

	local donors : list unitlist - treated
	local unitlist `treated' `donors'

	//  Identify earliest time period
	if `"`timelbl'"'~="" {
		lab var `time' `"`timelbl'"'
	}
	else {
		lab var `time' "Time"
	}
	sum `time', meanonly
		local time0 = `r(min)' - 1

	if "`verbose'"=="verbose"  {
		di as txt "The outcome variable is: " as res "`outcome'"
		di as txt "The other explanatory variables are: " as res "`indepvars'"
		di as txt "Levels of the unit variable, " as res "`unit'" as txt ", are:"  ///
			_n as res `"`unitlist'"'
		di as txt "The treated unit is: " as res "`treated'"
	}

	`xeq' reshape clear
	`xeq' reshape i `time'
	`xeq' reshape j `unit'_str, string
	`xeq' reshape xij `varlist' , `atwl'
	`shhh' `xeq' reshape query

	*########################
	`shhh' `xeq' reshape wide
	*########################

	sort `time'
	`shhh' compress

	//  Re-order the variables, putting donor first among units
	//  and -outcome- first among analvars
	foreach v in `outcome' `indepvars' {
		foreach u in `treated' `donors' {
			local olist `olist' `v'`u'
		}
	}
	order `olist'
	`shhh' `xeq' des

	tempfile all_wide_temp
		label data `"This file is the -wide- version of `c(filename)'.  For details, type "reshape query"."'
		notes _dta : Type -reshape long- to recover the original data file."
	`shhh' save `all_wide_temp', replace

	//  Optionally save the -reshape-d file, that has not yet been stacked
	if `"`savewide'"'~="" {
		`shhh' di _n as txt "Saving the intermediate file, which has been reshaped but not yet -stacked."  ///
			_n "User may reshape this -wide- file to -long-, and back again with the commands"  ///
			_n "{help reshape:reshape long} and {help reshape:reshape wide}."
		`shhh' save `savewide'
	}

	*######################################################################
	****  Step 2:
	****     -stack- the data creating one Stata variable per donor
	****         or treatment unit, with each new Stata variable stacking
	****         the values of the analytical variables for all units
	****     This step is unnecessary if `varlist' contains only
	****         the one variable other than the panel and time variables.
	*######################################################################


	if  `Nanalvars' == 1 {  //  Only one variable to stack
		qui use `all_wide_temp', clear
		des , varlist

		foreach v of varlist `r(varlist)' {
			local outcomelen = length("`outcome'")
			if substr("`v'", 1,`outcomelen') == "`outcome'"  {
				local vnm_wo_outcome = substr("`v'",`outcomelen'+1,.)
				rename `v' `vnm_wo_outcome'
			}
		}
		gen analvars = 1
		label define analvars 1 `outcome'
		label values analvars analvars
	}
	else {  // When there are at least two variables in the -varlist-

		tempfile stack_temp

		local firstunit firstunit
		foreach u in `unitlist'  {

			qui use `all_wide_temp', clear

			*#######################################
			`shhh' `xeq' stack *`u', into(`u') clear
			*#######################################

			keep _stack `u'

			opts_exclusive "`float' `double'"
			local typeofu : type `u'
			if "`float'"=="float" & "`typeofu'"=="double" {
				`shhh' `xeq' recast float `u', force
			}
			if "`double'"=="double" & "`typeofu'"~="double" {
				`shhh' `xeq' recast double `u'
			}

			gen int analvars = _stack

			//  This works when time increments by one unit between observations.
			//  Will need to use the "delta" information from -tsset-'s returned macros
			//  to perform an error check.
			bysort _stack : gen long `time' = `time0' + _n

			drop _stack

			order analvars `time'
			sort analvars `time'

			if "`firstunit'"=="firstunit" {
				local firstunit otherunits
			}
			else {
				`shhh' merge 1:1 analvars `time' using `stack_temp'
				assert _merge==3
				drop _merge
			}
			//  It's always safe to replace a temporary file
			`shhh' save `stack_temp', replace
		}
		cap confirm variable _merge
		if _rc>0 {
			cap label drop _merge
		}
		label define analvars `varlabels'
		label values analvars analvars
	}	//  End of loop executed if there are at least two analysis variables to stack

	*	Use Cox's -egen- command -rcount- from his collection of
	*	user contributed -egen- commands -egenmore-, we can add a
	*	variable equal to the rank of the treated unit among the donor units.
	*	In years whan a variable has a rank less than or more than all the donor uits,
	*	the snthetic value of the donor unit is computed without support.
	if "`rank'"~="" {
		cap which _grank
		if _rc==0 {
			egen long rank = rcount(`unitlist'), cond(@ > `treated')
				lab var rank "Rank of `treated' among donor units, by year & variable"
			qui replace rank = rank + 1
			notes rank : Command: egen rank = rcount(`unitlist'), cond(@ > `treated')
			notes rank : The -egen- extension -rcount()- is part of Nick Cox's package -egenmore-, available from SSC.
		}
		else {
			di as txt "The option {cmd:rank} requires Nick Cox's package of additional {help egen} commands."  ///
				_n "To use this package of -egen- extensions click:"  ///
				_n "        {stata ssc describe egenmore:{ul:{bf:here}} to describe the package or}"  ///
				_n "        {stata ssc install egenmore:{ul:{bf:here}} to install the package.}"
		}
	}

	*	-vlbllst- is only for display
	local vlbllst = substr("`varlabellist'",1,40) + "..."
	label var analvars "Analytical variables: `vlbllst'"
	notes analvars : "Analytical variables: `varlabellist'"

	`shhh' `xeq' tsset analvars `time'
	order _all, alphabetic
	order analvars `time' `treated' , first
	`shhh' compress
	reshape clear

	//  Redefine and store with the new data
	//  the new -reshape- definitions with the -scset-
	//    Create a string version of -analvars-
	decode analvars, gen(analvars_str)
	qui levelsof analvars_str , clean
	char _dta[analvars] : `r(levels)'

	`xeq'  reshape clear
	`xeq'  reshape i analvars
	`xeq'  reshape j year
	`shhh' `xeq' reshape xi analvars_str
	`xeq'  reshape xij `unitlist'

	notes _dta : Data can be converted to a wide format by typing -reshape wide-, and back again with -reshape long-.  In -long- form there is a Stata variable (i.e. column) for each donor or treated unit and there is a Stata observation (i.e. row) for each timevar-panelvar, where -timevar- is the same as in the input data and -panelvar- is the new categorical variable -analvars- identifying the outcome and predictor variables in the input data set.

	if `"`saving'"'~="" {
		if "`inputdatalabel'"=="" {
			local inputdatalabel the input data file.
			di as txt `"Data from the input data file has been "rotated" to the synthetic control data structure"'
		}
		else {
			di as txt "Data from " as res "`inputdatalabel'" as txt " has been converted to the synthetic control data structure"

		}
		qui tsset
		di as txt "and saved to disk as " as res `"`saving'"'.  ///
			_n as txt "The data has been {help tsset} with timevar "as res "`r(timevar)'" as txt " and new categorical panelvar " as res "`r(panelvar)'" as txt "."
		label data "Synthetic Control version of file -`dtafileinmem'-"
		if "`std'"~="" {
			notes _dta : Analytical variables were standardized before being reshaped to the SC structure.
			qui levelsof analvars
			foreach analv in `r(levels)'  {
				local analvstr : label (analvars) `analv'
				char analvars[`analvstr'_mean] `char_mean_`analvstr''
				char analvars[`analvstr'_sd]   `char_sd_`analvstr''
				char analvars[`analvstr'_N]    `char_N_`analvstr''
				notes analvars : `note_`analvstr''
			}
		}
		save `saving'
	}

	cap matrix drop equalstartvalues
	local Ndonors = wordcount("`donors'")
	local d = 1
	foreach u in `donors'  {

		//  Assemble a row matrix of naive starting values which sum to one.
		mat def equalstartvalues = nullmat(equalstartvalues), logit(1/`Ndonors')

		//  Use the loop to assemble the syntax of several different
		//  approaches to regularizing the weight coefficients.

		//    For e.g. -regress-, -cnsreg-, -reg3-, etc.
		local testreg     `testreg'     `plus' `u'

		//    For e.g. -lincom-
		local synreg     `synreg'     `plus' {coef_`u'}*`u'

		//    For e.g. -nl-
		local syninvlgt      `syninvlgt'      `plus' invlogit({coef_`u'})*`u'  //  For e.g. -nl-

		//    For -nlcom- after  -nl (needs _b[] and does not want {})
		local testinvlgt `testinvlgt' `plus' invlogit(_b[coef_`u':_cons])

		//    To impose summation to 1.0 on the coefficient of the last donor
		// -syninvlgt- differs from -syninvto1- only in the last term
		if `d' < `Ndonors' {
			local syninvto1 `syninvto1' `plus' invlogit({coef_`u'})*`u'
		}
		else {
			local syninvto1 `syninvto1' `plus'   ///
				invlogit(1-(`syninvto1'))*`u'
		}

		local plus " + "
		local d = `d' + 1
	}

	return matrix equalstartvalues equalstartvalues

	//  Syntax to compute the sum of the coefficients or test Ho: Sum = 1.0
	//	  For -nlcom- after -nl- estimation with -invlogit- coef. transformations
	return local testinvlgt `testinvlgt'    //  Coefficients sum to one -b[coef_varname]  (Rename testinvlgt ?)
	//	  For -lincom- after -reg-, -reg3- estimation or to impose constraint on -cnsreg- or e.g. -reg3-
	return local testreg `testreg'   //  For e.g. -regress- & -cnsreg-           (Rename testreg ?)


	//  Coefficient specifications which define the coefficient on the last donor
	//  to equal [1.0 -(sum of coefficients on previously listed donors)]
	//	  For -nl- estimation with -invlogit- coef. transformations
	return local syninvto1 `syninvto1'  //  Coefficients sum to one {coef_varname} (Rename syninvto1 ?)

	//  -nl- syntax with no constraint on sum of coefficients
	//    for nl- using the invlogit() coef transformation
	return local syninvlgt `syninvlgt'              //  Syntax for invlogit estimation by -nl- (Rename syninvlgt ?)

	//    for -nl- using no coef transformation (for comparison to -regress-, -cnsreg-, -reg3-, etc.)
	return local synreg `synreg'   //  For e.g. -lincom- (Rename synreg ?)

	return local indepvars `indepvars'
	return local outcome `outcome'
	return scalar Nanalvars = `Nanalvars'
	return local donorlist `donors'
	return scalar Ndonors = `Ndonors'
	return local treated `treated'
	return local unitlist `unitlist'

	restore, not

end  /* End of program -scset-  */

program define Xeq
*!	Version 1.0		Copied from a Stata -*_examples.ado- file.
*	Utility to echo a command to the results window (and thus to the log file)
*	and then execute it. Used e.g. in programs or loops where the command would be hidden.
	version 14

	di as txt _n
	di as txt "-> " as inp _asis `"`0'"'
*	di as txt _n

	`0'

end   /*  End of subroutine Xeq  */


*  Ver.	  Date		Changes
*  1.0.1  25May2024 Add -donorlist-, -treated- and -outcome- to returned results
*  1.0.2  31May2024 Trap as an error when -varlist- contains only one variable.
*			Trap as an error when the time or unit variable is included among
*			the -varlist- to be reshaped & stacked.
*			Optionally standardize retained variables over retained sample
*  1.0.3  2Jun2024 Make -treated- a required option.  Reorder variables in -savewide- file.
*			Enable restructure of a single variable instead of requiring 2 or more.
*			Record name of input data file in restructured file.
*			Improve -verbose- output by, inter alia, incorporating -Xeq-.
*			Drop the -touse- indicator variable so it is not in savewide data.
*			Re-order the variables in the -savewide- data file, putting donor unit first.
*			For the output data file that has been -scset-, define -reshape- and -tsset-
*			parameters with -analvars- as the new -panelvar-.
*  1.1    3Jun2024 Allow input data to be either in memory (but not changed)
*			or in a file specified by the option -using-.
*			Move the -replace- options inside the two -save- commands.
*	1.2   4Jun2024 Change the names of the returned macros.
*			Deprecate the following options:
*				STDOPTs(string): Deprecate this option until it can
*					be generalized to allow variables to keep
*					their own means and or std. dev.'s or
*					apply only to certain variables (i.e. predictors)
*				atwl(passthru): Allowing this option would
*					unnecessarily complicate the reshaped variable names.

*	TODO:
*		Optionally return the syntax for additional transformations that
*			would constrain the estimated coefficient weights to be
*			non-negative or positive.  Possibilities include:
*				mlogit  (also constrains the sum of coefficients to equal 1.0)
*				quadratic
*				sqrt(abs())
*				exp()
*		Does this program work if the increment between time units is not 1
*			or varies.  Check the -delta- returned from -tsset-
*		Add an option to save the returned macros as global macros