import pandas as pd
import numpy as np
import statsmodels.formula.api as smf

# =============================================================================
# Objective:
# generate the fixed-design bootstrap t-values for four cases
# i.    predictive regression with lagged volatiliaty scaled excess return
# ii.   predictive regression with lagged volatiliaty scaled excess return's sign
# iii.  predictive regression with cumulative volatiliaty scaled excess return
# iv.   predictive regression with cumulative volatiliaty scaled excess return' sign
# =============================================================================

# =============================================================================
# create the lagged volatiliaty scaled excess returns,  
# cumulative volatiliaty scaled excess returns,
# and their signs
# =============================================================================

ret=pd.read_excel('#Asset_Return.xlsx','Sheet1')
ret=ret[ret['YM']>=198501]# make sure all returns are after 198501

ret['RF']=ret['Rf']+1#capital RF is the gross risk free rate
ret['Return']=ret['ExRet']+ret['RF']

ret_summary=pd.DataFrame()
for i in range(1,56):
    ret_sub=ret[ret['ID']==i]    
    #get the scaled excess return
    ret_sub['ExRetVol']=ret_sub['ExRet']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetVol_f1']=ret_sub['ExRetVol'].shift(-1)
    ret_sub['ExRetVol_l0']=ret_sub['ExRetVol'].shift(0)
    ret_sub['ExRetVol_l1']=ret_sub['ExRetVol'].shift(1)
    ret_sub['ExRetVol_l2']=ret_sub['ExRetVol'].shift(2)
    ret_sub['ExRetVol_l3']=ret_sub['ExRetVol'].shift(3)
    ret_sub['ExRetVol_l4']=ret_sub['ExRetVol'].shift(4)
    ret_sub['ExRetVol_l5']=ret_sub['ExRetVol'].shift(5)
    ret_sub['ExRetVol_l6']=ret_sub['ExRetVol'].shift(6)
    ret_sub['ExRetVol_l7']=ret_sub['ExRetVol'].shift(7)
    ret_sub['ExRetVol_l8']=ret_sub['ExRetVol'].shift(8)
    ret_sub['ExRetVol_l9']=ret_sub['ExRetVol'].shift(9)
    ret_sub['ExRetVol_l10']=ret_sub['ExRetVol'].shift(10)
    ret_sub['ExRetVol_l11']=ret_sub['ExRetVol'].shift(11)
    #get the cumulative excess return 
    ret_sub['ExRetCum1']=ret['ExRet']
    ret_sub['ExRetCum2']=(pd.rolling_apply(ret_sub['Return'],2,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],2,lambda x : x.prod()))/2
    ret_sub['ExRetCum3']=(pd.rolling_apply(ret_sub['Return'],3,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],3,lambda x : x.prod()))/3
    ret_sub['ExRetCum4']=(pd.rolling_apply(ret_sub['Return'],4,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],4,lambda x : x.prod()))/4
    ret_sub['ExRetCum5']=(pd.rolling_apply(ret_sub['Return'],5,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],5,lambda x : x.prod()))/5
    ret_sub['ExRetCum6']=(pd.rolling_apply(ret_sub['Return'],6,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],6,lambda x : x.prod()))/6
    ret_sub['ExRetCum7']=(pd.rolling_apply(ret_sub['Return'],7,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],7,lambda x : x.prod()))/7
    ret_sub['ExRetCum8']=(pd.rolling_apply(ret_sub['Return'],8,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],8,lambda x : x.prod()))/8
    ret_sub['ExRetCum9']=(pd.rolling_apply(ret_sub['Return'],9,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],9,lambda x : x.prod()))/9
    ret_sub['ExRetCum10']=(pd.rolling_apply(ret_sub['Return'],10,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],10,lambda x : x.prod()))/10
    ret_sub['ExRetCum11']=(pd.rolling_apply(ret_sub['Return'],11,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],11,lambda x : x.prod()))/11
    ret_sub['ExRetCum12']=(pd.rolling_apply(ret_sub['Return'],12,lambda x : x.prod())-pd.rolling_apply(ret_sub['RF'],12,lambda x : x.prod()))/12

    ret_sub['ExRetCumVol1']=ret_sub['ExRetCum1']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol2']=ret_sub['ExRetCum2']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol3']=ret_sub['ExRetCum3']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol4']=ret_sub['ExRetCum4']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol5']=ret_sub['ExRetCum5']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol6']=ret_sub['ExRetCum6']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol7']=ret_sub['ExRetCum7']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol8']=ret_sub['ExRetCum8']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol9']=ret_sub['ExRetCum9']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol10']=ret_sub['ExRetCum10']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol11']=ret_sub['ExRetCum11']/ret_sub['Vol'].shift(1)
    ret_sub['ExRetCumVol12']=ret_sub['ExRetCum12']/ret_sub['Vol'].shift(1)

    ret_summary=pd.concat([ret_summary,ret_sub.iloc[12:]])#drop the first 12 variables
    
for i in range(0,12):
    new_name='ExRetVolSign_l'+str(i)
    old_name='ExRetVol_l'+str(i)
    ret_summary[new_name]=np.where(ret_summary[old_name]>=0,1,np.where(ret_summary[old_name]<0,-1,np.nan))

    new_name='ExRetCumVolSign'+str(i+1)
    old_name='ExRetCumVol'+str(i+1)
    ret_summary[new_name]=np.where(ret_summary[old_name]>=0,1,np.where(ret_summary[old_name]<0,-1,np.nan))

# =============================================================================
# first demean the lhs variable and rhs variable
# =============================================================================
ret_Demean=pd.DataFrame()

for i in range(1,56):
    ret_sub=ret_summary[ret_summary['ID']==i]
    ret_sub['DemeanExRetVol_f1']=ret_sub['ExRetVol_f1']-ret_sub['ExRetVol'].mean()
    for j in range(0,12):
        ret_sub['DemeanExRetVol_l'+str(j)]=ret_sub['ExRetVol_l'+str(j)]-ret_sub['ExRetVol'].mean()
        ret_sub['DemeanExRetVolSign_l'+str(j)]=ret_sub['ExRetVolSign_l'+str(j)]-ret_sub['ExRetVolSign_l0'].mean()
        ret_sub['DemeanExRetCumVol'+str(j)]=ret_sub['ExRetCumVol'+str(j+1)]-ret_sub['ExRetCumVol'+str(j+1)].mean()
        ret_sub['DemeanExRetCumVolSign'+str(j)]=ret_sub['ExRetCumVolSign'+str(j+1)]-ret_sub['ExRetCumVolSign'+str(j+1)].mean()

    ret_Demean=pd.concat([ret_Demean,ret_sub])


# =============================================================================
# run the demean regression
# =============================================================================
for i in range(0,12):
    ret_Demean['alpha_Lag'+str(i)]=np.nan
    ret_Demean['beta_Lag'+str(i)]=np.nan
    ret_Demean['alpha_LagSign'+str(i)]=np.nan
    ret_Demean['beta_LagSign'+str(i)]=np.nan
    ret_Demean['alpha_Cum'+str(i)]=np.nan
    ret_Demean['beta_Cum'+str(i)]=np.nan
    ret_Demean['alpha_CumSign'+str(i)]=np.nan
    ret_Demean['beta_CumSign'+str(i)]=np.nan


for i in range(0,12):
    new_name='DemeanExRetVol_l'+str(i)
    regression=ret_Demean[['YM','DemeanExRetVol_f1',new_name]].dropna()
    form="DemeanExRetVol_f1 ~ DemeanExRetVol_l"+str(i)+'-1'
    result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
    beta_Lag=result.params[0]
    ret_Demean['epsilon_Lag'+str(i)]=result.resid
    
    new_name='DemeanExRetVolSign_l'+str(i)
    regression=ret_Demean[['YM','DemeanExRetVol_f1',new_name]].dropna()
    form="DemeanExRetVol_f1 ~ DemeanExRetVolSign_l"+str(i)+'-1'
    result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
    beta_LagSign=result.params[0]
    ret_Demean['epsilon_LagSign'+str(i)]=result.resid

    new_name='DemeanExRetCumVol'+str(i)
    regression=ret_Demean[['YM','DemeanExRetVol_f1',new_name]].dropna()
    form="DemeanExRetVol_f1 ~ DemeanExRetCumVol"+str(i)+'-1'
    result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
    beta_Cum=result.params[0]
    ret_Demean['epsilon_Cum'+str(i)]=result.resid

    new_name='DemeanExRetCumVolSign'+str(i)
    regression=ret_Demean[['YM','DemeanExRetVol_f1',new_name]].dropna()
    form="DemeanExRetVol_f1 ~ DemeanExRetCumVolSign"+str(i)+'-1'
    result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
    beta_CumSign=result.params[0]
    ret_Demean['epsilon_CumSign'+str(i)]=result.resid

    for j in range(1,56):
        ret_sub=ret_Demean[ret_Demean['ID']==j]
        ret_Demean['beta_Lag'+str(i)]=beta_Lag
        ret_Demean['beta_LagSign'+str(i)]=beta_LagSign
        ret_Demean['beta_Cum'+str(i)]=beta_Cum
        ret_Demean['beta_CumSign'+str(i)]=beta_CumSign       
        
        ret_Demean['alpha_Lag'+str(i)].loc[ret_sub.index]=ret_sub['ExRetVol'].mean()-beta_Lag*ret_sub['ExRetVol'].mean()
        ret_Demean['alpha_LagSign'+str(i)].loc[ret_sub.index]=ret_sub['ExRetVol'].mean()-beta_LagSign*ret_sub['ExRetVolSign_l0'].mean()
        ret_Demean['alpha_Cum'+str(i)].loc[ret_sub.index]=ret_sub['ExRetVol'].mean()-beta_Cum*ret_sub['ExRetCumVol'+str(i+1)].mean()
        ret_Demean['alpha_CumSign'+str(i)].loc[ret_sub.index]=ret_sub['ExRetVol'].mean()-beta_CumSign*ret_sub['ExRetCumVolSign'+str(i+1)].mean()

# =============================================================================
# run the Fixed Design bootstrap regression
# =============================================================================

tLag_Fixed=np.empty((10,12))
tLagSign_Fixed=np.empty((10,12))
tCum_Fixed=np.empty((10,12))
tCumSign_Fixed=np.empty((10,12))


for n in range(0,10):
    size=len(ret_Demean)
    ret_Demean['vera']=np.random.uniform(0,1,size)
    ret_Demean['vera']=np.where(ret_Demean['vera']>=0.5,1,-1)
    
    for i in range(0,12):
    
        ret_Demean['star_Lag'+str(i)]=ret_Demean['alpha_Lag'+str(i)]+ret_Demean['beta_Lag'+str(i)]*ret_Demean['ExRetVol_l'+str(i)]+ret_Demean['vera']*ret_Demean['epsilon_Lag'+str(i)]
        ret_Demean['star_LagSign'+str(i)]=ret_Demean['alpha_LagSign'+str(i)]+ret_Demean['beta_LagSign'+str(i)]*ret_Demean['ExRetVolSign_l'+str(i)]+ret_Demean['vera']*ret_Demean['epsilon_LagSign'+str(i)]
        ret_Demean['star_Cum'+str(i)]=ret_Demean['alpha_Cum'+str(i)]+ret_Demean['beta_Cum'+str(i)]*ret_Demean['ExRetCumVol'+str(i+1)]+ret_Demean['vera']*ret_Demean['epsilon_Cum'+str(i)]
        ret_Demean['star_CumSign'+str(i)]=ret_Demean['alpha_CumSign'+str(i)]+ret_Demean['beta_CumSign'+str(i)]*ret_Demean['ExRetCumVolSign'+str(i+1)]+ret_Demean['vera']*ret_Demean['epsilon_CumSign'+str(i)]

        mean=pd.DataFrame(ret_Demean.groupby(['ID'])['star_Lag'+str(i),'star_LagSign'+str(i),'star_Cum'+str(i),'star_CumSign'+str(i)].mean())
        mean.columns=['Mean_Lag','Mean_LagSign','Mean_Cum','Mean_CumSign']
        mean['ID']=mean.index
        
        ret_DemeanStar=pd.merge(ret_Demean,mean,on='ID',how='left')
        ret_DemeanStar['Demean_star_Lag'+str(i)]=ret_DemeanStar['star_Lag'+str(i)]-ret_DemeanStar['Mean_Lag']
        ret_DemeanStar['Demean_star_LagSign'+str(i)]=ret_DemeanStar['star_LagSign'+str(i)]-ret_DemeanStar['Mean_LagSign']
        ret_DemeanStar['Demean_star_Cum'+str(i)]=ret_DemeanStar['star_Cum'+str(i)]-ret_DemeanStar['Mean_Cum']
        ret_DemeanStar['Demean_star_CumSign'+str(i)]=ret_DemeanStar['star_CumSign'+str(i)]-ret_DemeanStar['Mean_CumSign']
        
        regression=ret_DemeanStar[['YM',"Demean_star_Lag"+str(i),"DemeanExRetVol_l"+str(i)]].dropna()
        form="Demean_star_Lag"+str(i)+" ~ DemeanExRetVol_l"+str(i)+'-1'
        result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
        tLag_Fixed[n,i]=result.tvalues[0]

        regression=ret_DemeanStar[['YM',"Demean_star_LagSign"+str(i),"DemeanExRetVolSign_l"+str(i)]].dropna()
        form="Demean_star_LagSign"+str(i)+" ~ DemeanExRetVolSign_l"+str(i)+'-1'
        result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
        tLagSign_Fixed[n,i]=result.tvalues[0]

        regression=ret_DemeanStar[['YM',"Demean_star_Cum"+str(i),"DemeanExRetCumVol"+str(i)]].dropna()
        form="Demean_star_Cum"+str(i)+" ~ DemeanExRetCumVol"+str(i)+'-1'
        result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
        tCum_Fixed[n,i]=result.tvalues[0]

        regression=ret_DemeanStar[['YM',"Demean_star_CumSign"+str(i),"DemeanExRetCumVolSign"+str(i)]].dropna()
        form="Demean_star_CumSign"+str(i)+" ~ DemeanExRetCumVolSign"+str(i)+'-1'
        result=smf.ols(formula=form, data=regression).fit(cov_type='cluster',cov_kwds={'groups': regression['YM']},use_t=True)
        tCumSign_Fixed[n,i]=result.tvalues[0]


Table3_Fixed=pd.DataFrame(columns=['Lag','Lag_Sign','Cum','Cum_Sign'],index=np.arange(1,13))

for i in range(1,13):
    Table3_Fixed.loc[[i],['Lag']]=np.percentile(tLag_Fixed[:,i-1],97.5)
    Table3_Fixed.loc[[i],['Lag_Sign']]=np.percentile(tLagSign_Fixed[:,i-1],97.5)
    Table3_Fixed.loc[[i],['Cum']]=np.percentile(tCum_Fixed[:,i-1],97.5)
    Table3_Fixed.loc[[i],['Cum_Sign']]=np.percentile(tCumSign_Fixed[:,i-1],97.5)

pd.DataFrame.to_csv(Table3_Fixed,'Table3_Fixed.csv')