clear;clc

%% For Table 9

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Apply KNS(2020) and FGX(2020) methodologies.
% Lasso penalty: fixed-number-of-factors L1, CV optimal L1.
% Basis assets for FGX(2020): 48 portoflios
% Four sets of test assets: FF48, 202port, All stocks, Non-micro stocks


%% options
st_date = 197401; 
et_date = 201612;

T_is = 360;
T_os = st_date+T_is/12*100;

%% import data

% riskfree rate: to calculate excess returns
z = xlsread('Portfolio_ret.xlsx','FF5','A2:I653','basic'); 
tt = find(z(:,1)>=st_date & z(:,1)<=et_date);
rf = z(tt,7)/100; 
tdate = z(tt,1);


% Anomalies
z = xlsread('Portfolio_ret.xlsx','Anomaly','A3:BS518','basic'); 
tt = find(z(:,1)>=st_date & z(:,1)<=et_date);
G = z(tt,2:end)/100;
[T,L] = size(G);
display(['The number of factor proxies L =', num2str(L)]);


% 48 Industry 
z = xlsread('Portfolio_ret.xlsx','FF48vw','A3:AW530','basic'); 
tt = find(z(:,1)>=st_date & z(:,1)<=et_date);
N = size(z,2)-1;
R48 = z(tt,2:end)/100 - rf*ones(1,N); %excess returns of FF48


% 202 chr managed portfolios
z = xlsread('Portfolio_ret.xlsx','Dacheng202vw','A3:GU554','basic'); 
tt = find(z(:,1)>=st_date & z(:,1)<=et_date);
N = size(z,2)-1;
R202 = z(tt,2:end)/100 - rf*ones(1,N); %excess returns of 202 chr portfolios

% common stock returns (exclude micro caps)
load('stock_ret_nomicro.mat');
ret = st_dataset; clear st_dataset;
ret = double(ret); % permno, year, month, xret, me, exchcd
ret_but = [ret(:,1), ret(:,2)*100+ret(:,3), ret(:,4)]; 

% all common stock returns
load stock_ret.mat;
ret = st_dataset; clear st_dataset;
ret = double(ret); % permno, year, month, xret, me, exchcd
ret_all = [ret(:,1), ret(:,2)*100+ret(:,3), ret(:,4)]; 


% KNS factors
scs_is = load('SCS_IS.mat'); 
scs_is = scs_is.output_is; % [date, L1=1, L1=3, L1=5, CV L1, market portfolios]
scs_is(:,1) = round(yyyymmdd(datetime(scs_is(:,1),'ConvertFrom','datenum'))/100); % convert dates from datenum to yyyymm
assert(scs_is(1,1)==st_date & scs_is(end,1)==et_date, 'The sample period for SCS estimates does not match. Check the sample period in scs_main.m'); 
scs_is = scs_is(:,2:end);


% FGX factors
fgx_is = load('FGX_factors.mat'); 
fgx_is = fgx_is.step1_factors;
for i=1:size(fgx_is,2)
    fgx_is{i} = fgx_is{i}(:,2:end);
end


%% In-sample performance (RMSA & Total adj-R2)

% to store the performance measures

MPE = nan(2,4,4); % RMS alpha
TR2 = nan(2,4,4); % total adj-R2 

for i = 1:2; %i=1, KNS; i=2, FGX
    if i==1
      FF = scs_is;  % KNS factors
    else
      FF = fgx_is; % FGX factors  
    end
    
    for j = 1:4;   % [3,5,6,CV]  
          if i==1
              ff = FF(:,[j,end]);
          else
              ff = FF{j};
          end
          
          res  = performance_portfolio(R48, ff); % OUTres = [MPE, TotalR2, SR, A2R];
          MPE(i,j,1) = res(1); TR2(i,j,1) = res(2); 
          SR(i,j,1)=res(3); A2R(i,j,1) = res(4);

          res = performance_portfolio(R202, ff);
          MPE(i,j,2)=res(1); TR2(i,j,2)= res(2); 
          SR(i,j,2) =res(3); A2R(i,j,2)= res(4);

          res = performance_stock(tdate, ret_all, ff);
          MPE(i,j,3)=res(1);  TR2(i,j,3)= res(2); 
          SR(i,j,3) =res(3); A2R(i,j,3)= res(4);

          res = performance_stock(tdate, ret_but, ff);
          MPE(i,j,4)=res(1);  TR2(i,j,4)= res(2); 
          SR(i,j,4) =res(3); A2R(i,j,4)= res(4);

    end
end

KNS_MPE = MPE(1,:,:);
KNS_MPE = reshape(KNS_MPE,[],4)';
FGX_MPE = MPE(2,:,:);
FGX_MPE = reshape(FGX_MPE,[],4)';

KNS_TR2 = TR2(1,:,:);
KNS_TR2 = reshape(KNS_TR2,[],4)';
FGX_TR2 = TR2(2,:,:);
FGX_TR2 = reshape(FGX_TR2,[],4)';

%%  Table 9 Performance of LASSO selected factors
Table9=[[FGX_MPE, FGX_TR2];nan(1,8);...
    [KNS_MPE,  KNS_TR2]]

















