clear all;
load('vowel_test.txt');
vowel_test=vowel_test(vowel_test(:,2)<3,2:end);
header2={'y','x_1','x_2','x_3','x_4','x_5','x_6','x_7','x_8','x_9','x_10'};
vowel_test=dataset({vowel_test,header2{:}});
vowel_test.y=sign(2*vowel_test.y-3);

load('vowel_training.txt');
vowel_train=vowel_training(vowel_training(:,2)<3,2:end);
vowel_train=dataset({vowel_train,header2{:}});
vowel_train.y=sign(2*vowel_train.y-3);
warning('off','MATLAB:dispatcher:InexactCaseMatch')

    
svm_train_error=zeros(10,1);
svm_test_error=zeros(10,1);

%% non linear classification

loss_name='hinge_loss';
loss_gridsel='class_loss';
algo=svm(kernel('rbf',1));
algo.optimizer='andre';
d=vowel_train;
d_to_test=vowel_test;
%d=data('d',[d.temp,d.invHt,d.press,d.vis,d.milPress,d.hum,d.invTemp,d.wind],d.ozone);
d=data('d',[d.x_1,d.x_2,d.x_3,d.x_4,d.x_5,d.x_6,d.x_7,d.x_8,d.x_9,d.x_10],d.y);
d_to_test=data('d_to_test',[d_to_test.x_1,d_to_test.x_2,d_to_test.x_3,d_to_test.x_4,d_to_test.x_5,d_to_test.x_6,d_to_test.x_7,d_to_test.x_8,d_to_test.x_9,d_to_test.x_10],d_to_test.y);
cross_validation_names={'rbf','C'};        
cross_validation_values= {[1,1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8,1.9,2],[0.01,0.1,1,10,100]};
g=gridsel(param(algo,cross_validation_names,cross_validation_values),{'score=cv','score.folds=5'});
g.loss=loss_gridsel;
[ret_d,ret_algo]=train(g,d);
algo=ret_algo.best;
algo.rbf=1.7;
%% SVM without RFE

svm_object=algo; 
%svm_object=algo1;
svm_object.loss=loss_name;
[d_svm,svm_object]=train(svm_object,d);

svm_test_nrfe=test(svm_object,d_to_test,'class_loss');
svm_train_nrfe=test(svm_object,d,'class_loss');

%% svm with RFE

rfe_object=risk_rfe; 
rfe_object.child=algo;
rfe_object.loss=loss_name;
rfe_object.speed=10;
rfe_object.output_rank=1;
rfe_object.feat=1; 

[d_rfe,rfe_object]=train(rfe_object,d);
val=rfe_object.val; ranks=rfe_object.rank;

svm_train=algo; svm_train.loss=loss_name;
train_X=d.X;



for j=1:10;
imp_dim=j;
train_X_rfe=train_X(:,ranks(1:imp_dim));
train_Y=d.Y;
d_rfe_train=data(train_X_rfe,train_Y);
[d_rfe_trained,svm_train]=train(svm_train,d_rfe_train);

% misclassification rates

d_test=vowel_test;
%d_test=ozone_train;
d_test=data('d_test',[d_test.x_1,d_test.x_2,d_test.x_3,d_test.x_4,d_test.x_5,d_test.x_6,d_test.x_7,d_test.x_8,d_test.x_9,d_test.x_10],d_test.y);
test_X=d_test.X;
test_X_rfe=test_X(:,ranks(1:imp_dim));
test_Y=d_test.Y;
d_rfe_test=data(test_X_rfe,test_Y);
svm_test=test(svm_train,d_rfe_test,'class_loss');
svm_test_error(j)=svm_test.Y;
svm_train_rfe=test(svm_train,d_rfe_train,'class_loss');
svm_train_error(j)=svm_train_rfe.Y;

end;


%% SCAD

Cov=get_X(d); Y=get_Y(d);
svm_object=algo; svm_object.loss=loss_name;
[d_svm,svm_object]=train(svm_object,d);

dim=size(Cov,2);
w_init= svm_object.Xsv.X'*svm_object.alpha;
b_init=svm_object.b0;

lam=[0.65:-0.05:0];
svm_test_scadsvc=zeros(length(lam),1);

for j=1:length(lam);
    [w,b,xind]=scadsvc_spider(Cov,Y,w_init,b_init,3.7,lam(j));
    svm_train=algo; svm_train.loss=loss_name;
train_X=d.X;
train_X_scadsvc=train_X(:,xind);
train_Y=d.Y;
d_scadsvc_train=data(train_X_scadsvc,train_Y);
[d_scadsvc_train,svm_train]=train(svm_train,d_scadsvc_train);

test_X=d_test.X;
test_X_scadsvc=test_X(:,xind);
test_Y=d_test.Y;
d_scadsvc_test=data(test_X_scadsvc,test_Y);
t_svm_c=test(svm_train,d_scadsvc_test,loss_gridsel);
svm_test_scadsvc(j)=t_svm_c.Y;

end;






Cov=get_X(d); Y=double(get_Y(d)==1)+1;
cv=cvglmnet(Cov,Y,5,[],'response','binomial',glmnetSet,1);
lam_opt=cv.lambda_min;
lasso_fit = glmnet(Cov, Y); % regression

beta= glmnetCoef(lasso_fit,lam_opt);
lasso_set=find(beta(2:end)~=0);


Cov_lasso=Cov(:,lasso_set);
p = glmfit(Cov_lasso,(get_Y(d)==1),'binomial');
Cov_lasso_test=d_test.X(:,lasso_set);
y_hat = glmval(p, Cov_lasso_test, 'logit');
class=(y_hat > 0.5)-(y_hat <= 0.5);
misclass = mean(d_test.Y ~= class);

















set(0,'DefaultAxesFontSize',12);
plot(val(end:-1:1),'.')
hold on;
plot(val(end:-1:1))
title('Reverse Scree Graph');
xlabel('Cycle number');
ylabel('Value of the Objective Function');

set(0,'DefaultAxesFontSize',12);
plot(svm_test_error(end:-1:1),'.')
hold on;
plot(svm_test_error(end:-1:1))
title('Reverse Scree Graph');
xlabel('Cycle number');
ylabel('Misclassification_error');

untrained=algo;
sel_data=d;
numEx=330;
lss=loss(loss_name);
lss.param=0.1;
[res,trained]=train(untrained,sel_data);
        res=test(trained,sel_data);
        r=train(lss,res);
        %r=test(trained,sel_data,lss);
        alpha = trained.alpha;
        kernel_sv=train(trained.child,trained.Xsv);
        Ct=0.5*alpha'*kernel_sv.X*alpha/numEx+get_y(r)*trained.C;

