LRaven1_traditional

010: Get started IA REPORT
020: Load H and event sequence data
030: Get session params and Define IA references to parse IA report
040: Gather AOI dwell times and AOI dwell time prc for matrix and response
050: Define constants and target scores
060: Compute "Matrix Time Distribution Index" see VigneauCaissieBors06
070: Run regression and cross-validation on MTD index
080: Generate toggles for all sbjs
Gather trial latency to first toggle from matrix to response
090: Toggle rate on easy items (first 5 trials only)
095: Toggle count on easy items (first 5 trials only)
100: Item latency on easy items (first 5 trials only)
110: Run regression and cross validation on toggle rate on easy items
115: Run regression and cross validation on toggle count on easy items
120: Run regression and cross validation on item latency on easy items
130: Run regression on matrix dwell time for completeness
135: Run regression on latency to first toggle
140: Run model 1 Vigneau,Caissie,Bors
150: Run model 2 Vigneau,Caissie,Bors
160: Run model 3 Vigneau,Caissie,Bors
170: Run model 4 Vigneau,Caissie,Bors
180: What is our best model using these measures
999: Conclusions

% LRaven1_traditional_comparison - Use to generate the best R2 performance
%                                 using measures described by
%                                 VigneauCaissieBors06.

% Computes each measure, matrix time distribution index, item latency on
% easy items and toggling on easy items and calculates R2 and cvR2 score
% prediction performance
%
%
% 1.0.0 2011-02-21 TRH: Wrote it

010: Get started IA REPORT

cd(fullfile(LRaven1_pathstr,'data')) ;
fprintf('\n\nLRaven1_plot executed on %s.\n\n',datestr(now)) ;
fprintf('cd %s\n',pwd) ;

clear all ;


LRaven1_plot executed on 22-Aug-2011 15:07:11.

cd /Users/apetrov/a/r/w/work/RelExper/LRaven1/matlab/data

020: Load H and event sequence data

% Load H: <1x35 struct> produced by LRaven1_hitcount
filename = fullfile(LRaven1_pathstr,'data','H.mat') ;
fprintf('load %s \n\n',filename) ;
load(filename) ;
clear filename

% Load E
filename = fullfile(LRaven1_pathstr,'data','E.mat') ;
fprintf('load %s \n\n',filename) ;
load(filename) ;
clear filename

load /Users/apetrov/a/r/w/work/RelExper/LRaven1/matlab/data/H.mat 

load /Users/apetrov/a/r/w/work/RelExper/LRaven1/matlab/data/E.mat

030: Get session params and Define IA references to parse IA report

% Experimental params
session1 = 1:14 ;
session2 = 15:28 ;
session12 = 1:28 ;

% define IA references in IA_Report
AOI_tot_scr = 20 ;
AOI_tot_mtrx = 10 ;
AOI_tot_rsp = 19 ;
AOI_mtrx_c1 = 1 ;
AOI_mtrx_c2 = 2 ;
AOI_mtrx_c3 = 3 ;
AOI_mtrx_c4 = 4 ;
AOI_mtrx_c5 = 5 ;
AOI_mtrx_c6 = 6 ;
AOI_mtrx_c7 = 7 ;
AOI_mtrx_c8 = 8 ;
AOI_mtrx_c9 = 9 ;
AOI_rsp_c1 = 11 ;
AOI_rsp_c2 = 12 ;
AOI_rsp_c3 = 13 ;
AOI_rsp_c4 = 14 ;
AOI_rsp_c5 = 15 ;
AOI_rsp_c6 = 16 ;
AOI_rsp_c7 = 17 ;
AOI_rsp_c8 = 18 ;

040: Gather AOI dwell times and AOI dwell time prc for matrix and response

% Loop to get AI values for both sessions
L = length(H) ;
R = length(H(1,1).IA_dwell_time) ;
dwell_time = zeros(R,L) ;
dwell_time_prc = zeros(R,L) ;
for k = 1:L
    dwell_time(:,k) = H(k).IA_dwell_time ;
    dwell_time_prc(:,k) = H(k).IA_dwell_time_prc ;
end

% Get the idx for IA_Label: AOI_tot_mtrx and AOI_tot_rsp
IA_label = H(1,1).IA_label ;
idx_mtrx = find(IA_label== AOI_tot_mtrx)' ;
idx_rsp = find(IA_label == AOI_tot_rsp)' ;
% Pull out IA matrix values only
dt_mtrx_raw = dwell_time(idx_mtrx,:) ;
dt_mtrx = dwell_time(idx_mtrx,:)/1000 ;    % RAW dwell time mtrx
dt_prc_mtrx = dwell_time_prc(idx_mtrx,:) ; % Proportional dwell time mtrx
% Pull out IA rsp values only
dt_rsp = dwell_time(idx_rsp,:)/1000 ;    % Raw dwell time mtrx
dt_prc_rsp = dwell_time_prc(idx_rsp,:) ; % Proportional dwell time mtrx

% Set mistrials to nan = sbj2 T25, sbj14 T5 11, sbj35 T21
dt_mtrx(25,2)= NaN ;
dt_mtrx(5,14) = NaN ;
dt_mtrx(11,14) = NaN ;
dt_mtrx(21,35) = NaN ;

dt_prc_mtrx(25,2) = NaN ;
dt_prc_mtrx(5,14) = NaN ;
dt_prc_mtrx(11,14) = NaN ;
dt_prc_mtrx(21,35) = NaN ;

dt_rsp(25,2) = NaN ;
dt_rsp(5,14) = NaN ;
dt_rsp(11,14) = NaN ;
dt_rsp(21,35) = NaN ;

dt_prc_rsp(25,2) = NaN ;
dt_prc_rsp(5,14) = NaN ;
dt_prc_rsp(11,14) = NaN ;
dt_prc_rsp(21,35) = NaN ;

050: Define constants and target scores

% Get target scores
score_ttl_s1 = [H.session1_score]' ;          % Session1 scores
score_ttl_s2 = + [H.session2_score]' ;        % Session2 scores
score_ttl_s12 = score_ttl_s1 + score_ttl_s2 ; % Session12 scores

060: Compute "Matrix Time Distribution Index" see VigneauCaissieBors06

% "For each item it is computed as the proportion of time on cells
% 3, 6, 7, 8, 9 relative to the overall Matrix dwell time
% subtracted from
% the proportion of time on cells 1, 2, 4, and 5 relative to
% the overall Matrix dwell time
% Negative values of this difference score reflect the fact that gaze
% was mainly directed at the last column and the last row, arguably
% reflecting partial analysis, whereas values near zero
% indicate an even distribution of gaze time across the
% matrix cells, arguably reflecting a more complete
% analysis of the matrix. Averaging across the 14 test
% items produced an average Matrix Time Distribution
% Index for the test".


% Loop to get AI values for both sessions
L = length(H) ;
R = length(H(1,1).IA_dwell_time) ;
dwell_time = zeros(R,L) ;
dwell_time_prc = zeros(R,L) ;
for k = 1:L
    dwell_time(:,k) = H(k).IA_dwell_time ;
end

% Get the idx for IA_Label: for cells 3,6,7,8,9 and 1,2,4,5
IA_label = H(1,1).IA_label ;
idx_mtrx_c3 = find(IA_label== AOI_mtrx_c3)' ;
idx_mtrx_c6 = find(IA_label== AOI_mtrx_c6)' ;
idx_mtrx_c7 = find(IA_label== AOI_mtrx_c7)' ;
idx_mtrx_c8 = find(IA_label== AOI_mtrx_c8)' ;
idx_mtrx_c9 = find(IA_label== AOI_mtrx_c9)' ;

idx_mtrx_c1 = find(IA_label== AOI_mtrx_c1)' ;
idx_mtrx_c2 = find(IA_label== AOI_mtrx_c2)' ;
idx_mtrx_c4 = find(IA_label== AOI_mtrx_c4)' ;
idx_mtrx_c5 = find(IA_label== AOI_mtrx_c5)' ;


% Pull out IA matrix cell values from dwell time and divide by total matrix dwell time
dt_prc_mtrx_c3 = dwell_time(idx_mtrx_c3,:)./dt_mtrx_raw; % c3
dt_prc_mtrx_c6 = dwell_time(idx_mtrx_c6,:)./dt_mtrx_raw ; % c6
dt_prc_mtrx_c7 = dwell_time(idx_mtrx_c7,:)./dt_mtrx_raw ; % c7
dt_prc_mtrx_c8 = dwell_time(idx_mtrx_c8,:)./dt_mtrx_raw ; % c8
dt_prc_mtrx_c9 = dwell_time(idx_mtrx_c9,:)./dt_mtrx_raw ; % c9

dt_prc_mtrx_c1 = dwell_time(idx_mtrx_c1,:)./dt_mtrx_raw ; % c6
dt_prc_mtrx_c2 = dwell_time(idx_mtrx_c2,:)./dt_mtrx_raw ; % c7
dt_prc_mtrx_c4 = dwell_time(idx_mtrx_c4,:)./dt_mtrx_raw ; % c8
dt_prc_mtrx_c5 = dwell_time(idx_mtrx_c5,:)./dt_mtrx_raw ; % c9



% Matrix Time Distribution (add proportions together and subtract) ses12
last_line = (dt_prc_mtrx_c3+dt_prc_mtrx_c6+dt_prc_mtrx_c7+dt_prc_mtrx_c8+dt_prc_mtrx_c9) ;
other_cells = (dt_prc_mtrx_c1+dt_prc_mtrx_c2+dt_prc_mtrx_c4+dt_prc_mtrx_c5) ;
MTDI = last_line - other_cells ;
MTDI = MTDI' ;

070: Run regression and cross-validation on MTD index

% Average across trials
MTDI_s1 = nanmean(MTDI(:,1:14),2) ;
MTDI_s2 = nanmean(MTDI(:,15:28),2) ;
MTDI_s12 = nanmean(MTDI(:,1:28),2) ;

% Regression MTDI
reg_MTDI_s1 = regress_subset(score_ttl_s1,MTDI_s1) ;
reg_MTDI_s2 = regress_subset(score_ttl_s2,MTDI_s2) ;
reg_MTDI_s12 = regress_subset(score_ttl_s12,MTDI_s12) ;

% Cross-validated MTDI
case_sets = nchoosek(1:35,34) ;
cv_MTDI_s1 = cross_validated_R2(score_ttl_s1,MTDI_s1,case_sets) ;
cv_MTDI_s2 = cross_validated_R2(score_ttl_s2,MTDI_s2,case_sets) ;
cv_MTDI_s12 = cross_validated_R2(score_ttl_s12,MTDI_s12,case_sets) ;

080: Generate toggles for all sbjs

output toggle counts (trial 1:28,sbj) format

sbjs_eventseq = [E.sbj] ;

% for all sbjs
for S = 1:length(sbjs_eventseq)

    % for all sessions
    for b = 1:2
        ses = find(E(S).session == b) ;
        trial = E(S).data(ses,1) ;
        ia = E(S).data(ses,10) ;

         % Find and idx matrix, response, or other category ia's
         matrix = find(ia==1 | ia==2 | ia==3 | ia==4 | ia==5 | ia==6|...
               ia==7 | ia==8 | ia==9 | ia==10| ia==10.1) ;
         response = find(ia==11 | ia==12 | ia==13 | ia==14 | ia==15 | ...
                 ia==16 | ia==17 | ia==18 | ia==19 | ia==19.1) ;
         other = find(ia==0 | ia==20 | ia==21) ;
         % Replace matrix and response ia indices with 0 and .5
         ia(matrix,:) = 0 ;
         ia(response,:) = 1 ;
         ia(other,:) = .5 ;

        % for all trials
        for j = 1:14

            % Get id to grab trial data
            idx = find(trial==j) ;

            % Get ia sequence values by trial
            ia_by_trial = ia(idx,:) ;
            toggles = tcounter(ia_by_trial) ;
            tc_fin(j,1) = toggles ;       %#ok<SAGROW>
        end

        % Pack values into matrix 1 column per subject
        if b==1
        tc_sbj(1:14,S) = tc_fin ;                  %#ok<SAGROW>
        else
        tc_sbj(15:28,S) = tc_fin;                  %#ok<SAGROW>
        end
    end
end


% Calculate toggle rate to control for item latency
% * since item latency on easy items is used in the model as well

% take sbj item toggle counts and divide item by item by sbj item latencies
item_latency = [H.session1_RT;H.session2_RT];
toggle_r = tc_sbj./item_latency ;

% Set mistrials to NaN = sbj2 T25, sbj14 T5 11, sbj 35 T21
toggle_r(25,2) = NaN ;
toggle_r(5,14) = NaN ;
toggle_r(11,14) = NaN ;
toggle_r(21,35) = NaN ;

% Toggle rate
toggle_r = toggle_r' ; % convert to [35x28]
% Raw toggles
tc_sbj = tc_sbj' ;

Gather trial latency to first toggle from matrix to response

Same as cell above but trial time is in % of trial time

%sbjs_eventseq = [E.sbj] ;

for S = 1:length(E) ;
    % for every session 1 and 2
    for b=1:2
    ses = find(E(S).session == b) ;
    trial = E(S).data(ses,1) ;
    dur = E(S).data(ses,5) ;
    ia = E(S).data(ses,10) ;

    % Find and idx matrix, response, or other category ia's
    matrix = find(ia==1 | ia==2 | ia==3 | ia==4 | ia==5 | ia==6|...
              ia==7 | ia==8 | ia==9 | ia==10| ia==10.1) ;
    response = find(ia==11 | ia==12 | ia==13 | ia==14 | ia==15 | ...
                ia==16 | ia==17 | ia==18 | ia==19 | ia==19.1) ;
    other = find(ia==0 | ia==20 | ia==21) ;

    % Replace matrix and response ia indices with 0 and .5
    ia(matrix,:) = 0 ;
    ia(response,:) = 1 ;
    ia(other,:) = .5 ;

    % For all trials
    for j = 1:14

        % Get idx to grab trial data
        idx = find(trial==j); %j

        % Get ia data
        ia_by_trial = ia(idx,:) ;
        counts = tcounter_list(ia_by_trial) ;
        first_idx = find(counts==1) ;
        if length(first_idx) < 1;    % identify miss trials for sbj2,10,14,20
        t_latency(j,1) = 0 ; %#ok<SAGROW>
        else
        first_toggle = first_idx(1) ;     % single ia idx of first toggle
        % Normalize duration time to % of trial time
        dur1 = dur(idx,:) ;
        cs = cumsum(dur1) ;
        new_x = (cs-min(cs))/(max(cs)-min(cs))*100 ; % Set 0-100%
        t_latency(j,1) = new_x(first_toggle) ; %#ok<SAGROW>
        end
    end

        % Pack values into columns 1 per subject
        if b==1
        t_latency_sbj(1:14,S) = t_latency ;%#ok<SAGROW>
        else
        t_latency_sbj(15:28,S) = t_latency;%#ok<SAGROW>
        end
    end
end

t_latency_sbj = t_latency_sbj' ;
ses12_FTL = nanmean(t_latency_sbj,2) ;

090: Toggle rate on easy items (first 5 trials only)

% Session 1 easy toggle rate
ses1_toggle_r = nanmean(toggle_r(:,1:5),2) ;

% Session 2 easy toggle rate
ses2_toggle_r = nanmean(toggle_r(:,15:19),2) ;

% Session 12 easy toggle rate
ses12_toggle = zeros([35 10]) ;
ses12_toggle(:,1:5) = toggle_r(:,1:5) ;
ses12_toggle(:,6:10) = toggle_r(:,15:19) ;
ses12_toggle_r = nanmean(ses12_toggle,2) ;

095: Toggle count on easy items (first 5 trials only)

% Session 1 easy toggle count
ses1_toggle_ct = nanmean(tc_sbj(:,1:5),2) ;

% Session 2 easy toggle count
ses2_toggle_ct = nanmean(tc_sbj(:,15:19),2) ;

% Session 12 easy toggle rate
ses12_toggle_c = zeros([35 10]) ;
ses12_toggle_c(:,1:5) = tc_sbj(:,1:5) ;
ses12_toggle_c(:,6:10) = tc_sbj(:,15:19) ;
ses12_toggle_ct = nanmean(ses12_toggle_c,2) ;

100: Item latency on easy items (first 5 trials only)

item_latency = item_latency' ; % convert to [35x28]

% Session 1 easy item latency
ses1_item_latency = nanmean(item_latency(:,1:5),2) ;

% Session 2 easy item latency
ses2_item_latency = nanmean(item_latency(:,15:19),2) ;

% Session 12 easy item latency
ses12_item = zeros([35 10]) ;
ses12_item(:,1:5) = item_latency(:,1:5) ;
ses12_item(:,6:10) = item_latency(:,15:19) ;
ses12_item_latency = nanmean(ses12_item,2) ;

110: Run regression and cross validation on toggle rate on easy items

% Regression toggle rate on easy items
reg_ses1_toggle_r = regress_subset(score_ttl_s1,ses1_toggle_r) ;
reg_ses2_toggle_r = regress_subset(score_ttl_s2,ses2_toggle_r) ;
reg_ses12_toggle_r = regress_subset(score_ttl_s12,ses12_toggle_r) ;

% Cross-validated toggle rate on easy items
case_sets = nchoosek(1:35,34) ;
cv_ses1_toggle_r = cross_validated_R2(score_ttl_s1,ses1_toggle_r,case_sets) ;
cv_ses2_toggle_r = cross_validated_R2(score_ttl_s2,ses2_toggle_r,case_sets) ;
cv_ses12_toggle_r = cross_validated_R2(score_ttl_s12,ses12_toggle_r,case_sets) ;

115: Run regression and cross validation on toggle count on easy items

% Regression toggle count on easy items
reg_ses1_toggle_ct = regress_subset(score_ttl_s1,ses1_toggle_ct) ;
reg_ses2_toggle_ct = regress_subset(score_ttl_s2,ses2_toggle_ct) ;
reg_ses12_toggle_ct = regress_subset(score_ttl_s12,ses12_toggle_ct) ;

% Cross-validated toggle count on easy items
case_sets = nchoosek(1:35,34) ;
cv_ses1_toggle_ct = cross_validated_R2(score_ttl_s1,ses1_toggle_ct,case_sets) ;
cv_ses2_toggle_ct = cross_validated_R2(score_ttl_s2,ses2_toggle_ct,case_sets) ;
cv_ses12_toggle_ct = cross_validated_R2(score_ttl_s12,ses12_toggle_ct,case_sets) ;

120: Run regression and cross validation on item latency on easy items

% Regression item latency on easy items
reg_ses1_latency = regress_subset(score_ttl_s1,ses1_item_latency) ;
reg_ses2_latency = regress_subset(score_ttl_s2,ses2_item_latency) ;
reg_ses12_latency = regress_subset(score_ttl_s12,ses12_item_latency) ;

% Cross-validated item latency on easy items
case_sets = nchoosek(1:35,34) ;
cv_ses1_latency = cross_validated_R2(score_ttl_s1,ses1_item_latency,case_sets) ;
cv_ses2_latency = cross_validated_R2(score_ttl_s2,ses2_item_latency,case_sets) ;
cv_ses12_latency = cross_validated_R2(score_ttl_s12,ses12_item_latency,case_sets) ;

130: Run regression on matrix dwell time for completeness

% dt_prc_mtrx as predictor
mtrx_regress = dt_prc_mtrx' ;
mtrx_regress_s1 = nanmean(mtrx_regress(:,1:14),2) ;
mtrx_regress_s2 = nanmean(mtrx_regress(:,15:28),2) ;
mtrx_regress_s12 = nanmean(mtrx_regress(:,1:28),2) ;

% Regression prc_dt_mtrx
reg_prc_dt_mtrx_s12 = regress_subset(score_ttl_s12,mtrx_regress_s12) ;
reg_prc_dt_mtrx_s1 = regress_subset(score_ttl_s1,mtrx_regress_s1) ;
reg_prc_dt_mtrx_s2 = regress_subset(score_ttl_s2,mtrx_regress_s2) ;

% Cross-validated prc_dt_mtrx
case_sets = nchoosek(1:35,34) ;
cv_prc_dt_mtrx_s12 = cross_validated_R2(score_ttl_s12,mtrx_regress_s12,case_sets) ;
cv_prc_dt_mtrx_s1 = cross_validated_R2(score_ttl_s1,mtrx_regress_s1,case_sets) ;
cv_prc_dt_mtrx_s2 = cross_validated_R2(score_ttl_s2,mtrx_regress_s2,case_sets) ;

135: Run regression on latency to first toggle

% regression latency to first toggle
reg_FTL_s12 = regress_subset(score_ttl_s12,ses12_FTL) ;
cv_FTL_s12 = cross_validated_R2(score_ttl_s12,ses12_FTL,case_sets) ;

140: Run model 1 Vigneau,Caissie,Bors

% Multiple regression 2 predictors: MTDI and proportional time on matrix
X1 = [mtrx_regress_s12 MTDI_s12] ;
model_1 = regress_subset(score_ttl_s12,X1) ;
model_1

cv_model_1 = cross_validated_R2(score_ttl_s12,X1,case_sets) ;
cv_model_1

model_1 = 

           y: [35x1 double]
           X: [35x2 double]
     var_idx: [1 2]
    case_idx: [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35]
          X1: [35x3 double]
        beta: [44.8596 4.9825 -14.0918]
          R2: 0.1775
        yhat: [35x1 double]


cv_model_1 = 

             y: [35x1 double]
             X: [35x2 double]
     case_sets: [35x34 double]
          beta: [35x3 double]
        R2_set: [35x1 double]
       y_train: [35x34 double]
        y_test: [35x1 double]
    yhat_train: [35x34 double]
     yhat_test: [35x1 double]
      R2_train: 0.1796
       R2_test: 0.0642
    RMSE_train: 3.3470
     RMSE_test: 3.6417

150: Run model 2 Vigneau,Caissie,Bors

% Multiple regression with MTDI and latency to first toggle
X2 = [MTDI_s12 ses12_FTL] ;
model_2 = regress_subset(score_ttl_s12,X2) ;
model_2

cv_model_2 = cross_validated_R2(score_ttl_s12,X2,case_sets) ;
cv_model_2

model_2 = 

           y: [35x1 double]
           X: [35x2 double]
     var_idx: [1 2]
    case_idx: [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35]
          X1: [35x3 double]
        beta: [-3.6456 0.0318 20.1956]
          R2: 0.0242
        yhat: [35x1 double]


cv_model_2 = 

             y: [35x1 double]
             X: [35x2 double]
     case_sets: [35x34 double]
          beta: [35x3 double]
        R2_set: [35x1 double]
       y_train: [35x34 double]
        y_test: [35x1 double]
    yhat_train: [35x34 double]
     yhat_test: [35x1 double]
      R2_train: 0.0263
       R2_test: -0.0911
    RMSE_train: 3.6462
     RMSE_test: 3.9203

160: Run model 3 Vigneau,Caissie,Bors

% Multiple regression 3 predictors: MTDI, proportional time on matrix, latency easy items

X3 = [MTDI_s12 mtrx_regress_s12 ses12_item_latency] ;
model_3 = regress_subset(score_ttl_s12,X3) ;
model_3

cv_model_3 = cross_validated_R2(score_ttl_s12,X3,case_sets) ;
cv_model_3

model_3 = 

           y: [35x1 double]
           X: [35x3 double]
     var_idx: [1 2 3]
    case_idx: [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35]
          X1: [35x4 double]
        beta: [6.6957 38.9334 0.0470 -11.1345]
          R2: 0.2265
        yhat: [35x1 double]


cv_model_3 = 

             y: [35x1 double]
             X: [35x3 double]
     case_sets: [35x34 double]
          beta: [35x4 double]
        R2_set: [35x1 double]
       y_train: [35x34 double]
        y_test: [35x1 double]
    yhat_train: [35x34 double]
     yhat_test: [35x1 double]
      R2_train: 0.2288
       R2_test: 0.0936
    RMSE_train: 3.2449
     RMSE_test: 3.5872

170: Run model 4 Vigneau,Caissie,Bors

% Multiple regression 3 predictors: MTDI, Toggle rate easy, latency easy

X4 = [MTDI_s12 ses12_toggle_r ses12_item_latency] ;
model_4 = regress_subset(score_ttl_s12,X4) ;
model_4

cv_model_4 = cross_validated_R2(score_ttl_s12,X4,case_sets) ;
cv_model_4


% Multiple regression 3 predictors: MTDI, number of toggles easy, latency easy

X5 = [MTDI_s12 ses12_toggle_ct ses12_item_latency] ;
model_4_count = regress_subset(score_ttl_s12,X5) ;
model_4_count

cv_model_4_count = cross_validated_R2(score_ttl_s12,X5,case_sets) ;
cv_model_4_count

model_4 = 

           y: [35x1 double]
           X: [35x3 double]
     var_idx: [1 2 3]
    case_idx: [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35]
          X1: [35x4 double]
        beta: [4.9513 -15.4499 0.0466 23.7423]
          R2: 0.1709
        yhat: [35x1 double]


cv_model_4 = 

             y: [35x1 double]
             X: [35x3 double]
     case_sets: [35x34 double]
          beta: [35x4 double]
        R2_set: [35x1 double]
       y_train: [35x34 double]
        y_test: [35x1 double]
    yhat_train: [35x34 double]
     yhat_test: [35x1 double]
      R2_train: 0.1734
       R2_test: 0.0426
    RMSE_train: 3.3595
     RMSE_test: 3.7140


model_4_count = 

           y: [35x1 double]
           X: [35x3 double]
     var_idx: [1 2 3]
    case_idx: [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35]
          X1: [35x4 double]
        beta: [3.9995 -0.2971 0.1032 20.4595]
          R2: 0.1584
        yhat: [35x1 double]


cv_model_4_count = 

             y: [35x1 double]
             X: [35x3 double]
     case_sets: [35x34 double]
          beta: [35x4 double]
        R2_set: [35x1 double]
       y_train: [35x34 double]
        y_test: [35x1 double]
    yhat_train: [35x34 double]
     yhat_test: [35x1 double]
      R2_train: 0.1613
       R2_test: 0.0267
    RMSE_train: 3.3839
     RMSE_test: 3.8096

180: What is our best model using these measures

X6 = [ses12_toggle_r ses12_item_latency mtrx_regress_s12] ;
our_best_model = regress_subset(score_ttl_s12,X6) ;
our_best_model

cv_our_best_model = cross_validated_R2(score_ttl_s12,X6,case_sets) ;
cv_our_best_model

our_best_model = 

           y: [35x1 double]
           X: [35x3 double]
     var_idx: [1 2 3]
    case_idx: [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35]
          X1: [35x4 double]
        beta: [-1.5622 0.0404 29.6479 -3.4953]
          R2: 0.2052
        yhat: [35x1 double]


cv_our_best_model = 

             y: [35x1 double]
             X: [35x3 double]
     case_sets: [35x34 double]
          beta: [35x4 double]
        R2_set: [35x1 double]
       y_train: [35x34 double]
        y_test: [35x1 double]
    yhat_train: [35x34 double]
     yhat_test: [35x1 double]
      R2_train: 0.2073
       R2_test: 0.0848
    RMSE_train: 3.2899
     RMSE_test: 3.5972

999: Conclusions

% % RESULTS comparison with VigneauCaissieBors06
%
% MTDI                       R2       cvR2
% ses1->ses1                .01       -
% ses2->ses2                .0004     -
% ses12->ses12              .01       -
%
% Toggle rate easy items     R2       cvR2
% ses1->ses1                .04       .007
% ses2->ses2                .07       .004
% ses12->ses12              .11       .04

% Number of tog. easy items  R2       cvR2
% ses1->ses1                .001      -
% ses2->ses2                .001      -
% ses12->ses12              .001      -
%
% Latency easy items         R2       cvR2
% ses1->ses1                .06       .004
% ses2->ses2                .08       .01
% ses12->ses12              .10       .04
%
% Prc dt on matrix           R2       cvR2
% ses1->ses1                .04       .04
% ses2->ses2                .12       .04
% ses12->ses12              .16       .08

% While the toggle rate on easy items and item latency on easy items can
% account for some variance in some of the runs
% it is no where even close to the amount described in VigneauCassieBors06.
% Moreover, the Matrix time distribution index is horrible on all days.  I am
% not sure why there is such a large disparity in the numbers?  I
% calculated the easy index the exact way described in the paper.

% In the paper don't focus too much on VigneauCassieBors06, but rather use it
% to say "we tried these measures and here is how well they predict scores on
% our data set compared to our novel SR approach".


% NEW RESULTS MULTIPLE REGRESSION MODELS from VigneauCaissieBors06
% model_1 =
%
%            y: [35x1 double]
%            X: [35x2 double]
%      var_idx: [1 2]
%     case_idx: [1x35 double]
%           X1: [35x3 double]
%         beta: [44.8596 4.9825 -14.0918]
%           R2: 0.1775
%         yhat: [35x1 double]
%
%
% model_2 =
%
%            y: [35x1 double]
%            X: [35x2 double]
%      var_idx: [1 2]
%     case_idx: [1x35 double]
%           X1: [35x3 double]
%         beta: [-3.6456 0.0318 20.1956]
%           R2: 0.0242
%         yhat: [35x1 double]
%
%
% model_3 =
%
%            y: [35x1 double]
%            X: [35x3 double]
%      var_idx: [1 2 3]
%     case_idx: [1x35 double]
%           X1: [35x4 double]
%         beta: [6.6957 38.9334 0.0470 -11.1345]
%           R2: 0.2265
%         yhat: [35x1 double]
%
%
% model_4 =
%
%            y: [35x1 double]
%            X: [35x3 double]
%      var_idx: [1 2 3]
%     case_idx: [1x35 double]
%           X1: [35x4 double]
%         beta: [4.9513 -15.4499 0.0466 23.7423]
%           R2: 0.1709
%         yhat: [35x1 double]
%
% model_4_count =
%
%            y: [35x1 double]
%            X: [35x3 double]
%      var_idx: [1 2 3]
%     case_idx: [1x35 double]
%           X1: [35x4 double]
%         beta: [3.9995 -0.2971 0.1032 20.4595]
%           R2: 0.1584
%         yhat: [35x1 double]
%

%-----
%%%%% end of file LRaven1_traditional_comparison.m

Contents

010: Get started IA REPORT

020: Load H and event sequence data

030: Get session params and Define IA references to parse IA report

040: Gather AOI dwell times and AOI dwell time prc for matrix and response

050: Define constants and target scores

060: Compute "Matrix Time Distribution Index" see VigneauCaissieBors06

070: Run regression and cross-validation on MTD index

080: Generate toggles for all sbjs

Gather trial latency to first toggle from matrix to response

090: Toggle rate on easy items (first 5 trials only)

095: Toggle count on easy items (first 5 trials only)

100: Item latency on easy items (first 5 trials only)

110: Run regression and cross validation on toggle rate on easy items

115: Run regression and cross validation on toggle count on easy items

120: Run regression and cross validation on item latency on easy items

130: Run regression on matrix dwell time for completeness

135: Run regression on latency to first toggle

140: Run model 1 Vigneau,Caissie,Bors

150: Run model 2 Vigneau,Caissie,Bors

160: Run model 3 Vigneau,Caissie,Bors

170: Run model 4 Vigneau,Caissie,Bors

180: What is our best model using these measures

999: Conclusions