S3 method for summarizing CAISEr objects output by run_experiment()).
Input parameters test, alternative and sig.level can be used to
override the ones used in the call to run_experiment().
# S3 method for CAISEr
summary(object, test = NULL, alternative = NULL, sig.level = NULL, ...)list object of class CAISEr
(generated by run_experiment())
type of test to be used ("t.test", "wilcoxon" or "binomial")
type of alternative hypothesis ("two.sided" or
"less" or "greater"). See calc_instances() for details.
desired family-wise significance level (alpha) for the experiment
other parameters to be passed down to specific summary functions (currently unused)
A list object is returned invisibly, containing the details of all tests performed as well as information on the total number of runs dedicated to each algorithm.
# Example using four dummy algorithms and 100 dummy instances.
# See [dummyalgo()] and [dummyinstance()] for details.
# Generating 4 dummy algorithms here, with means 15, 10, 30, 15 and standard
# deviations 2, 4, 6, 8.
algorithms <- mapply(FUN = function(i, m, s){
list(FUN = "dummyalgo",
alias = paste0("algo", i),
distribution.fun = "rnorm",
distribution.pars = list(mean = m, sd = s))},
i = c(alg1 = 1, alg2 = 2, alg3 = 3, alg4 = 4),
m = c(15, 10, 30, 15),
s = c(2, 4, 6, 8),
SIMPLIFY = FALSE)
# Generate 100 dummy instances with centered exponential distributions
instances <- lapply(1:100,
function(i) {rate <- runif(1, 1, 10)
list(FUN = "dummyinstance",
alias = paste0("Inst.", i),
distr = "rexp", rate = rate,
bias = -1 / rate)})
my.results <- run_experiment(instances, algorithms,
d = 1, se.max = .1,
power = .9, sig.level = .05,
power.target = "mean",
dif = "perc", comparisons = "all.vs.all",
seed = 1234, ncpus = 1)
#> CAISEr running
#> -----------------------------
#> Required number of instances: 17
#> Available number of instances: 100
#> Using 1 cores.
#> -----------------------------
#> Sampling algorithms on instance Inst.28 : ..
#> Sampling algorithms on instance Inst.80 : ..
#> Sampling algorithms on instance Inst.22 : ..
#> Sampling algorithms on instance Inst.9 : ..
#> Sampling algorithms on instance Inst.5 : ..
#> Sampling algorithms on instance Inst.38 : ..
#> Sampling algorithms on instance Inst.16 : ..
#> Sampling algorithms on instance Inst.4 : ..
#> Sampling algorithms on instance Inst.86 : ..
#> Sampling algorithms on instance Inst.90 : ..
#> Sampling algorithms on instance Inst.70 : ..
#> Sampling algorithms on instance Inst.79 : ..
#> Sampling algorithms on instance Inst.78 : ..
#> Sampling algorithms on instance Inst.14 : ..
#> Sampling algorithms on instance Inst.56 : ..
#> Sampling algorithms on instance Inst.62 : ..
#> Sampling algorithms on instance Inst.93 : ..
summary(my.results)
#> #====================================
#> CAISEr object:
#> Number of instances sampled: 17
#> Number of instances required: 17
#> Adequate power: TRUE
#> Total runs of algo1 : 340
#> Total runs of algo2 : 369
#> Total runs of algo3 : 505
#> Total runs of algo4 : 621
#> #====================================
#> Pairwise comparisons of interest:
#> Test: t.test
#> H1: two.sided
#> Comparisons: all.vs.all
#> Alpha (FWER): 0.05
#> Power target: mean
#> Desired power: 0.9
#> #====================================
#> Tests using Holm's step-down procedure:
#>
#> Test 1 : algo2 x algo3
#> H0: mean = 0
#> alpha = 0.008333
#> p-value = 7.553e-30
#> Est. mean = -1.211
#> CI{1-alpha} = [ -1.227 -1.196 ]
#> d = -57.90465
#>
#> Test 2 : algo1 x algo3
#> H0: mean = 0
#> alpha = 0.01
#> p-value = 1.462e-29
#> Est. mean = -0.8402
#> CI{1-alpha} = [ -0.8509 -0.8295 ]
#> d = -55.56286
#>
#> Test 3 : algo3 x algo4
#> H0: mean = 0
#> alpha = 0.0125
#> p-value = 4.493e-23
#> Est. mean = 0.8085
#> CI{1-alpha} = [ 0.7832 0.8338 ]
#> d = 21.82551
#>
#> Test 4 : algo2 x algo4
#> H0: mean = 0
#> alpha = 0.01667
#> p-value = 2.056e-20
#> Est. mean = -0.4029
#> CI{1-alpha} = [ -0.4204 -0.3853 ]
#> d = -14.86694
#>
#> Test 5 : algo1 x algo2
#> H0: mean = 0
#> alpha = 0.025
#> p-value = 1.283e-19
#> Est. mean = 0.3712
#> CI{1-alpha} = [ 0.3544 0.388 ]
#> d = 13.25217
#>
#> Test 6 : algo1 x algo4
#> H0: mean = 0
#> alpha = 0.05
#> p-value = 0.008827
#> Est. mean = -0.0317
#> CI{1-alpha} = [ -0.05424 -0.009154 ]
#> d = -0.7229391
#> #====================================
# You can override some defaults if you want:
summary(my.results, test = "wilcoxon")
#> #====================================
#> CAISEr object:
#> Number of instances sampled: 17
#> Number of instances required: 17
#> Adequate power: TRUE
#> Total runs of algo1 : 340
#> Total runs of algo2 : 369
#> Total runs of algo3 : 505
#> Total runs of algo4 : 621
#> #====================================
#> Pairwise comparisons of interest:
#> Test: wilcoxon
#> H1: two.sided
#> Comparisons: all.vs.all
#> Alpha (FWER): 0.05
#> Power target: mean
#> Desired power: 0.9
#> #====================================
#> Tests using Holm's step-down procedure:
#>
#> Test 1 : algo1 x algo2
#> H0: median = 0
#> alpha = 0.008333
#> p-value = 1.526e-05
#> Est. median = 0.3768
#> CI{1-alpha} = [ 0.3547 0.3977 ]
#> d = 13.25217
#>
#> Test 2 : algo1 x algo3
#> H0: median = 0
#> alpha = 0.01
#> p-value = 1.526e-05
#> Est. median = -0.8388
#> CI{1-alpha} = [ -0.8532 -0.8289 ]
#> d = -55.56286
#>
#> Test 3 : algo2 x algo3
#> H0: median = 0
#> alpha = 0.0125
#> p-value = 1.526e-05
#> Est. median = -1.215
#> CI{1-alpha} = [ -1.227 -1.196 ]
#> d = -57.90465
#>
#> Test 4 : algo2 x algo4
#> H0: median = 0
#> alpha = 0.01667
#> p-value = 1.526e-05
#> Est. median = -0.3987
#> CI{1-alpha} = [ -0.4211 -0.382 ]
#> d = -14.86694
#>
#> Test 5 : algo3 x algo4
#> H0: median = 0
#> alpha = 0.025
#> p-value = 1.526e-05
#> Est. median = 0.8062
#> CI{1-alpha} = [ 0.7821 0.8341 ]
#> d = 21.82551
#>
#> Test 6 : algo1 x algo4
#> H0: median = 0
#> alpha = 0.05
#> p-value = 0.02016
#> Est. median = -0.0296
#> CI{1-alpha} = [ -0.06164 -0.01526 ]
#> d = -0.7229391
#> #====================================