Chapter 14 Bootstrap

This chapter deals with bootstrap.

The students are expected to acquire the following knowledge:

  • How to use bootstrap to generate coverage intervals.

Exercise 14.1 Ideally, a \(1-\alpha\) CI would have \(1-\alpha\) coverage. That is, say a 95% CI should, in the long run, contain the true value of the parameter 95% of the time. In practice, it is impossible to assess the coverage of our CI method, because we rarely know the true parameter. In simulation, however, we can. Let’s assess the coverage of bootstrap percentile intervals.

  1. Pick a univariate distribution with readily available mean and one that you can easily sample from.

  2. Draw \(n = 30\) random samples from the chosen distribution and use the bootstrap (with large enough m) and percentile CI method to construct 95% CI. Repeat the process many times and count how many times the CI contains the true mean. That is, compute the actual coverage probability (don’t forget to include the standard error of the coverage probability!). What can you observe?

  3. Try one or two different distributions. What can you observe?

  4. Repeat (b) and (c) using BCa intervals (R package boot). How does the coverage compare to percentile intervals?

  5. As (d) but using intervals based on asymptotic normality (+/- 1.96 SE).

  6. How do results from (b), (d), and (e) change if we increase the sample size to n = 200? What about n = 5?

library(boot)
set.seed(0)
nit   <- 1000  # Repeat the process "many times"
alpha <- 0.05  # CI parameter
nboot <- 100   # m parameter for bootstrap ("large enough m")
# f: change this to 200 or 5.
nsample <- 30  # n = 30 random samples from the chosen distribution. Comment out BCa code if it breaks.
covers     <- matrix(nrow = nit, ncol = 3)
covers_BCa <- matrix(nrow = nit, ncol = 3)
covers_asymp_norm <- matrix(nrow = nit, ncol = 3)

isin <- function (x, lower, upper) {
  (x > lower) & (x < upper)
}

for (j in 1:nit) {  # Repeating many times
  # a: pick a univariate distribution - standard normal
  x1 <- rnorm(nsample)
  
  # c: one or two different distributions - beta and poisson
  x2 <- rbeta(nsample, 1, 2)
  x3 <- rpois(nsample, 5)
  
  X1 <- matrix(data = NA, nrow = nsample, ncol = nboot)
  X2 <- matrix(data = NA, nrow = nsample, ncol = nboot)
  X3 <- matrix(data = NA, nrow = nsample, ncol = nboot)
  for (i in 1:nboot) {
    X1[ ,i] <- sample(x1, nsample, replace = T)
    X2[ ,i] <- sample(x2, nsample, T)
    X3[ ,i] <- sample(x3, nsample, T)
  }
  X1_func <- apply(X1, 2, mean)
  X2_func <- apply(X2, 2, mean)
  X3_func <- apply(X3, 2, mean)
  X1_quant <- quantile(X1_func, probs = c(alpha / 2, 1 - alpha / 2))
  X2_quant <- quantile(X2_func, probs = c(alpha / 2, 1 - alpha / 2))
  X3_quant <- quantile(X3_func, probs = c(alpha / 2, 1 - alpha / 2))
  covers[j,1] <- (0 > X1_quant[1]) & (0 < X1_quant[2])
  covers[j,2] <- ((1 / 3) > X2_quant[1]) & ((1 / 3) < X2_quant[2])
  covers[j,3] <- (5 > X3_quant[1]) & (5 < X3_quant[2])

  mf     <- function (x, i) return(mean(x[i]))
  bootX1 <- boot(x1, statistic = mf, R = nboot)
  bootX2 <- boot(x2, statistic = mf, R = nboot)
  bootX3 <- boot(x3, statistic = mf, R = nboot)

  X1_quant_BCa <- boot.ci(bootX1, type = "bca")$bca
  X2_quant_BCa <- boot.ci(bootX2, type = "bca")$bca
  X3_quant_BCa <- boot.ci(bootX3, type = "bca")$bca
  
  covers_BCa[j,1] <- (0 > X1_quant_BCa[4]) & (0 < X1_quant_BCa[5])
  covers_BCa[j,2] <- ((1 / 3) > X2_quant_BCa[4]) & ((1 / 3) < X2_quant_BCa[5])
  covers_BCa[j,3] <- (5 > X3_quant_BCa[4]) & (5 < X3_quant_BCa[5])
  
  # e: estimate mean and standard error
  # sample mean:
  x1_bar <- mean(x1)
  x2_bar <- mean(x2)
  x3_bar <- mean(x3)
  
  # standard error (of the sample mean) estimate: sample standard deviation / sqrt(n)
  x1_bar_SE <- sd(x1) / sqrt(nsample)
  x2_bar_SE <- sd(x2) / sqrt(nsample)
  x3_bar_SE <- sd(x3) / sqrt(nsample)
  
  covers_asymp_norm[j,1] <- isin(0, x1_bar - 1.96 * x1_bar_SE, x1_bar + 1.96 * x1_bar_SE)
  covers_asymp_norm[j,2] <- isin(1/3, x2_bar - 1.96 * x2_bar_SE, x2_bar + 1.96 * x2_bar_SE)
  covers_asymp_norm[j,3] <- isin(5, x3_bar - 1.96 * x3_bar_SE, x3_bar + 1.96 * x3_bar_SE)

}
apply(covers, 2, mean)
## [1] 0.918 0.925 0.905
apply(covers, 2, sd) / sqrt(nit)
## [1] 0.008680516 0.008333333 0.009276910
apply(covers_BCa, 2, mean)
## [1] 0.927 0.944 0.927
apply(covers_BCa, 2, sd) / sqrt(nit)
## [1] 0.008230355 0.007274401 0.008230355
apply(covers_asymp_norm, 2, mean)
## [1] 0.939 0.937 0.930
apply(covers_asymp_norm, 2, sd) / sqrt(nit)
## [1] 0.007572076 0.007687008 0.008072494

Exercise 14.2 You are given a sample of independent observations from a process of interest:

Index 1 2 3 4 5 6 7 8
X 7 2 4 6 4 5 9 10

  1. Compute the plug-in estimate of mean and 95% symmetric CI based on asymptotic normality. Use the plug-in estimate of SE.

  2. Same as (a), but use the unbiased estimate of SE.

  3. Apply nonparametric bootstrap with 1000 bootstrap replications and estimate the 95% CI for the mean with percentile-based CI.

# a
x <- c(7, 2, 4, 6, 4, 5, 9, 10)
n <- length(x)
mu <- mean(x)

SE <- sqrt(mean((x - mu)^2)) / sqrt(n)
SE
## [1] 0.8915839
z <- qnorm(1 - 0.05 / 2)
c(mu - z * SE, mu + z * SE)
## [1] 4.127528 7.622472
# b
SE <- sd(x) / sqrt(n)
SE
## [1] 0.9531433
c(mu - z * SE, mu + z * SE)
## [1] 4.006873 7.743127
# c
set.seed(0)

m  <- 1000
T_mean <- function(x) {mean(x)}

est_boot <- array(NA, m)
for (i in 1:m) {
  x_boot <- x[sample(1:n, n, rep = T)]
  est_boot[i] <- T_mean(x_boot)
}

quantile(est_boot, p = c(0.025, 0.975))
##  2.5% 97.5% 
## 4.250 7.625

Exercise 14.3 We are given a sample of 10 independent paired (bivariate) observations:

Index 1 2 3 4 5 6 7 8 9 10
X 1.26 -0.33 1.33 1.27 0.41 -1.54 -0.93 -0.29 -0.01 2.40
Y 2.64 0.33 0.48 0.06 -0.88 -2.14 -2.21 0.95 0.83 1.45

  1. Compute Pearson correlation between X and Y.

  2. Use the cor.test() from R to estimate a 95% CI for the estimate from (a).

  3. Apply nonparametric bootstrap with 1000 bootstrap replications and estimate the 95% CI for the Pearson correlation with percentile-based CI.

  4. Compare CI from (b) and (c). Are they similar?

  5. How would the bootstrap estimation of CI change if we were interested in Spearman or Kendall correlation instead?

x <- c(1.26, -0.33,  1.33,  1.27,  0.41, -1.54, -0.93, -0.29, -0.01,  2.40)
y <- c(2.64,  0.33,  0.48,  0.06, -0.88, -2.14, -2.21,  0.95,  0.83,  1.45)

# a
cor(x, y)
## [1] 0.6991247
# b
res <- cor.test(x, y)
res$conf.int[1:2]
## [1] 0.1241458 0.9226238
# c
set.seed(0)
m  <- 1000
n  <- length(x) 
T_cor <- function(x, y) {cor(x, y)}

est_boot <- array(NA, m)
for (i in 1:m) {
  idx <- sample(1:n, n, rep = T) # !!! important to use same indices to keep dependency between x and y
  est_boot[i] <- T_cor(x[idx], y[idx])
}

quantile(est_boot, p = c(0.025, 0.975))
##      2.5%     97.5% 
## 0.2565537 0.9057664
# d
# Yes, but the bootstrap CI is more narrow.

# e
# We just use the functions for Kendall/Spearman coefficients instead:
T_kendall <- function(x, y) {cor(x, y, method = "kendall")}
T_spearman <- function(x, y) {cor(x, y, method = "spearman")}

# Put this in a function that returns the CI
bootstrap_95_ci <- function(x, y, t, m = 1000) {
  n <- length(x)
  est_boot <- array(NA, m)
  for (i in 1:m) {
    idx <- sample(1:n, n, rep = T) # !!! important to use same indices to keep dependency between x and y
    est_boot[i] <- t(x[idx], y[idx])
  }
  quantile(est_boot, p = c(0.025, 0.975))
}

bootstrap_95_ci(x, y, T_kendall)
##        2.5%       97.5% 
## -0.08108108  0.78378378
bootstrap_95_ci(x, y, T_spearman)
##       2.5%      97.5% 
## -0.1701115  0.8867925

Exercise 14.4 In this problem we will illustrate the use of the nonparametric bootstrap for estimating CIs of regression model coefficients.

  1. Load the longley dataset from base R with data(longley).

  2. Use lm() to apply linear regression using “Employed” as the target (dependent) variable and all other variables as the predictors (independent). Using lm() results, print the estimated regression coefficients and standard errors. Estimate 95% CI for the coefficients using +/- 1.96 * SE.

  3. Use nonparametric bootstrap with 100 replications to estimate the SE of the coefficients from (b). Compare the SE from (c) with those from (b).

# a
data(longley)

# b
res <- lm(Employed ~ . , longley)
tmp <- data.frame(summary(res)$coefficients[,1:2])
tmp$LB <- tmp[,1] - 1.96 * tmp[,2]
tmp$UB <- tmp[,1] + 1.96 * tmp[,2]
tmp
##                   Estimate   Std..Error            LB            UB
## (Intercept)  -3.482259e+03 8.904204e+02 -5.227483e+03 -1.737035e+03
## GNP.deflator  1.506187e-02 8.491493e-02 -1.513714e-01  1.814951e-01
## GNP          -3.581918e-02 3.349101e-02 -1.014616e-01  2.982320e-02
## Unemployed   -2.020230e-02 4.883997e-03 -2.977493e-02 -1.062966e-02
## Armed.Forces -1.033227e-02 2.142742e-03 -1.453204e-02 -6.132495e-03
## Population   -5.110411e-02 2.260732e-01 -4.942076e-01  3.919994e-01
## Year          1.829151e+00 4.554785e-01  9.364136e-01  2.721889e+00
# c
set.seed(0)
m <- 100
n <- nrow(longley)
T_coef <- function(x) {
  lm(Employed ~ . , x)$coefficients
}

est_boot <- array(NA, c(m, ncol(longley)))
for (i in 1:m) {
  idx <- sample(1:n, n, rep = T)
  est_boot[i,] <- T_coef(longley[idx,])
}

SE <- apply(est_boot, 2, sd)
SE
## [1] 1.826011e+03 1.605981e-01 5.693746e-02 8.204892e-03 3.802225e-03
## [6] 3.907527e-01 9.414436e-01
# Show the standard errors around coefficients
library(ggplot2)
library(reshape2)
df <- data.frame(index = 1:7, bootstrap_SE = SE, lm_SE = tmp$Std..Error)
melted_df <- melt(df[2:nrow(df), ], id.vars = "index")  # Ignore bias which has a really large magnitude
ggplot(melted_df, aes(x = index, y = value, fill = variable)) +
  geom_bar(stat="identity", position="dodge") +
  xlab("Coefficient") +
  ylab("Standard error") # + scale_y_continuous(trans = "log") # If you want to also plot bias

Exercise 14.5 This exercise shows a shortcoming of the bootstrap method when using the plug in estimator for the maximum.

  1. Compute the 95% bootstrap CI for the maximum of a standard normal distribution.

  2. Compute the 95% bootstrap CI for the maximum of a binomial distribution with n = 15 and p = 0.2.

  3. Repeat (b) using p = 0.9. Why is the result different?

# bootstrap CI for maximum

alpha <- 0.05
T_max <- function(x) {max(x)}  # Equal to T_max = max
bootstrap <- function(x, t, m = 1000) {
  n <- length(x)
  values <- rep(0, m)
  for (i in 1:m) {
    values[i] <- t(sample(x, n, replace = T))
  }
  quantile(values, probs = c(alpha / 2, 1 - alpha / 2))
}

# a
# Meaningless, as the normal distribution can yield arbitrarily large values.
x <- rnorm(100)
bootstrap(x, T_max)
##     2.5%    97.5% 
## 1.819425 2.961743
# b
x <- rbinom(100, size = 15, prob = 0.2) # min = 0, max = 15
bootstrap(x, T_max)
##  2.5% 97.5% 
##     6     7
# c
x <- rbinom(100, size = 15, prob = 0.9) # min = 0, max = 15
bootstrap(x, T_max)
##  2.5% 97.5% 
##    15    15
# Observation: to estimate the maximum, we need sufficient probability mass near the maximum value the distribution can yield.
# Using bootstrap is pointless when there is too little mass near the true maximum.
# In general, bootstrap will fail when estimating the CI for the maximum.