Analysis
tdata <- read_csv("exp_data.csv")# Exclude subjects who failed the control questions
tdata <- subset(tdata, desktop_conf == "1: I confirm" & attent_conf == "1: I will take it seriously")
tdata <- subset(tdata, intro_check == "2: A little squid swam from the right to the left.")
tdata <- subset(tdata, timing_check_correct == "correct")
tdata <- subset(tdata, learning_check_correct == "correct")1 Subject demographics
# demographics
min(tdata$age)## [1] 18max(tdata$age)## [1] 79mean(tdata$age)## [1] 37.65417sd(tdata$age)## [1] 12.66899# 1 = male, 2 = female, 3 = other
table(tdata$gender)##
## 1: male 2: female 3: non-binary
## 134 100 4
## 4: prefer not to say
## 21 = male, 2 = female, 3 = non-binary, 4 = prefer not to say
2 Data preparations
Check n in each condition:
table(tdata$condition)##
## 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
## 15 15 15 15 15 15 15 15 15 15 15 15 15 15 15 15table(tdata$structure, tdata$test_statement_type)##
## maintainer_first maintainer_second trigger_first trigger_second
## irreversible 30 30 30 30
## reversible 30 30 30 30Factorize:
tdata$structure <- factor(tdata$structure, levels = c("irreversible", "reversible"), labels = c("irreversible", "reversible"))
tdata$test_statement_type <- factor(tdata$test_statement_type, levels = c("trigger_first", "trigger_second", "maintainer_first", "maintainer_second"),
labels = c("first \ncaused", "second \ncaused","first \nmaintained", "second \nmaintained"))3 Graphs and summaries
myTheme <- theme(plot.title = element_text(face="bold", size = 22),
axis.title.x = element_text(size = 20),
axis.title.y = element_text(size = 20),
axis.text.x = element_text(size = 14, angle = 0),
axis.text.y = element_text(size = 16, angle = 0),
legend.text = element_text(size = 18),
legend.title = element_text(face = "bold", size = 18),
strip.text.x = element_text(size = 18),
panel.grid.major = element_blank(),
panel.grid.minor = element_blank(),
panel.background = element_blank(),
axis.line.x = element_line(colour = "black"),
axis.line.y = element_line(colour = "black"),
axis.text = element_text(colour ="black"),
axis.ticks = element_line(colour ="black"))
tdata_long <- tdata
tdata_sub <- tdata_long
library(see)
## first, turn sID into a factor
tdata_sub$subj_code <- factor(tdata_sub$subj_code)
pd <- position_dodge(width = 0.3)
tdata_sub$valueJitter <- jitter(tdata_sub$statement_rating, factor = 0.01, amount = 0.004)
theme_set(theme_light(base_size = 20, base_family = "Poppins"))
g <- ggplot(tdata_sub, aes(x = test_statement_type, y = valueJitter)) +
guides(fill=FALSE)+
facet_grid( ~ structure,labeller = label_both)+
scale_y_continuous(limits = c(-0.3, 10.3), breaks=seq(0, 10, 1), expand = c(0,0)) +
geom_jitter(aes(color = structure), alpha = 0.5, width = 0.15, height = 0.2) +
stat_summary(aes(y = statement_rating, group=1), fun.data = mean_cl_boot,
geom = "errorbar", width = 0, size = 1) +
stat_summary(aes(y = statement_rating, group=1, color = structure), fun.y=mean, geom="line",
shape = 22, size = 1.5, alpha = .7)+
stat_summary(aes(y = statement_rating, group=1, fill = structure), fun.y=mean, geom="point",
color = "black", shape = 22, size = 3, group=1, alpha = 1)+
stat_summary(aes(y = statement_rating,group=1), fun.y=median, geom="point", color = "black", shape = 3, size = 4,
group=1, alpha = 1, position = position_dodge(width = 0.5))+
labs(x = "Causal Statement", y = "Agreement Rating") +
scale_color_manual(name = "Strength",values=c("#66c2a5", "#e78ac3", "#8da0cb", "#a6d854"))+
scale_fill_manual(name = "Strength",values=c("#66c2a5", "#e78ac3", "#8da0cb", "#a6d854"))+
theme(legend.position = "none")+
myTheme+
theme(panel.grid.major = element_line(color = "lightgrey",
size = 0.5,
linetype = 'dotted'))+
stat_summary(aes(label=round(after_stat(y),2)), fun.y=mean, geom="text", size=5,
vjust = -6)+
annotate("text", x = 0.5, y = 2, label = c("completely disagree"), angle = 90)+
annotate("text", x = 0.5, y = 8.2, label = c("completely agree"), angle = 90)
g
ggsave("results_means_mainDV.svg",width=12,height=5)
ggsave("results_means_mainDV.pdf",width=12,height=5)library(ggridges)
g2 <- ggplot(tdata_long, aes(x = statement_rating, y = test_statement_type, fill = structure)) +
facet_grid( ~ structure,labeller = label_both)+
scale_x_continuous(breaks = seq(0, 10, 1))+
geom_density_ridges(alpha = 0.5)+
#stat_summary(aes(x = rating_rec), fun.x=mean, geom="point",
# color = "black", shape = 22, size = 2, group=1, alpha = 1)+
scale_fill_manual(values=c("#66c2a5", "#e78ac3", "#8da0cb", "#a6d854"))+
#scale_fill_viridis_c(name = "Explanation \nRating", option = "C", breaks=c(-5,0,5), labels=c("narrow scope", "no preference", "broad scope"))+
labs(x = "Agreement Rating", y = "Causal Statement") +
scale_y_discrete(limits=rev)+
myTheme+
theme_light(base_family = "Poppins", base_size = 20)+
theme(panel.grid = element_blank(), axis.text = element_text(colour ="black"))+
theme(legend.position="none",
legend.title=element_blank(),legend.key.width = unit(1.95, 'cm'))+
theme(axis.text.y = element_text(size = 14, angle = 0))+
annotate("text", y = 0.7, x = 0, label = c("completely disagree"), angle = 0)+
annotate("text", y = 0.7, x = 10, label = c("completely agree"), angle = 0)
g2
ggsave("results_dist.svg",width=12,height=5)
ggsave("results_dist.pdf",width=12,height=5)3.1 Between-subejcts ANOVA
library(afex)
library(emmeans)
a1 <- aov_car(statement_rating ~ structure*test_statement_type + Error(subj_code), tdata_long, anova_table = list(es = "pes"))
a1## Anova Table (Type 3 tests)
##
## Response: statement_rating
## Effect df MSE F pes p.value
## 1 structure 1, 232 8.40 5.16 * .022 .024
## 2 test_statement_type 3, 232 8.40 63.28 *** .450 <.001
## 3 structure:test_statement_type 3, 232 8.40 32.29 *** .295 <.001
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '+' 0.1 ' ' 1Predicted interaction effect was significant-
Visualize also the obtained main effect of “structure” (reversibility)
tdata_sub <- tdata_long
library(see)
## first, turn sID into a factor
tdata_sub$subj_code <- factor(tdata_sub$subj_code)
pd <- position_dodge(width = 0.3)
tdata_sub$valueJitter <- jitter(tdata_sub$statement_rating, factor = 0.01, amount = 0.004)
theme_set(theme_light(base_size = 20, base_family = "Poppins"))
g <- ggplot(tdata_sub, aes(x = structure, y = valueJitter)) +
guides(fill=FALSE)+
scale_y_continuous(limits = c(-0.3, 10.3), breaks=seq(0, 10, 1), expand = c(0,0)) +
geom_jitter(aes(color = structure), alpha = 0.5, width = 0.15, height = 0.2) +
stat_summary(aes(y = statement_rating, group=1), fun.data = mean_cl_boot,
geom = "errorbar", width = 0, size = 1) +
stat_summary(aes(y = statement_rating, group=1, fill = structure), fun.y=mean, geom="point",
color = "black", shape = 22, size = 3, group=1, alpha = 1)+
stat_summary(aes(y = statement_rating,group=1), fun.y=median, geom="point", color = "black", shape = 3, size = 4,
group=1, alpha = 1, position = position_dodge(width = 0.5))+
labs(x = "Causal Statement", y = "Agreement Rating") +
scale_color_manual(name = "Strength",values=c("#66c2a5", "#e78ac3", "#8da0cb", "#a6d854"))+
scale_fill_manual(name = "Strength",values=c("#66c2a5", "#e78ac3", "#8da0cb", "#a6d854"))+
theme(legend.position = "none")+
myTheme+
theme(panel.grid.major = element_line(color = "lightgrey",
size = 0.5,
linetype = 'dotted'))+
stat_summary(aes(label=round(after_stat(y),2)), fun.y=mean, geom="text", size=5,
vjust = -6)+
annotate("text", x = 0.5, y = 2, label = c("completely disagree"), angle = 90)+
annotate("text", x = 0.5, y = 8.2, label = c("completely agree"), angle = 90)
g
Plot main effect of causal statement:
tdata_sub <- tdata_long
library(see)
## first, turn sID into a factor
tdata_sub$subj_code <- factor(tdata_sub$subj_code)
pd <- position_dodge(width = 0.3)
tdata_sub$valueJitter <- jitter(tdata_sub$statement_rating, factor = 0.01, amount = 0.004)
theme_set(theme_light(base_size = 20, base_family = "Poppins"))
g <- ggplot(tdata_sub, aes(x = test_statement_type, y = valueJitter)) +
guides(fill=FALSE)+
scale_y_continuous(limits = c(-0.3, 10.3), breaks=seq(0, 10, 1), expand = c(0,0)) +
geom_jitter(aes(color = test_statement_type), alpha = 0.5, width = 0.15, height = 0.2) +
stat_summary(aes(y = statement_rating, group=1), fun.data = mean_cl_boot,
geom = "errorbar", width = 0, size = 1) +
stat_summary(aes(y = statement_rating, group=1, fill = test_statement_type), fun.y=mean, geom="point",
color = "black", shape = 22, size = 3, group=1, alpha = 1)+
stat_summary(aes(y = statement_rating,group=1), fun.y=median, geom="point", color = "black", shape = 3, size = 4,
group=1, alpha = 1, position = position_dodge(width = 0.5))+
labs(x = "Causal Statement", y = "Agreement Rating") +
scale_color_manual(name = "Strength",values=c("#66c2a5", "#e78ac3", "#8da0cb", "#a6d854"))+
scale_fill_manual(name = "Strength",values=c("#66c2a5", "#e78ac3", "#8da0cb", "#a6d854"))+
theme(legend.position = "none")+
annotate("text", x = 0.5, y = 1.7, label = c("completely disagree"), angle = 90)+
annotate("text", x = 0.5, y = 6.7, label = c("completely agree"), angle = 90)+
myTheme+
theme(panel.grid.major = element_line(color = "lightgrey",
size = 0.5,
linetype = 'dotted'))+
stat_summary(aes(label=round(after_stat(y),2)), fun.y=mean, geom="text", size=5,
vjust = -6)
g
3.2 Contrasts
library(lsmeans)
# means
ls2 <- lsmeans(a1, c("structure", "test_statement_type"))
ls2## structure test_statement_type lsmean SE df lower.CL upper.CL
## irreversible first \ncaused 9.63 0.529 232 8.591 10.68
## reversible first \ncaused 8.70 0.529 232 7.657 9.74
## irreversible second \ncaused 0.30 0.529 232 -0.743 1.34
## reversible second \ncaused 3.67 0.529 232 2.624 4.71
## irreversible first \nmaintained 6.80 0.529 232 5.757 7.84
## reversible first \nmaintained 2.57 0.529 232 1.524 3.61
## irreversible second \nmaintained 3.27 0.529 232 2.224 4.31
## reversible second \nmaintained 8.47 0.529 232 7.424 9.51
##
## Confidence level used: 0.95contrasts <- emmeans(a1, ~ test_statement_type|structure)
s <- pairs(contrasts, adjust = "none")
s## structure = irreversible:
## contrast estimate SE df t.ratio p.value
## first \ncaused - second \ncaused 9.333 0.749 232 12.469 <.0001
## first \ncaused - first \nmaintained 2.833 0.749 232 3.785 0.0002
## first \ncaused - second \nmaintained 6.367 0.749 232 8.506 <.0001
## second \ncaused - first \nmaintained -6.500 0.749 232 -8.684 <.0001
## second \ncaused - second \nmaintained -2.967 0.749 232 -3.963 0.0001
## first \nmaintained - second \nmaintained 3.533 0.749 232 4.720 <.0001
##
## structure = reversible:
## contrast estimate SE df t.ratio p.value
## first \ncaused - second \ncaused 5.033 0.749 232 6.724 <.0001
## first \ncaused - first \nmaintained 6.133 0.749 232 8.194 <.0001
## first \ncaused - second \nmaintained 0.233 0.749 232 0.312 0.7555
## second \ncaused - first \nmaintained 1.100 0.749 232 1.470 0.1430
## second \ncaused - second \nmaintained -4.800 0.749 232 -6.413 <.0001
## first \nmaintained - second \nmaintained -5.900 0.749 232 -7.882 <.0001confint(s, level = 0.95)## structure = irreversible:
## contrast estimate SE df lower.CL upper.CL
## first \ncaused - second \ncaused 9.333 0.749 232 7.859 10.81
## first \ncaused - first \nmaintained 2.833 0.749 232 1.359 4.31
## first \ncaused - second \nmaintained 6.367 0.749 232 4.892 7.84
## second \ncaused - first \nmaintained -6.500 0.749 232 -7.975 -5.03
## second \ncaused - second \nmaintained -2.967 0.749 232 -4.441 -1.49
## first \nmaintained - second \nmaintained 3.533 0.749 232 2.059 5.01
##
## structure = reversible:
## contrast estimate SE df lower.CL upper.CL
## first \ncaused - second \ncaused 5.033 0.749 232 3.559 6.51
## first \ncaused - first \nmaintained 6.133 0.749 232 4.659 7.61
## first \ncaused - second \nmaintained 0.233 0.749 232 -1.241 1.71
## second \ncaused - first \nmaintained 1.100 0.749 232 -0.375 2.57
## second \ncaused - second \nmaintained -4.800 0.749 232 -6.275 -3.33
## first \nmaintained - second \nmaintained -5.900 0.749 232 -7.375 -4.43
##
## Confidence level used: 0.95The planned contrasts from the list above are ” first..caused - second..caused” in both “irreversible” and “reversible”. Both are significant.
Get effect standardized effect sizes for the contrasts
First get the SDs that need to be pooled.
library(dplyr)
tdata_long %>%
group_by(structure, test_statement_type) %>%
summarise_at(vars(statement_rating), list(name=sd))## # A tibble: 8 × 3
## # Groups: structure [2]
## structure test_statement_type name
## <fct> <fct> <dbl>
## 1 irreversible "first \ncaused" 1.67
## 2 irreversible "second \ncaused" 0.651
## 3 irreversible "first \nmaintained" 3.47
## 4 irreversible "second \nmaintained" 4.10
## 5 reversible "first \ncaused" 2.51
## 6 reversible "second \ncaused" 3.24
## 7 reversible "first \nmaintained" 3.41
## 8 reversible "second \nmaintained" 2.60# using the functions from the MOTE package (see https://matthewbjane.quarto.pub/guide-to-effect-sizes-and-confidence-intervals/Standardized-Mean-Differences.html#sec-two-ind-group-pooled
library(MOTE)
# irreversible condition
stats <- d.ind.t(
m1 = 9.63,
m2 = 0.30,
sd1 = 1.6709141,
sd2 = 0.6512587,
n1 = 30,
n2 = 30,
a = 0.05
)
stats$estimate## [1] "$d_s$ = 7.36, 95\\% CI [5.92, 8.78]"# reversible condition
#structure entity lsmean SE df lower.CL upper.CL
# reversible first..caused 9.40 0.178 98 9.046 9.75
# reversible second..caused 2.74 0.408 98 1.930 3.55
stats <- d.ind.t(
m1 = 8.70,
m2 = 3.67,
sd1 = 2.5072309,
sd2 = 3.2412570,
n1 = 30,
n2 = 30,
a = 0.05
)
stats$estimate## [1] "$d_s$ = 1.74, 95\\% CI [1.13, 2.33]"The final planned contrast compares the maintainer ratings for the second cause in “irreversible” and “reversible”. It must be looked up from the long list of contrasts below:
contrasts <- emmeans(a1, ~ test_statement_type*structure)
s <- pairs(contrasts, adjust = "none")
s## contrast estimate
## first \ncaused irreversible - second \ncaused irreversible 9.333
## first \ncaused irreversible - first \nmaintained irreversible 2.833
## first \ncaused irreversible - second \nmaintained irreversible 6.367
## first \ncaused irreversible - first \ncaused reversible 0.933
## first \ncaused irreversible - second \ncaused reversible 5.967
## first \ncaused irreversible - first \nmaintained reversible 7.067
## first \ncaused irreversible - second \nmaintained reversible 1.167
## second \ncaused irreversible - first \nmaintained irreversible -6.500
## second \ncaused irreversible - second \nmaintained irreversible -2.967
## second \ncaused irreversible - first \ncaused reversible -8.400
## second \ncaused irreversible - second \ncaused reversible -3.367
## second \ncaused irreversible - first \nmaintained reversible -2.267
## second \ncaused irreversible - second \nmaintained reversible -8.167
## first \nmaintained irreversible - second \nmaintained irreversible 3.533
## first \nmaintained irreversible - first \ncaused reversible -1.900
## first \nmaintained irreversible - second \ncaused reversible 3.133
## first \nmaintained irreversible - first \nmaintained reversible 4.233
## first \nmaintained irreversible - second \nmaintained reversible -1.667
## second \nmaintained irreversible - first \ncaused reversible -5.433
## second \nmaintained irreversible - second \ncaused reversible -0.400
## second \nmaintained irreversible - first \nmaintained reversible 0.700
## second \nmaintained irreversible - second \nmaintained reversible -5.200
## first \ncaused reversible - second \ncaused reversible 5.033
## first \ncaused reversible - first \nmaintained reversible 6.133
## first \ncaused reversible - second \nmaintained reversible 0.233
## second \ncaused reversible - first \nmaintained reversible 1.100
## second \ncaused reversible - second \nmaintained reversible -4.800
## first \nmaintained reversible - second \nmaintained reversible -5.900
## SE df t.ratio p.value
## 0.749 232 12.469 <.0001
## 0.749 232 3.785 0.0002
## 0.749 232 8.506 <.0001
## 0.749 232 1.247 0.2137
## 0.749 232 7.971 <.0001
## 0.749 232 9.441 <.0001
## 0.749 232 1.559 0.1204
## 0.749 232 -8.684 <.0001
## 0.749 232 -3.963 0.0001
## 0.749 232 -11.222 <.0001
## 0.749 232 -4.498 <.0001
## 0.749 232 -3.028 0.0027
## 0.749 232 -10.911 <.0001
## 0.749 232 4.720 <.0001
## 0.749 232 -2.538 0.0118
## 0.749 232 4.186 <.0001
## 0.749 232 5.656 <.0001
## 0.749 232 -2.227 0.0269
## 0.749 232 -7.259 <.0001
## 0.749 232 -0.534 0.5936
## 0.749 232 0.935 0.3507
## 0.749 232 -6.947 <.0001
## 0.749 232 6.724 <.0001
## 0.749 232 8.194 <.0001
## 0.749 232 0.312 0.7555
## 0.749 232 1.470 0.1430
## 0.749 232 -6.413 <.0001
## 0.749 232 -7.882 <.0001confint(s, level = 0.95)## contrast estimate
## first \ncaused irreversible - second \ncaused irreversible 9.333
## first \ncaused irreversible - first \nmaintained irreversible 2.833
## first \ncaused irreversible - second \nmaintained irreversible 6.367
## first \ncaused irreversible - first \ncaused reversible 0.933
## first \ncaused irreversible - second \ncaused reversible 5.967
## first \ncaused irreversible - first \nmaintained reversible 7.067
## first \ncaused irreversible - second \nmaintained reversible 1.167
## second \ncaused irreversible - first \nmaintained irreversible -6.500
## second \ncaused irreversible - second \nmaintained irreversible -2.967
## second \ncaused irreversible - first \ncaused reversible -8.400
## second \ncaused irreversible - second \ncaused reversible -3.367
## second \ncaused irreversible - first \nmaintained reversible -2.267
## second \ncaused irreversible - second \nmaintained reversible -8.167
## first \nmaintained irreversible - second \nmaintained irreversible 3.533
## first \nmaintained irreversible - first \ncaused reversible -1.900
## first \nmaintained irreversible - second \ncaused reversible 3.133
## first \nmaintained irreversible - first \nmaintained reversible 4.233
## first \nmaintained irreversible - second \nmaintained reversible -1.667
## second \nmaintained irreversible - first \ncaused reversible -5.433
## second \nmaintained irreversible - second \ncaused reversible -0.400
## second \nmaintained irreversible - first \nmaintained reversible 0.700
## second \nmaintained irreversible - second \nmaintained reversible -5.200
## first \ncaused reversible - second \ncaused reversible 5.033
## first \ncaused reversible - first \nmaintained reversible 6.133
## first \ncaused reversible - second \nmaintained reversible 0.233
## second \ncaused reversible - first \nmaintained reversible 1.100
## second \ncaused reversible - second \nmaintained reversible -4.800
## first \nmaintained reversible - second \nmaintained reversible -5.900
## SE df lower.CL upper.CL
## 0.749 232 7.859 10.808
## 0.749 232 1.359 4.308
## 0.749 232 4.892 7.841
## 0.749 232 -0.541 2.408
## 0.749 232 4.492 7.441
## 0.749 232 5.592 8.541
## 0.749 232 -0.308 2.641
## 0.749 232 -7.975 -5.025
## 0.749 232 -4.441 -1.492
## 0.749 232 -9.875 -6.925
## 0.749 232 -4.841 -1.892
## 0.749 232 -3.741 -0.792
## 0.749 232 -9.641 -6.692
## 0.749 232 2.059 5.008
## 0.749 232 -3.375 -0.425
## 0.749 232 1.659 4.608
## 0.749 232 2.759 5.708
## 0.749 232 -3.141 -0.192
## 0.749 232 -6.908 -3.959
## 0.749 232 -1.875 1.075
## 0.749 232 -0.775 2.175
## 0.749 232 -6.675 -3.725
## 0.749 232 3.559 6.508
## 0.749 232 4.659 7.608
## 0.749 232 -1.241 1.708
## 0.749 232 -0.375 2.575
## 0.749 232 -6.275 -3.325
## 0.749 232 -7.375 -4.425
##
## Confidence level used: 0.95The relevant contrast is: “second..maintained irreversible - second..maintained reversible”, which is significant.
Get effect size:
#structure entity lsmean SE df lower.CL upper.CL
# reversible second \nmaintained 8.47 0.529 232 7.424 9.51
# irreversible second \nmaintained 3.27 0.529 232 2.224 4.31
#SDs
# reversible second \nmaintained 2.5961953
# irreversible second \nmaintained 4.1015837
stats <- d.ind.t(
m1 = 8.47,
m2 = 3.27,
sd1 = 2.5961953,
sd2 = 4.1015837,
n1 = 30,
n2 = 30,
a = 0.05
)
stats$estimate## [1] "$d_s$ = 1.51, 95\\% CI [0.93, 2.09]"