OpenShift SUM Benchmarking

Statistical side note:

This does not use a benchmark value (which based on research would be 78%), but takes the mean completion proportion rate per task to convert directly to a percentage. This has its flaws (mainly an inflated value of midrange completion rates from 50%-78%). The alternative would be to use a bernoulli variance calculation with the benchmark of 78%; this is perhaps more problematic as the small sample size leaves us with extreme confidence intervals on the edge of possible proportional outcomes.

#create completion df ----
df %>%
  group_by(group,task) %>% #group analyses broadly by product/version group and then by each task
  summarise(pass=sum(comp),n=n()) %>% #get n successes and n trials
  mutate(prop = pass / n) %>% #exact proportion from succesess/trials
  mutate(laplace = (pass + 1) / (n + 2)) %>% #laplace point estimate
  mutate(p_adj = (n * prop + (zval * zval) / 2) / (n + (zval * zval))) %>% #adjust p for wald calculation
  mutate(n_adj = n + (zval * zval)) %>% #adjust n for wald calculation
  mutate(marg =  zval * sqrt(p_adj * (1 - p_adj) / n_adj)) %>% #wald margin value
  mutate(lowerci = p_adj - marg) %>% #lower wald ci
  mutate(lowerci = ifelse(lowerci <= 0, 0, lowerci)) %>% #keep lower ci above 0
  mutate(upperci = p_adj + marg) %>% #upper wald ci
  mutate(upperci = ifelse(upperci >= 1, 1, upperci)) %>% #keep upper ci below 1
  mutate(point_est.z = qnorm(laplace) ) %>% #z score transform based on .78 baseline and bernouli variance
  mutate(lowerci.z= qnorm(laplace)-qnorm(marg) ) %>% #z score transform for conf intervals
  mutate(upperci.z = qnorm(laplace)+qnorm(marg) ) %>% #z score transform for conf intervals
  rename(point.est=laplace) %>% #rename
  mutate(point_est.nps=(point.est - .5) * 200)%>% #nps-ify
  mutate(lowerci.nps=(lowerci- .5 )* 200)%>% #nps-ify
  mutate(upperci.nps=(upperci- .5 )* 200)%>%# nps-ify
  mutate(Measure="Completion") %>% #name measure as var
  mutate(spec=.78 #define spec var for raw plots
  ) -> df_comp

head(df_comp)

## # A tibble: 6 x 19
## # Groups:   group [2]
##   group  task  pass     n  prop point.est p_adj n_adj   marg lowerci
##   <int> <int> <int> <int> <dbl>     <dbl> <dbl> <dbl>  <dbl>   <dbl>
## 1     1     1    20    20  1        0.955 0.941  22.7 0.0813  0.859 
## 2     1     2    20    20  1        0.955 0.941  22.7 0.0813  0.859 
## 3     1     3     4    20  0.2      0.227 0.236  22.7 0.146   0.0895
## 4     1     4    11    20  0.55     0.545 0.544  22.7 0.171   0.373 
## 5     2     1    13    13  1        0.933 0.914  15.7 0.116   0.798 
## 6     2     2    13    13  1        0.933 0.914  15.7 0.116   0.798 
## # … with 9 more variables: upperci <dbl>, point_est.z <dbl>,
## #   lowerci.z <dbl>, upperci.z <dbl>, point_est.nps <dbl>,
## #   lowerci.nps <dbl>, upperci.nps <dbl>, Measure <chr>, spec <dbl>

Then we combine all our newly created dataframes into one. We also clean up some labels.

#bind all the dataframes we just made into one.
bind_rows(df_comp,df_sat,df_time) -> df_summarised 

df_summarised <- df_summarised %>% 
                    mutate(Group=recode_factor(group,"1"="Version A","2"="Version B")) #rename product groups, ignore errors

## Warning in mutate_impl(.data, dots, caller_env()): Unequal factor levels:
## coercing to character

## Warning in mutate_impl(.data, dots, caller_env()): binding character and
## factor vector, coercing into character vector

## Warning in mutate_impl(.data, dots, caller_env()): binding character and
## factor vector, coercing into character vector

df_summarised$Group <- as.factor(df_summarised$Group) #make it a factor so it will plot properly, not as numeric
df_summarised$Group <- factor(df_summarised$Group, levels = c("Version A","Version B"))  #reorder factor groups so Version B is last

#add better labels for task varirable, mainly used for plotting more descriptively
df_summarised$task_named[df_summarised$task==1] <- "Task 1: Front End"

## Warning: Unknown or uninitialised column: 'task_named'.

df_summarised$task_named[df_summarised$task==2] <- "Task 2: Database"
df_summarised$task_named[df_summarised$task==3] <- "Task 3: Connect"
df_summarised$task_named[df_summarised$task==4] <- "Task 4: Debug"

###Plotting

We can begin to plot our data out. This will give us a sense of what tasks have changes in usability metrics. Let’s look at the raw metrics.

#completion
df_summarised %>%
  filter(Measure=="Completion") %>% #only completion
  ggplot(aes(x=task_named, y=point.est,fill=Group)) +  #variables to plot
  geom_bar(aes(fill=Group), #make a bar plot
           position=position_dodge(), #cluster the bars
           stat="identity") + #use raw numbers to bar location
  geom_abline(intercept=.78,slope=0, color = "gray",linetype = 2, size=2) + #horizontal benchmark line
  coord_cartesian(ylim=c(0,1)) + #limit y axis between 0-1
  geom_errorbar(aes(ymin=lowerci, ymax=upperci),position=position_dodge(.9), stat="identity",color="darkgray",width=.2) + #add error bars
  scale_y_continuous(labels = scales::percent) + #make the y axis a percentage
  labs(x="Task", y="Estimated proportion") + #add proper axis labels
  ggtitle(label = "Estimated completion rates across all tasks", #add titles
          subtitle = "Confidence intervals at 90%, gray line indicates benchmark") +
  ggthemes::theme_tufte(base_family="GillSans") + #add minimal theming
  theme(
    axis.text.x = element_text(size = 13), #adjust text size
    axis.title.x = element_blank()) #remove x axis title

#satisfaction
df_summarised %>%
  filter(Measure=="Satisfaction") %>% #only Satisfaction
  ggplot(aes(x=task_named, y=point.est,fill=Group,ymin=lowerci, ymax=upperci)) + #variables to plot
  geom_bar(aes(fill=Group),#make a bar plot
           position=position_dodge(), #cluser the bars
           stat="identity") + #use raw numbers to bar location
  geom_abline(intercept=4,slope=0, color = "gray",linetype = 2, size=2)+ #add a line for the benchmark
  coord_cartesian(ylim=c(1,5)) + #constrain y axis to 1 and 5
  geom_errorbar(position=position_dodge(.9), stat="identity",color="darkgray",width=.2) + #add error bars
  labs(x="Task", y="Satisfaction score") + #add titles
  ggtitle(label = "Average satisfaction measure scores across all tasks",
          subtitle = "Confidence Intervals at 90%, gray line indicates benchmark") +
  ggthemes::theme_tufte(base_family="GillSans") +  #add minimal theming
  theme(
    axis.text.x = element_text(size = 13), #adjust text size
    axis.title.x = element_blank())  #remove x axis title

#time
df_summarised %>%
  filter(Measure=="Time") %>%
  group_by(task) %>%
  ggplot(aes(x=task_named, y=point.est,fill=Group)) + 
  geom_bar(aes(fill=Group),position=position_dodge(), stat="identity") +
  geom_errorbar(aes(ymax=spec,ymin=spec),position=position_dodge(), stat="identity", color = "#3C3C3C",linetype = 2, size=2)+
  #coord_cartesian(ylim=c(0,1)) +
  geom_errorbar(aes(ymin=lowerci, ymax=upperci),position=position_dodge(.9), stat="identity",color="darkgray",width=.2) +
  labs(x="Task", y="Time geometric average in seconds") +
  ggtitle(label = "Raw time averages across all tasks",
          subtitle = "Confidence Intervals at 90%, gray lines indicate benchmark") +
  #scale_y_time(labels=date_labels("%M $S"))+
  facet_grid(.~task, scales="free")+
  ggthemes::theme_tufte(base_family="GillSans") + 
  scale_y_reverse() +
  theme(
    axis.text.x = element_text(size = 13),
    axis.title.x = element_blank(),
    #legend.position="none",
    strip.background = element_blank(), #remove face label area
    strip.text.x = element_blank() #remove facet label text
  ) 

It’s clear after some time that Task 1 is in the best shape, closely followed by Task 2. Task 3 is in the roughest position and Task 4 is around the middle. There is some improvement across these metrics from Version A to B. We can use our sub-task converted scores to compare measures within tasks on a standard scale.

#all sub scores on nps version
df_summarised %>%
  group_by(task_named) %>%
  ggplot(aes(x=Measure, y=point_est.nps,fill=Group)) + 
  geom_bar(aes(fill=Group),position=position_dodge(), stat="identity") +
  coord_cartesian(ylim=c(-100,100)) +
  geom_errorbar(aes(ymin=lowerci.nps, ymax=upperci.nps),position=position_dodge(.9), stat="identity",color="gray",width=.2) +
  labs(x="", y="Standard score") +
  ggtitle(label = "Standardized Scores for All Tasks",
          subtitle = "Confidence Intervals at 90%") +
  ggthemes::theme_tufte(base_family="GillSans") + 
  facet_wrap(.~task_named) 

Again, we get the same information as going over the raw plots. And again, it takes a few moments to visually analyze all the data. If we (or a stakeholder) had less time, we can summarize this by looking at the task-level SUM scores.

We need to make a new dataframe for this more summarized view to get a single score for each task based on the sub-metrics. Again, we add some nicer version number labels and descriptive task names.

#Task level SUM score----- 
df_summarised %>% 
  group_by(group,task) %>% #keep product/versions separate, and only aggregate at task level
  summarise(point_est.z=mean(point_est.z), #average of point estimate std values
            lowerci.z=mean(lowerci.z), #average of lower CI std values
            upperci.z=mean(upperci.z), #average of upper CI std values
            point_est.nps=mean(point_est.nps), #average of pointe estimate NPS-like values
            lowerci.nps=mean(lowerci.nps), #average of lower CI NPS-like values
            upperci.nps=mean(upperci.nps) #average of upper CI NPS-like values
  ) -> df_task

#get better labels for group variable
df_task <- df_task %>%
  mutate(Group=as.factor(recode(group,"1"="Version A","2"="Version B")))

## Warning in mutate_impl(.data, dots, caller_env()): Unequal factor levels:
## coercing to character

## Warning in mutate_impl(.data, dots, caller_env()): binding character and
## factor vector, coercing into character vector

## Warning in mutate_impl(.data, dots, caller_env()): binding character and
## factor vector, coercing into character vector

df_task$Group <- as.factor(df_task$Group)
df_task$Group <- factor(df_task$Group, levels = c("Version A","Version B"))


#add better labels for task varirable
df_task$task_named[df_task$task==1] <- "Task 1: Front End"

## Warning: Unknown or uninitialised column: 'task_named'.

df_task$task_named[df_task$task==2] <- "Task 2: Database"
df_task$task_named[df_task$task==3] <- "Task 3: Connect"
df_task$task_named[df_task$task==4] <- "Task 4: Debug"

With this new dataframe, we can plot task-level scores.

df_task %>%
  ggplot(aes(x=task_named, y=point_est.nps, fill=Group)) + #choose plotting variables
  geom_bar(aes(fill=Group),position=position_dodge(), stat="identity") + #add a clustered bar
  geom_errorbar(aes(ymin=lowerci.nps,  #add error bars
                    ymax=upperci.nps),
                position=position_dodge(.9), 
                stat="identity",
                color="gray",width=.2) +
  coord_cartesian(ylim=c(-100,100)) + #choose nps-like axis limits
  labs(title="SUM scores for each measure on all tasks",
      subtitle = "Confidence Intervals at 90%",
      x="", 
      y="SUM scores") +
  ggthemes::theme_tufte(base_family="GillSans") + #minimal theming
  theme(
    axis.text.x = element_text(size = 10),
    axis.title.x = element_blank())

If we need the simplest, highest level summary, we can plot it by version. (Of course, the sub-metrics are still available for digging into if need be.)

Now the dataframe will come down to just as many rows as you have versions (two, in our case).

#Overall SUM score----

df_summarised %>%
  group_by(group) %>% #keep products separate
  summarise(point_est.z=mean(point_est.z), #average point estimates
            lowerci.z=mean(lowerci.z), #average lower CI
            upperci.z=mean(upperci.z) #average upper CI
  ) %>%
  mutate(point_est.nps=(point_est.z - .5) * 200)%>% #nps-ify
  mutate(lowerci.nps=(lowerci.z- .5 )* 200)%>% #nps-ify
  mutate(upperci.nps=(upperci.z- .5 )* 200 #nps-ify
  )%>%
  mutate(Group=as.factor(recode(group,"1"="Version A","2"="Version B"))) %>% #better labels for version
  mutate(Group=factor(Group, levels = c("Version A","Version B")) #order labels for version
         )-> df_SUM

head(df_SUM)

## # A tibble: 2 x 8
##   group point_est.z lowerci.z upperci.z point_est.nps lowerci.nps
##   <int>       <dbl>     <dbl>     <dbl>         <dbl>       <dbl>
## 1     1       0.516     0.853     0.183          3.22        70.6
## 2     2       0.602     0.873     0.346         20.4         74.7
## # … with 2 more variables: upperci.nps <dbl>, Group <fct>

The final plot will display the version-level scores for the most summarized comparison.

df_SUM %>% 
  ggplot(aes(x=Group, y=point_est.nps,fill=Group,ymin=lowerci.nps, ymax=upperci.nps)) + #maps variables
  geom_bar(position=position_dodge(), stat="identity") + #bar graph
  coord_cartesian(ylim=c(-100,100)) + #specificy y axis limits
  geom_errorbar(position=position_dodge(), stat="identity",color="gray",width=.2) + #add error bars
  geom_text(aes(label = round(point_est.nps,0),hjust=2,vjust=1.5,y=0), size = 10, position = "identity") + #add some text for the final SUM numbers
  labs(title = "Final Product SUM Score",
          subtitle = "Confidence Intervals at 90%",
          y="SUM scores",
          x="") +
  ggthemes::theme_tufte(base_family="GillSans") #minimal theming

And here we have it, the SUM score of version B is markedly higher than version A. This is good evidence of improvement between the two versions.