@Inbook{Vogel2020,
author="Vogel, Carl
and Koutsombogera, Maria
and Costello, Rachel",
editor="Esposito, Anna
and Faundez-Zanuy, Marcos
and Morabito, Francesco Carlo
and Pasero, Eros",
title="Analyzing Likert Scale Inter-annotator Disagreement",
bookTitle="Neural Approaches to Dynamics of Signal Exchanges",
year="2020",
publisher="Springer Singapore",
address="Singapore",
pages="383--393",
abstract="Assessment of annotation reliability is typically undertaken as a qualityVogel, Carl assurance measure in order to provide a sound fulcrum for establishingKoutsombogera, Maria the answers to research questions that requireCostello, Rachel the annotated data. We argue that the assessment of inter-rater reliability can provide a source of information more directly related to the background research. The discussion is anchored in the analysis of conversational dominance in the MULTISIMO corpus. Other research has explored factors in dialogue (e.g. big-five personality traits and conversational style of participants) as predictors of independently perceived dominance. Rather than assessing the contributions of experimental factors to perceived dominance as a unitary aggregated response variable following verification of an acceptable level of inter-rater reliability, we use the variability in inter-annotator agreement as a response variable. We argue the general applicability of this in exploring research hypotheses that focus on qualities assessed with multiple annotations.",
isbn="978-981-13-8950-4",
doi="10.1007/978-981-13-8950-4_34",
url="https://doi.org/10.1007/978-981-13-8950-4_34"
}