@inproceedings{299e64fa76bd487cad965c46317670fb,
title = "Aleph: An indicator of decision-making patterns within collaborative marking sessions for writing assessment that can predict reliability",
abstract = "Assessing writing is difficult due to language inherent subjectivity. Rubrics can help tackle this problem, although collaborative training sessions on their interpretation are required in order to reach consensus between markers and ensure minimum reliability levels. This article reports the construction of Aleph; an indicator built on raters' decision-making behavior in collaborative environments, which can explain agreement between markers. Evaluated within real training sessions of three markers for a high-stake written test, it showed excellent results on identifying markers' patterns and a good correlation with Fleiss' Kappa on agreement.",
keywords = "Collaborative marking, Inter-rater agreement, Rubric training",
author = "Nicolas Herrera and Jorge Villalon and Gonzalo Munoz and Gabriela Baez",
note = "Publisher Copyright: {\textcopyright} 2020 IEEE.; 20th IEEE International Conference on Advanced Learning Technologies, ICALT 2020 ; Conference date: 06-07-2020 Through 09-07-2020",
year = "2020",
month = jul,
doi = "10.1109/ICALT49669.2020.00041",
language = "English",
series = "Proceedings - IEEE 20th International Conference on Advanced Learning Technologies, ICALT 2020",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "114--116",
editor = "Maiga Chang and Sampson, {Demetrios G} and Ronghuai Huang and Danial Hooshyar and Nian-Shing Chen and Kinshuk Kinshuk and Margus Pedaste",
booktitle = "Proceedings - IEEE 20th International Conference on Advanced Learning Technologies, ICALT 2020",
}