@inproceedings{299e64fa76bd487cad965c46317670fb,
title = "Aleph: An indicator of decision-making patterns within collaborative marking sessions for writing assessment that can predict reliability",
abstract = "Assessing writing is difficult due to language inherent subjectivity. Rubrics can help tackle this problem, although collaborative training sessions on their interpretation are required in order to reach consensus between markers and ensure minimum reliability levels. This article reports the construction of Aleph; an indicator built on raters' decision-making behavior in collaborative environments, which can explain agreement between markers. Evaluated within real training sessions of three markers for a high-stake written test, it showed excellent results on identifying markers' patterns and a good correlation with Fleiss' Kappa on agreement.",
keywords = "Collaborative marking, Inter-rater agreement, Rubric training",
author = "Nicolas Herrera and Jorge Villalon and Gonzalo Munoz and Gabriela Baez",
note = "Funding Information: ACKNOWLEDGMENT This study was partially funded by projects FONDECYT 11130055, FONDEF IT 15 i10002 and Basal Fund FB0003. Publisher Copyright: {\textcopyright} 2020 IEEE.; null ; Conference date: 06-07-2020 Through 09-07-2020",
year = "2020",
month = jul,
doi = "10.1109/ICALT49669.2020.00041",
language = "English",
series = "Proceedings - IEEE 20th International Conference on Advanced Learning Technologies, ICALT 2020",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "114--116",
editor = "Maiga Chang and Sampson, {Demetrios G} and Ronghuai Huang and Danial Hooshyar and Nian-Shing Chen and Kinshuk Kinshuk and Margus Pedaste",
booktitle = "Proceedings - IEEE 20th International Conference on Advanced Learning Technologies, ICALT 2020",
}