{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,10]],"date-time":"2026-10-10T04:55:56Z","timestamp":1791608156624,"version":"4.3.4"},"reference-count":128,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","license":[{"start":{"date-parts":[[2021,1,1]],"date-time":"2021-01-01T00:00:00Z","timestamp":1609459200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Access"],"published-print":{"date-parts":[[2021]]},"DOI":"10.1109\/access.2021.3116857","type":"journal-article","created":{"date-parts":[[2021,9,29]],"date-time":"2021-09-29T21:37:51Z","timestamp":1632951471000},"page":"136182-136216","source":"Crossref","is-referenced-by-count":27,"title":["Towards Meaningful Statements in IR Evaluation: Mapping Evaluation Measures to Interval Scales"],"prefix":"10.1109","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0894-4175","authenticated-orcid":false,"given":"Marco","family":"Ferrante","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9219-6239","authenticated-orcid":false,"given":"Nicola","family":"Ferro","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0441-6949","authenticated-orcid":false,"given":"Norbert","family":"Fuhr","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1145\/2808194.2809452"},{"key":"ref38","first-page":"331","article-title":"Quantitative estimates of sensory events: Final report of the committee appointed to consider and report upon the possibility of quantitative estimates of sensory events","volume":"1","author":"ferguson","year":"1940","journal-title":"The Advancement of Science"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1145\/290941"},{"key":"ref32","author":"cozby","year":"2018","journal-title":"Methods in Behavioral Research"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291009"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1108\/eb050097"},{"key":"ref37","doi-asserted-by":"publisher","DOI":"10.1007\/BF00485824"},{"key":"ref36","year":"2019","journal-title":"European Commission Commission Delegated Regulation (EC) No 2019\/33 of 17 October 2018 Supplementing Regulation (EU) No 1308\/2013 of the European Parliament and of the Council as Regards Applications for Protection of Designations of Origin Geographical Indications and Traditional Terms in the Wine Sector the Objection Procedure Restrictions of Use Amendments to Product Specifications Cancellation of Protection and Labelling and Presentation"},{"key":"ref35","year":"2009","journal-title":"European Commission Commission Regulation (EC) No 607\/2009 of 14 July 2009 Laying Down Certain Detailed Rules for the Implementation of Council Regulation (EC) No 479\/2008 as Regards Protected Designations of Origin and Geographical Indications Traditional Terms Labelling and Presentation of Certain Wine Sector Products"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809088"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390445"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.1145\/1390334"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277755"},{"key":"ref22","author":"campbell","year":"1920","journal-title":"Elements of Physiology"},{"key":"ref21","author":"b\u00fcttcher","year":"2010","journal-title":"Information Retrieval Implementing and Evaluating Search Engines"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835683"},{"key":"ref23","author":"campbell","year":"1928","journal-title":"An Account of the Principles of Measurement and Calculation"},{"key":"ref101","author":"siegel","year":"1956","journal-title":"Nonparametric statistics for the behavioral science"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1145\/2094072.2094076"},{"key":"ref100","author":"senders","year":"1958","journal-title":"Measurement and Statistics A Basic Text Emphasizing Behavioral Science Applications"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010037"},{"key":"ref50","doi-asserted-by":"crossref","first-page":"115","DOI":"10.2466\/pr0.1959.5.g.115","article-title":"Non-parametric methods in psychological research","volume":"5","author":"gaito","year":"1959","journal-title":"Psychol Rep"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.87.3.564"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1145\/160688.160758"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1002\/SERIES1345"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0271(73)90066-1"},{"key":"ref56","author":"harman","year":"2005","journal-title":"TREC Experiment and Evaluation in Information Retrieval"},{"key":"ref55","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-031-02276-0","author":"harman","year":"2011","journal-title":"Experiment and Evaluation in Information Retrieval"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.2307\/2983326"},{"key":"ref53","author":"gibbons","year":"2011","journal-title":"Nonparametric Statistical Inference"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.3102\/00346543045001043"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.1145\/3121050.3121058"},{"key":"ref4","first-page":"1","article-title":"TREC 2018 common core track overview","author":"allan","year":"2019","journal-title":"Proc 27th Text Retr Conf (TREC)"},{"key":"ref3","first-page":"1","article-title":"TREC 2017 common core track overview","author":"allan","year":"2018","journal-title":"Proc 26th Text Retr Conf (TREC)"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-008-9066-8"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-020-09374-0"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3209978.3210024"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/3190580.3190586"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484081"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-019-09355-y"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.2307\/2279372"},{"key":"ref45","author":"fisher","year":"1925","journal-title":"Statistical Methods for Research Workers"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/2422256.2422259"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.2307\/2279169"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-019-09362-z"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1109\/TKDE.2018.2840708"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-22948-1"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1145\/3106371"},{"key":"ref127","doi-asserted-by":"publisher","DOI":"10.1145\/3077136.3080841"},{"key":"ref126","author":"yu","year":"2006","journal-title":"Proc 15th Int Conf Inf Knowl Manage (CIKM)"},{"key":"ref125","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390435"},{"key":"ref124","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183633"},{"key":"ref73","first-page":"17","article-title":"Axiometrics: Axioms of information retrieval effectiveness metrics","author":"maddalena","year":"2014","journal-title":"Proc 6th Int Workshop Evaluating Inf Access (EVIA)"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1561\/1500000027"},{"key":"ref71","volume":"3","author":"luce","year":"1990","journal-title":"Foundations of Measurement Representation Axiomatization and Invariance"},{"key":"ref128","doi-asserted-by":"publisher","DOI":"10.1145\/3397271.3401162"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1016\/0022-2496(64)90015-X"},{"key":"ref76","author":"michel","year":"1990","journal-title":"An Introduction to the Logic of Psychological Measurement"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-45068-6_1"},{"key":"ref74","doi-asserted-by":"publisher","DOI":"10.3102\/10769986012004383"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.100.3.398"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1145\/1416950.1416952"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2507665"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10137"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/2854946.2855005"},{"key":"ref63","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1561\/1500000012","article-title":"Methods for evaluating interactive information retrieval systems with users","volume":"3","author":"kelly","year":"2009","journal-title":"Found Trends Inf Retr"},{"key":"ref64","author":"kendall","year":"1948","journal-title":"Rank Correlation Methods"},{"key":"ref65","volume":"1","author":"krantz","year":"1971","journal-title":"Foundations of Measurement Additive and Polynomial Representations"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.2307\/2280779"},{"key":"ref67","author":"kutner","year":"2005","journal-title":"Applied Linear Statistical Models"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1037\/h0063675"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1007\/BF02289443"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.1145\/2851613.2851692"},{"key":"ref1","first-page":"226","article-title":"Efficient conversion of non-metric information into metric information","author":"abelson","year":"1959","journal-title":"Proc Social Statist Sect Amer Stat Assoc"},{"key":"ref109","doi-asserted-by":"publisher","DOI":"10.1145\/3331184.3331315"},{"key":"ref95","doi-asserted-by":"publisher","DOI":"10.1561\/1500000009"},{"key":"ref108","doi-asserted-by":"publisher","DOI":"10.2307\/3001913"},{"key":"ref94","doi-asserted-by":"publisher","DOI":"10.1145\/321439.321441"},{"key":"ref107","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.96.2.394"},{"key":"ref93","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-15-5554-1_1"},{"key":"ref106","first-page":"385","article-title":"A statistical Analysis of the TREC-3 Data","author":"tague-sutcliffe","year":"1994","journal-title":"Proc 3rd Text Retr Conf (TREC)"},{"key":"ref92","doi-asserted-by":"publisher","DOI":"10.1145\/3451964.3451976"},{"key":"ref105","volume":"2","author":"suppes","year":"1989","journal-title":"Foundations of Measurement Geometrical Threshold and Probabilistic Representations"},{"key":"ref91","doi-asserted-by":"publisher","DOI":"10.1145\/2911451.2911492"},{"key":"ref104","doi-asserted-by":"crossref","first-page":"1","DOI":"10.2307\/2331554","article-title":"The probable error of a mean","volume":"6","author":"gosset","year":"1908","journal-title":"Biometrika"},{"key":"ref90","doi-asserted-by":"publisher","DOI":"10.1145\/2970398.2970399"},{"key":"ref103","doi-asserted-by":"publisher","DOI":"10.1126\/science.103.2684.677"},{"key":"ref102","first-page":"227","article-title":"AT&T at TREC-6: SDR track","author":"singhal","year":"1997","journal-title":"Proc 6th Text Retr Conf (TREC)"},{"key":"ref111","author":"van rijsbergen","year":"1979","journal-title":"Information Retrieval"},{"key":"ref112","first-page":"32","article-title":"Retrieval effectiveness","author":"van rijsbergen","year":"1981","journal-title":"Information Retrieval Experiment"},{"key":"ref110","doi-asserted-by":"publisher","DOI":"10.1108\/eb026584"},{"key":"ref98","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmp.2009.01.002"},{"key":"ref99","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-019-09363-y"},{"key":"ref96","author":"sauro","year":"2016","journal-title":"Quantifying the User Experience Practical Statistics for User Research"},{"key":"ref97","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(97)00027-7"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1037\/h0042576"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1561\/1500000024"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1023\/A:1009984519381"},{"key":"ref13","first-page":"233","article-title":"Two axioms for evaluation measures in information retrieval","author":"bollmann","year":"1984","journal-title":"Proc 3rd Joint BCS ACM Symp Res Develop Inf Retr"},{"key":"ref14","first-page":"256","article-title":"Measurement-theoretical investigation of the MZ-metric","author":"bollmann","year":"1980","journal-title":"Proc 3rd Annu Int ACM SIGIR Conf Res Develop Inf Retr (SIGIR)"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1145\/511754.511757"},{"key":"ref118","doi-asserted-by":"publisher","DOI":"10.1145\/3269206.3271766"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1145\/1008992.1009000"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1007\/BF00170998"},{"key":"ref117","article-title":"Overview of the TREC 2005 robust retrieval track","author":"voorhees","year":"2005","journal-title":"Proc 14th Text Retr Conf (TREC)"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835654"},{"key":"ref81","author":"narens","year":"2002","journal-title":"Theories Meaningfullness"},{"key":"ref18","author":"burnett","year":"2006","journal-title":"Enterprise Search and Retrieval Unlocking the Organisation’s Potential"},{"key":"ref84","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183630"},{"key":"ref119","first-page":"1","article-title":"Overview of the seventh text REtrieval conference (TREC-7)","author":"voorhees","year":"1998","journal-title":"Proc 7th Text Retr Conf (TREC)"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/2499178.2499182"},{"key":"ref83","doi-asserted-by":"publisher","DOI":"10.1016\/0022-2496(85)90011-2"},{"key":"ref114","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291017"},{"key":"ref113","doi-asserted-by":"publisher","DOI":"10.2307\/2684788"},{"key":"ref116","first-page":"1","article-title":"Overview of the TREC 2004 robust track","author":"voorhees","year":"2004","journal-title":"Proc 13th Text Retr Conf (TREC)"},{"key":"ref80","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-36415-0_10"},{"key":"ref115","doi-asserted-by":"publisher","DOI":"10.1016\/S0306-4573(00)00010-8"},{"key":"ref120","first-page":"1","article-title":"Overview of the Eigth text REtrieval conference (TREC-8)","author":"voorhees","year":"1999","journal-title":"Proc 8th Text Retr Conf (TREC)"},{"key":"ref89","doi-asserted-by":"publisher","DOI":"10.1145\/2641383.2641385"},{"key":"ref121","doi-asserted-by":"publisher","DOI":"10.1002\/sce.3730590417"},{"key":"ref122","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390346"},{"key":"ref123","doi-asserted-by":"publisher","DOI":"10.2307\/3001968"},{"key":"ref85","author":"rowe","year":"2010","journal-title":"Economic Impact Assessment of NIST's Text REtrieval Conference (TREC) Program"},{"key":"ref86","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148261"},{"key":"ref87","doi-asserted-by":"publisher","DOI":"10.1145\/2187836.2187904"},{"key":"ref88","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-54798-0_6"}],"container-title":["IEEE Access"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/6287639\/9312710\/09552926.pdf?arnumber=9552926","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,10]],"date-time":"2023-01-10T19:41:35Z","timestamp":1673379695000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/9552926\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021]]},"references-count":128,"URL":"https:\/\/doi.org\/10.1109\/access.2021.3116857","relation":{},"ISSN":["2169-3536"],"issn-type":[{"value":"2169-3536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021]]}}}