{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,4]],"date-time":"2026-04-04T06:08:10Z","timestamp":1775282890565,"version":"3.50.1"},"reference-count":26,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2013,1,1]],"date-time":"2013-01-01T00:00:00Z","timestamp":1356998400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Int J Multimed Info Retr"],"published-print":{"date-parts":[[2013,3]]},"DOI":"10.1007\/s13735-012-0030-4","type":"journal-article","created":{"date-parts":[[2012,12,31]],"date-time":"2012-12-31T13:14:15Z","timestamp":1356959655000},"page":"59-70","source":"Crossref","is-referenced-by-count":6,"title":["Minimal test collections for low-cost evaluation of Audio Music Similarity and Retrieval systems"],"prefix":"10.1007","volume":"2","author":[{"given":"Juli\u00e1n","family":"Urbano","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Markus","family":"Schedl","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2013,1,1]]},"reference":[{"key":"30_CR1","doi-asserted-by":"crossref","unstructured":"Carterette B (2007) Robust test collections for retrieval evaluation. In: International ACM SIGIR conference on research and development in information retrieval, pp 55\u201362","DOI":"10.1145\/1277741.1277754"},{"key":"30_CR2","doi-asserted-by":"crossref","unstructured":"Carterette B (2008) Low-cost and robust evaluation of information retrieval systems. Ph.D. thesis, University of Massachusetts Amherst","DOI":"10.1145\/1480506.1480527"},{"key":"30_CR3","doi-asserted-by":"crossref","unstructured":"Carterette B, Allan J, Sitaraman R (2006) Minimal test collections for retrieval evaluation. In: International ACM SIGIR conference on research and development in information retrieval, pp 268\u2013275","DOI":"10.1145\/1148170.1148219"},{"key":"30_CR4","unstructured":"Carterette B, Jones R (2007) Evaluating search engines by modeling the relationship between relevance and clicks. In: Annual conference on neural information processing systems"},{"key":"30_CR5","doi-asserted-by":"crossref","unstructured":"Carterette B, Pavlu V, Fang H, Kanoulas E (2009) Million query track 2009 overview. In: Text retrieval conference","DOI":"10.6028\/NIST.SP.500-278.million-query-overview"},{"key":"30_CR6","unstructured":"Downie JS (2003) The MIR\/MDL evaluation project white paper collection, 3rd edn. URL http:\/\/www.music-ir.org\/evaluation\/wp.html"},{"issue":"2","key":"30_CR7","doi-asserted-by":"crossref","first-page":"12","DOI":"10.1162\/014892604323112211","volume":"28","author":"JS Downie","year":"2004","unstructured":"Downie JS (2004) The scientific evaluation of music information retrieval systems: foundations and future. Comput Music J 28(2):12\u201323","journal-title":"Comput Music J"},{"key":"30_CR8","doi-asserted-by":"crossref","unstructured":"Downie JS, Ehmann AF, Bay M, Jones MC (2010) The music information retrieval evaluation exchange: some observations and insights. In: Zbigniew WR, Wieczorkowska AA (eds) Advances in music information retrieval. Springer, Berlin, pp 93\u2013115","DOI":"10.1007\/978-3-642-11674-2_5"},{"issue":"3","key":"30_CR9","doi-asserted-by":"crossref","first-page":"20","DOI":"10.1162\/COMJ_a_00004","volume":"34","author":"A Flexer","year":"2010","unstructured":"Flexer A, Schnitzer D (2010) Effects of album and artist filters in audio similarity computed for very large music databases. Comput Music J 34(3):20\u201328","journal-title":"Comput Music J"},{"issue":"2","key":"30_CR10","first-page":"1","volume":"3","author":"DK Harman","year":"2011","unstructured":"Harman DK (2011) Information retrieval evaluation. Synth Lect Inf Concept Retr Serv 3(2):1\u2013119","journal-title":"Synth Lect Inf Concept Retr Serv"},{"key":"30_CR11","unstructured":"Jones MC, Downie JS, Ehmann AF (2007) Human similarity judgments: implications for the design of formal evaluations. In: International conference on music information retrieval, pp 539\u2013542"},{"key":"30_CR12","doi-asserted-by":"crossref","unstructured":"Liu I, Agresti A (2005) The analysis of ordered categorical data: an overview and a survey of recent developments. Sociedad Estad\u00edstica e Investigaci\u00f3n Operativa Test 14(1):1\u201373","DOI":"10.1007\/BF02595397"},{"key":"30_CR13","unstructured":"Long JS (1997) Regression models for categorical and limited dependent variables, 1st edn. Sage Publications, New York"},{"key":"30_CR14","unstructured":"Pohle T (2010) Automatic characterization of music for intuitive retrieval. Ph.D. thesis, Johannes Kepler University"},{"key":"30_CR15","unstructured":"Salamon J, Urbano J (2012) Current challenges in the evaluation of predominant melody extraction algorithms. In: International society for music information retrieval conference, pp 289\u2013294"},{"key":"30_CR16","doi-asserted-by":"crossref","unstructured":"Soboroff I, Nicholas C, Cahan P (2001) Ranking retrieval systems without relevance judgments. In: International ACM SIGIR conference on research and development in information retrieval, pp 66\u201373","DOI":"10.1145\/383952.383961"},{"key":"30_CR17","unstructured":"Urbano J (2011) Information retrieval meta-evaluation: challenges and opportunities in the music domain. In: International society for music information retrieval conference, pp 609\u2013614"},{"key":"30_CR18","unstructured":"Urbano J, Downie JS, Mcfee B, Schedl M (2012) How significant is statistically significant? The case of audio music similarity and retrieval. In: International society for music information retrieval conference, pp 181\u2013186"},{"key":"30_CR19","unstructured":"Urbano J, Mart\u00edn D, Marrero M, Morato J (2011) Audio music similarity and retrieval: evaluation power and stability. In: International society for music information retrieval conference, pp 597\u2013602"},{"key":"30_CR20","doi-asserted-by":"crossref","unstructured":"Urbano J, Schedl M (2012) Towards minimal test collections for evaluation of audio music similarity and retrieval. In: WWW international workshop on advances in music, information research, pp 917\u2013923","DOI":"10.1145\/2187980.2188223"},{"issue":"5","key":"30_CR21","doi-asserted-by":"crossref","first-page":"697","DOI":"10.1016\/S0306-4573(00)00010-8","volume":"36","author":"EM Voorhees","year":"2000","unstructured":"Voorhees EM (2000) Variations in relevance judgments and the measurement of retrieval effectiveness. Inf Process Manag 36(5):697\u2013716","journal-title":"Inf Process Manag"},{"key":"30_CR22","doi-asserted-by":"crossref","unstructured":"Voorhees EM (2002) The philosophy of information retrieval evaluation. In: Workshop of the cross-language evaluation, forum, pp 355\u2013370","DOI":"10.1007\/3-540-45691-0_34"},{"key":"30_CR23","unstructured":"Voorhees EM (2002) Whither music IR evaluation infrastructure: lessons to be learned from TREC. In: JCDL workshop on the creation of standardized test collections, tasks, and metrics for music information retrieval (MIR) and music digital library (MDL), evaluation, pp 7\u201313"},{"key":"30_CR24","unstructured":"Voorhees EM, Harman DK (2005) TREC: experiment and evaluation in information retrieval. MIT Press, Cambridge"},{"key":"30_CR25","doi-asserted-by":"crossref","unstructured":"Yee T (2010) The VGAM package for categorical data analysis. J Stat Softw 32(10):1\u201334","DOI":"10.18637\/jss.v032.i10"},{"issue":"3","key":"30_CR26","doi-asserted-by":"crossref","first-page":"481","DOI":"10.1111\/j.2517-6161.1996.tb02095.x","volume":"58","author":"T Yee","year":"1996","unstructured":"Yee T, Wild C (1996) Vector generalized additive models. J R Stat Soc 58(3):481\u2013493","journal-title":"J R Stat Soc"}],"container-title":["International Journal of Multimedia Information Retrieval"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13735-012-0030-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s13735-012-0030-4\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s13735-012-0030-4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,4,29]],"date-time":"2025-04-29T15:27:45Z","timestamp":1745940465000},"score":1,"resource":{"primary":{"URL":"http:\/\/link.springer.com\/10.1007\/s13735-012-0030-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,1,1]]},"references-count":26,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2013,3]]}},"alternative-id":["30"],"URL":"https:\/\/doi.org\/10.1007\/s13735-012-0030-4","relation":{},"ISSN":["2192-6611","2192-662X"],"issn-type":[{"value":"2192-6611","type":"print"},{"value":"2192-662X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,1,1]]}}}