{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T18:22:37Z","timestamp":1782325357264,"version":"3.54.5"},"reference-count":24,"publisher":"Oxford University Press (OUP)","issue":"9","license":[{"start":{"date-parts":[[2017,12,21]],"date-time":"2017-12-21T00:00:00Z","timestamp":1513814400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/academic.oup.com\/journals\/pages\/about_us\/legal\/notices"}],"funder":[{"DOI":"10.13039\/100000002","name":"National Institutes of Health","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000002","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100002923","name":"CONICET","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100002923","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2018,5,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n               <jats:sec>\n                  <jats:title>Motivation<\/jats:title>\n                  <jats:p>Computational methods for the prediction of peptide-MHC binding have become an integral and essential component for candidate selection in experimental T cell epitope discovery studies. The sheer amount of published prediction methods\u2014and often discordant reports on their performance\u2014poses a considerable quandary to the experimentalist who needs to choose the best tool for their research.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Results<\/jats:title>\n                  <jats:p>With the goal to provide an unbiased, transparent evaluation of the state-of-the-art in the field, we created an automated platform to benchmark peptide-MHC class II binding prediction tools. The platform evaluates the absolute and relative predictive performance of all participating tools on data newly entered into the Immune Epitope Database (IEDB) before they are made public, thereby providing a frequent, unbiased assessment of available prediction tools. The benchmark runs on a weekly basis, is fully automated, and displays up-to-date results on a publicly accessible website. The initial benchmark described here included six commonly used prediction servers, but other tools are encouraged to join with a simple sign-up procedure. Performance evaluation on 59 data sets composed of over 10 000 binding affinity measurements suggested that NetMHCIIpan is currently the most accurate tool, followed by NN-align and the IEDB consensus method.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Availability and implementation<\/jats:title>\n                  <jats:p>Weekly reports on the participating methods can be found online at: http:\/\/tools.iedb.org\/auto_bench\/mhcii\/weekly\/.<\/jats:p>\n               <\/jats:sec>\n               <jats:sec>\n                  <jats:title>Supplementary information<\/jats:title>\n                  <jats:p>Supplementary data are available at Bioinformatics online.<\/jats:p>\n               <\/jats:sec>","DOI":"10.1093\/bioinformatics\/btx820","type":"journal-article","created":{"date-parts":[[2017,12,20]],"date-time":"2017-12-20T22:03:02Z","timestamp":1513807382000},"page":"1522-1528","source":"Crossref","is-referenced-by-count":89,"title":["An automated benchmarking platform for MHC class II binding prediction methods"],"prefix":"10.1093","volume":"34","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8036-2647","authenticated-orcid":false,"given":"Massimo","family":"Andreatta","sequence":"first","affiliation":[{"name":"Instituto de Investigaciones Biotecnol\u00f3gicas, Universidad Nacional de San Mart\u00edn, San Mart\u00edn, Buenos Aires, Argentina"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Thomas","family":"Trolle","sequence":"additional","affiliation":[{"name":"Evaxion Biotech, Copenhagen N, Denmark"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhen","family":"Yan","sequence":"additional","affiliation":[{"name":"Bioinformatics Core Facility, La Jolla Institute for Allergy and Immunology, La Jolla, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jason A","family":"Greenbaum","sequence":"additional","affiliation":[{"name":"Bioinformatics Core Facility, La Jolla Institute for Allergy and Immunology, La Jolla, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bjoern","family":"Peters","sequence":"additional","affiliation":[{"name":"Division of Vaccine Discovery, La Jolla Institute for Allergy and Immunology, La Jolla, CA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7885-4311","authenticated-orcid":false,"given":"Morten","family":"Nielsen","sequence":"additional","affiliation":[{"name":"Instituto de Investigaciones Biotecnol\u00f3gicas, Universidad Nacional de San Mart\u00edn, San Mart\u00edn, Buenos Aires, Argentina"},{"name":"Department of Bio and Health Informatics, Technical University of Denmark, Lyngby, Denmark"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2017,12,21]]},"reference":[{"key":"2023012713020534100_btx820-B1","doi-asserted-by":"crossref","first-page":"641","DOI":"10.1007\/s00251-015-0873-y","article-title":"Accurate pan-specific prediction of peptide-MHC class II binding affinity with improved binding core identification","volume":"67","author":"Andreatta","year":"2015","journal-title":"Immunogenetics"},{"key":"2023012713020534100_btx820-B2","doi-asserted-by":"crossref","first-page":"443","DOI":"10.1146\/annurev-immunol-032712-095910","article-title":"Pathways of antigen processing","volume":"31","author":"Blum","year":"2013","journal-title":"Annu. Rev. Immunol"},{"key":"2023012713020534100_btx820-B3","doi-asserted-by":"crossref","first-page":"304","DOI":"10.1007\/s00251-005-0798-y","article-title":"Automated generation and evaluation of specific MHC binding predictive tools: ARB matrix applications","volume":"57","author":"Bui","year":"2005","journal-title":"Immunogenetics"},{"key":"2023012713020534100_btx820-B4","author":"Caron","year":"2015"},{"key":"2023012713020534100_btx820-B5","author":"Dhanda","year":"2016"},{"key":"2023012713020534100_btx820-B6","doi-asserted-by":"crossref","first-page":"2263","DOI":"10.1093\/bioinformatics\/btg312","article-title":"Towards the in silico identification of class II restricted T-cell epitopes: a partial least squares iterative self-consistent algorithm for affinity prediction","volume":"19","author":"Doytchinova","year":"2003","journal-title":"Bioinformatics"},{"key":"2023012713020534100_btx820-B7","doi-asserted-by":"crossref","first-page":"278","DOI":"10.3389\/fimmu.2017.00278","article-title":"The immune epitope database and analysis resource in epitope discovery and synthetic vaccine design","volume":"8","author":"Fleri","year":"2017","journal-title":"Front. Immunol"},{"key":"2023012713020534100_btx820-B8","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1186\/1745-7580-5-2","article-title":"Functional recombinant MHC class II molecules and high-throughput peptide-binding assays","volume":"5","author":"Justesen","year":"2009","journal-title":"Immunome Res"},{"key":"2023012713020534100_btx820-B9","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1186\/1471-2105-15-241","article-title":"Dataset size and composition impact the reliability of performance benchmarks for peptide-MHC binding predictions","volume":"15","author":"Kim","year":"2014","journal-title":"BMC Bioinformatics"},{"key":"2023012713020534100_btx820-B10","doi-asserted-by":"crossref","first-page":"S22.","DOI":"10.1186\/1471-2105-9-S12-S22","article-title":"Evaluation of MHC-II peptide binding prediction servers: applications for vaccine research","volume":"9","author":"Lin","year":"2008","journal-title":"BMC Bioinformatics"},{"key":"2023012713020534100_btx820-B11","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1016\/j.jim.2015.06.003","article-title":"Poor correlation between T-cell activation assays and HLA-DR binding prediction algorithms in an immunogenic fragment of Pseudomonas exotoxin A","volume":"425","author":"Mazor","year":"2015","journal-title":"J. Immunol. Methods"},{"key":"2023012713020534100_btx820-B12","doi-asserted-by":"crossref","first-page":"1412","DOI":"10.1074\/mcp.M115.055780","article-title":"Sampling from the proteome to the human leukocyte antigen-DR (HLA-DR) ligandome proceeds via high specificity","volume":"15","author":"Mommen","year":"2016","journal-title":"Mol. Cell. Proteomics"},{"key":"2023012713020534100_btx820-B13","doi-asserted-by":"crossref","first-page":"319","DOI":"10.1111\/j.1365-2567.2010.03268.x","article-title":"MHC class II epitope predictive algorithms","volume":"130","author":"Nielsen","year":"2010","journal-title":"Immunology"},{"key":"2023012713020534100_btx820-B14","doi-asserted-by":"crossref","first-page":"238","DOI":"10.1186\/1471-2105-8-238","article-title":"Prediction of MHC class II binding affinity using SMM-align, a novel stabilization matrix alignment method","volume":"8","author":"Nielsen","year":"2007","journal-title":"BMC Bioinformatics"},{"key":"2023012713020534100_btx820-B15","doi-asserted-by":"crossref","first-page":"e1000107.","DOI":"10.1371\/journal.pcbi.1000107","article-title":"Quantitative predictions of peptide binding to any HLA-DR molecule of known sequence: NetMHCIIpan","volume":"4","author":"Nielsen","year":"2008","journal-title":"PLoS Comput. Biol"},{"key":"2023012713020534100_btx820-B16","doi-asserted-by":"crossref","first-page":"296.","DOI":"10.1186\/1471-2105-10-296","article-title":"NN-align. An artificial neural network-based alignment algorithm for MHC class II peptide binding prediction","volume":"10","author":"Nielsen","year":"2009","journal-title":"BMC Bioinformatics"},{"key":"2023012713020534100_btx820-B17","doi-asserted-by":"crossref","first-page":"2.","DOI":"10.1186\/1745-7580-4-2","article-title":"Quantitative peptide binding motifs for 19 human and mouse MHC class I molecules derived using positional scanning combinatorial peptide libraries","volume":"4","author":"Sidney","year":"2008","journal-title":"Immunome Res"},{"key":"2023012713020534100_btx820-B18","doi-asserted-by":"crossref","first-page":"555","DOI":"10.1038\/9858","article-title":"Generation of tissue-specific and promiscuous HLA ligand databases using DNA microarrays and virtual HLA class II matrices","volume":"17","author":"Sturniolo","year":"1999","journal-title":"Nat. Biotechnol"},{"key":"2023012713020534100_btx820-B19","doi-asserted-by":"crossref","first-page":"2174","DOI":"10.1093\/bioinformatics\/btv123","article-title":"Automated benchmarking of peptide-MHC class I binding predictions","volume":"31","author":"Trolle","year":"2015","journal-title":"Bioinformatics"},{"key":"2023012713020534100_btx820-B20","doi-asserted-by":"crossref","first-page":"D405","DOI":"10.1093\/nar\/gku938","article-title":"The immune epitope database (IEDB) 3.0","volume":"43","author":"Vita","year":"2015","journal-title":"Nucleic Acids Res"},{"key":"2023012713020534100_btx820-B21","doi-asserted-by":"crossref","first-page":"463.","DOI":"10.1186\/1471-2105-7-463","article-title":"SVRMHC prediction server for MHC-binding peptides","volume":"7","author":"Wan","year":"2006","journal-title":"BMC Bioinformatics"},{"key":"2023012713020534100_btx820-B22","doi-asserted-by":"crossref","first-page":"e1000048.","DOI":"10.1371\/journal.pcbi.1000048","article-title":"A systematic assessment of MHC class II peptide binding predictions and evaluation of a consensus approach","volume":"4","author":"Wang","year":"2008","journal-title":"PLoS Comput. Biol"},{"key":"2023012713020534100_btx820-B23","doi-asserted-by":"crossref","first-page":"568.","DOI":"10.1186\/1471-2105-11-568","article-title":"Peptide binding predictions for HLA DR, DP and DQ molecules","volume":"11","author":"Wang","year":"2010","journal-title":"BMC Bioinformatics"},{"key":"2023012713020534100_btx820-B24","doi-asserted-by":"crossref","first-page":"e30483","DOI":"10.1371\/journal.pone.0030483","article-title":"TEPITOPEpan: extending TEPITOPE for peptide binding prediction covering over 700 HLA-DR molecules","volume":"7","author":"Zhang","year":"2012","journal-title":"PLoS One"}],"container-title":["Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/34\/9\/1522\/48914679\/bioinformatics_34_9_1522.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/34\/9\/1522\/48914679\/bioinformatics_34_9_1522.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,27]],"date-time":"2023-01-27T13:53:52Z","timestamp":1674827632000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article\/34\/9\/1522\/4769495"}},"subtitle":[],"editor":[{"given":"Bonnie","family":"Berger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2017,12,21]]},"references-count":24,"journal-issue":{"issue":"9","published-print":{"date-parts":[[2018,5,1]]}},"URL":"https:\/\/doi.org\/10.1093\/bioinformatics\/btx820","relation":{},"ISSN":["1367-4803","1367-4811"],"issn-type":[{"value":"1367-4803","type":"print"},{"value":"1367-4811","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2018,5,1]]},"published":{"date-parts":[[2017,12,21]]}}}