{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,3,21]],"date-time":"2026-03-21T20:25:44Z","timestamp":1774124744491,"version":"3.50.1"},"publisher-location":"Cham","reference-count":45,"publisher":"Springer Nature Switzerland","isbn-type":[{"value":"9783031434174","type":"print"},{"value":"9783031434181","type":"electronic"}],"license":[{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2023,1,1]],"date-time":"2023-01-01T00:00:00Z","timestamp":1672531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.1007\/978-3-031-43418-1_10","type":"book-chapter","created":{"date-parts":[[2023,9,16]],"date-time":"2023-09-16T09:02:26Z","timestamp":1694854946000},"page":"157-173","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":20,"title":["Train Your Own GNN Teacher: Graph-Aware Distillation on\u00a0Textual Graphs"],"prefix":"10.1007","author":[{"given":"Costas","family":"Mavromatis","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Vassilis N.","family":"Ioannidis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Shen","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Da","family":"Zheng","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Soji","family":"Adeshina","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Jun","family":"Ma","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Han","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Christos","family":"Faloutsos","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"George","family":"Karypis","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"297","published-online":{"date-parts":[[2023,9,17]]},"reference":[{"key":"10_CR1","doi-asserted-by":"crossref","unstructured":"Ando, R., Zhang, T.: Learning on graph with laplacian regularization. In: NIPS (2006)","DOI":"10.7551\/mitpress\/7503.003.0009"},{"key":"10_CR2","doi-asserted-by":"crossref","unstructured":"Beltagy, I., Lo, K., Cohan, A.: SciBERT: a pretrained language model for scientific text. In: EMNLP-IJCNLP (2019)","DOI":"10.18653\/v1\/D19-1371"},{"key":"10_CR3","unstructured":"Chien, E., et al.: Node feature extraction by self-supervised multi-scale neighborhood prediction. In: ICLR (2022)"},{"key":"10_CR4","doi-asserted-by":"crossref","unstructured":"Deng, X., Zhang, Z.: Graph-free knowledge distillation for graph neural networks. arXiv (2021)","DOI":"10.24963\/ijcai.2021\/320"},{"key":"10_CR5","unstructured":"Devlin, J., Chang, M.W., Lee, K., Toutanova, K.: BERT: pre-training of deep bidirectional transformers for language understanding. In: ACL (2019)"},{"key":"10_CR6","doi-asserted-by":"crossref","unstructured":"Dinh, T.A., Boef, J.D., Cornelisse, J., Groth, P.: E2EG: end-to-end node classification using graph topology and text-based node attributes. arXiv (2022)","DOI":"10.1109\/ICDMW60847.2023.00142"},{"key":"10_CR7","doi-asserted-by":"crossref","unstructured":"Dong, W., Wu, J., Luo, Y., Ge, Z., Wang, P.: Node representation learning in graph via node-to-neighbourhood mutual information maximization. In: IEEE\/CVF CVPR (2022)","DOI":"10.1109\/CVPR52688.2022.01612"},{"key":"10_CR8","unstructured":"Gilmer, J., Schoenholz, S.S., Riley, P.F., Vinyals, O., Dahl, G.E.: Neural message passing for quantum chemistry. In: ICML (2017)"},{"key":"10_CR9","doi-asserted-by":"crossref","unstructured":"Gou, J., Yu, B., Maybank, S.J., Tao, D.: Knowledge distillation: a survey. In: IJCV (2021)","DOI":"10.1007\/s11263-021-01453-z"},{"key":"10_CR10","unstructured":"Hamilton, W., Ying, Z., Leskovec, J.: Inductive representation learning on large graphs. In: NeurIPS (2017)"},{"key":"10_CR11","unstructured":"Hinton, G., Vinyals, O., Dean, J., et al.: Distilling the knowledge in a neural network. arXiv (2015)"},{"key":"10_CR12","unstructured":"Hu, W., et al.: Open graph benchmark: datasets for machine learning on graphs. In: NeurIPS (2020)"},{"key":"10_CR13","unstructured":"Hu, Y., You, H., Wang, Z., Wang, Z., Zhou, E., Gao, Y.: Graph-MLP: node classification without message passing in graph (2021)"},{"key":"10_CR14","doi-asserted-by":"crossref","unstructured":"Huang, L., Ma, D., Li, S., Zhang, X., Wang, H.: Text level graph neural network for text classification. In: EMNLP (2019)","DOI":"10.18653\/v1\/D19-1345"},{"key":"10_CR15","unstructured":"Ioannidis, V.N., et al.: Efficient and effective training of language and graph neural network models. arXiv (2022)"},{"key":"10_CR16","doi-asserted-by":"crossref","unstructured":"Jia, J., Benson, A.R.: Residual correlation in graph neural network regression. In: KDD (2020)","DOI":"10.1145\/3394486.3403101"},{"key":"10_CR17","unstructured":"Kipf, T.N., Welling, M.: Semi-supervised classification with graph convolutional networks. In: ICLR (2017)"},{"key":"10_CR18","doi-asserted-by":"crossref","unstructured":"Li, C., et al.: AdsGNN: behavior-graph augmented relevance modeling in sponsored search. In: ACM SIGIR (2021)","DOI":"10.1145\/3404835.3462926"},{"key":"10_CR19","unstructured":"Liu, Y., et al.: Roberta: a robustly optimized bert pretraining approach. arXiv (2019)"},{"key":"10_CR20","doi-asserted-by":"crossref","unstructured":"Mavromatis, C., Karypis, G.: ReaRev: adaptive reasoning for question answering over knowledge graphs. arXiv (2022)","DOI":"10.18653\/v1\/2022.findings-emnlp.181"},{"key":"10_CR21","unstructured":"Mikolov, T., Sutskever, I., Chen, K., Corrado, G.S., Dean, J.: Distributed representations of words and phrases and their compositionality. In: NIPS (2013)"},{"issue":"1","key":"10_CR22","first-page":"5485","volume":"21","author":"C Raffel","year":"2020","unstructured":"Raffel, C., et al.: Exploring the limits of transfer learning with a unified text-to-text transformer. JMLR 21(1), 5485\u20135551 (2020)","journal-title":"JMLR"},{"key":"10_CR23","unstructured":"Sanh, V., Debut, L., Chaumond, J., Wolf, T.: Distilbert, a distilled version of bert: smaller, faster, cheaper and lighter. arXiv (2019)"},{"key":"10_CR24","doi-asserted-by":"crossref","unstructured":"Schlichtkrull, M., Kipf, T.N., Bloem, P., Berg, R.V.D., Titov, I., Welling, M.: Modeling relational data with graph convolutional networks. In: ESWC (2018)","DOI":"10.1007\/978-3-319-93417-4_38"},{"key":"10_CR25","unstructured":"Vaswani, A., et al.: Attention is all you need. In: NIPS (2017)"},{"key":"10_CR26","unstructured":"Veli\u010dkovi\u0107, P., Cucurull, G., Casanova, A., Romero, A., Lio, P., Bengio, Y.: Graph attention networks. In: ICLR (2018)"},{"key":"10_CR27","doi-asserted-by":"crossref","unstructured":"Xu, Y., Zhang, Y., Guo, W., Guo, H., Tang, R., Coates, M.: Graphsail: graph structure aware incremental learning for recommender systems. In: CIKM (2020)","DOI":"10.1145\/3340531.3412754"},{"key":"10_CR28","doi-asserted-by":"crossref","unstructured":"Yan, B., Wang, C., Guo, G., Lou, Y.: Tinygnn: learning efficient graph neural networks. In: KDD (2020)","DOI":"10.1145\/3394486.3403236"},{"key":"10_CR29","doi-asserted-by":"crossref","unstructured":"Yang, C., Liu, J., Shi, C.: Extract the knowledge of graph neural networks and go beyond it: an effective knowledge distillation framework. In: WWW (2021)","DOI":"10.1145\/3442381.3450068"},{"key":"10_CR30","doi-asserted-by":"crossref","unstructured":"Yang, H., Ma, K., Cheng, J.: Rethinking graph regularization for graph neural networks. In: AAAI (2021)","DOI":"10.1609\/aaai.v35i5.16586"},{"key":"10_CR31","unstructured":"Yang, J., et al.: Graphformers: GNN-nested transformers for representation learning on textual graph. In: NeurIPS (2021)"},{"key":"10_CR32","doi-asserted-by":"crossref","unstructured":"Yang, Y., Qiu, J., Song, M., Tao, D., Wang, X.: Distilling knowledge from graph convolutional networks. In: IEEE\/CVF CVPR (2020)","DOI":"10.1109\/CVPR42600.2020.00710"},{"key":"10_CR33","doi-asserted-by":"crossref","unstructured":"Yao, L., Mao, C., Luo, Y.: Graph convolutional networks for text classification. In: AAAI (2019)","DOI":"10.1609\/aaai.v33i01.33017370"},{"key":"10_CR34","unstructured":"Yasunaga, M., et al.: Deep bidirectional language-knowledge graph pretraining. In: NeurIPS (2022)"},{"key":"10_CR35","doi-asserted-by":"crossref","unstructured":"Yasunaga, M., Leskovec, J., Liang, P.: Linkbert: pretraining language models with document links. In: ACL (2022)","DOI":"10.18653\/v1\/2022.acl-long.551"},{"key":"10_CR36","doi-asserted-by":"crossref","unstructured":"Yuan, L., Tay, F.E., Li, G., Wang, T., Feng, J.: Revisiting knowledge distillation via label smoothing regularization. In: IEEE\/CVF CVPR (2020)","DOI":"10.1109\/CVPR42600.2020.00396"},{"key":"10_CR37","unstructured":"Zhang, J., Zhang, H., Xia, C., Sun, L.: Graph-bert: only attention is needed for learning graph representations. arXiv (2020)"},{"key":"10_CR38","doi-asserted-by":"crossref","unstructured":"Zhang, L., Song, J., Gao, A., Chen, J., Bao, C., Ma, K.: Be your own teacher: improve the performance of convolutional neural networks via self distillation. In: IEEE\/CVF ICCV (2019)","DOI":"10.1109\/ICCV.2019.00381"},{"key":"10_CR39","unstructured":"Zhang, S., Liu, Y., Sun, Y., Shah, N.: Graph-less neural networks: teaching old MLPs new tricks via distillation. In: ICLR (2022)"},{"key":"10_CR40","unstructured":"Zhang, W., Deng, L., Zhang, L., Wu, D.: A survey on negative transfer. arXiv (2020)"},{"key":"10_CR41","unstructured":"Zhang, X., et al.: GreaseLM: graph REASoning enhanced language models. In: ICLR (2022)"},{"key":"10_CR42","unstructured":"Zhao, J., et al.: Learning on large-scale text-attributed graphs via variational inference. arXiv (2022)"},{"key":"10_CR43","unstructured":"Zheng, W., Huang, E.W., Rao, N., Katariya, S., Wang, Z., Subbian, K.: Cold brew: distilling graph node representations with incomplete or missing neighborhoods. In: ICLR (2022)"},{"key":"10_CR44","unstructured":"Zhou, D., Bousquet, O., Lal, T., Weston, J., Sch\u00f6lkopf, B.: Learning with local and global consistency. In: NIPS (2003)"},{"key":"10_CR45","doi-asserted-by":"crossref","unstructured":"Zhu, J., et al.: Textgnn: improving text encoder via graph neural network in sponsored search. In: WWW (2021)","DOI":"10.1145\/3442381.3449842"}],"container-title":["Lecture Notes in Computer Science","Machine Learning and Knowledge Discovery in Databases: Research Track"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/978-3-031-43418-1_10","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,6,26]],"date-time":"2024-06-26T13:06:01Z","timestamp":1719407161000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/978-3-031-43418-1_10"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"ISBN":["9783031434174","9783031434181"],"references-count":45,"URL":"https:\/\/doi.org\/10.1007\/978-3-031-43418-1_10","relation":{},"ISSN":["0302-9743","1611-3349"],"issn-type":[{"value":"0302-9743","type":"print"},{"value":"1611-3349","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023]]},"assertion":[{"value":"17 September 2023","order":1,"name":"first_online","label":"First Online","group":{"name":"ChapterHistory","label":"Chapter History"}},{"value":"<scp>GraD<\/scp> relies on informative input node features to learn effective shared LMs (or MLPs) that can generalize to unseen nodes, which is the case in textual graphs. Thus, one limitation is that it is not certain how <scp>GraD<\/scp> generalizes to other graphs, e.g., to featureless graphs. Moreover as a knowledge distillation approach, <scp>GraD<\/scp> trades accuracy for computation efficiency and it cannot adapt to dynamic graphs with edge changes the same way as GNN could. To overcome biases encoded in the training graph, e.g., standard stereotypes in recommender graphs, <scp>GraD<\/scp> needs to be retrained over the new unbiased graph.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Limitations and Ethical Statement"}},{"value":"ECML PKDD","order":1,"name":"conference_acronym","label":"Conference Acronym","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Joint European Conference on Machine Learning and Knowledge Discovery in Databases","order":2,"name":"conference_name","label":"Conference Name","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Turin","order":3,"name":"conference_city","label":"Conference City","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Italy","order":4,"name":"conference_country","label":"Conference Country","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"2023","order":5,"name":"conference_year","label":"Conference Year","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"18 September 2023","order":7,"name":"conference_start_date","label":"Conference Start Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"22 September 2023","order":8,"name":"conference_end_date","label":"Conference End Date","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"23","order":9,"name":"conference_number","label":"Conference Number","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"ecml2023","order":10,"name":"conference_id","label":"Conference ID","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"https:\/\/2023.ecmlpkdd.org\/","order":11,"name":"conference_url","label":"Conference URL","group":{"name":"ConferenceInfo","label":"Conference Information"}},{"value":"Double-blind","order":1,"name":"type","label":"Type","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"CMT","order":2,"name":"conference_management_system","label":"Conference Management System","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"829","order":3,"name":"number_of_submissions_sent_for_review","label":"Number of Submissions Sent for Review","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"196","order":4,"name":"number_of_full_papers_accepted","label":"Number of Full Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"0","order":5,"name":"number_of_short_papers_accepted","label":"Number of Short Papers Accepted","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"24% - The value is computed by the equation \"Number of Full Papers Accepted \/ Number of Submissions Sent for Review * 100\" and then rounded to a whole number.","order":6,"name":"acceptance_rate_of_full_papers","label":"Acceptance Rate of Full Papers","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"3.63","order":7,"name":"average_number_of_reviews_per_paper","label":"Average Number of Reviews per Paper","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"4.5","order":8,"name":"average_number_of_papers_per_reviewer","label":"Average Number of Papers per Reviewer","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Yes","order":9,"name":"external_reviewers_involved","label":"External Reviewers Involved","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}},{"value":"Applied Data Science Track: 239 submissions, 58 accepted papers; Demo Track: 31 submissions, 16 accepted papers.","order":10,"name":"additional_info_on_review_process","label":"Additional Info on Review Process","group":{"name":"ConfEventPeerReviewInformation","label":"Peer Review Information (provided by the conference organizers)"}}]}}