{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,6,25]],"date-time":"2026-06-25T16:19:12Z","timestamp":1782404352768,"version":"3.54.5"},"reference-count":50,"publisher":"Springer Science and Business Media LLC","issue":"3","license":[{"start":{"date-parts":[[2021,4,16]],"date-time":"2021-04-16T00:00:00Z","timestamp":1618531200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2021,4,16]],"date-time":"2021-04-16T00:00:00Z","timestamp":1618531200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["SN COMPUT. SCI."],"published-print":{"date-parts":[[2021,5]]},"DOI":"10.1007\/s42979-021-00574-z","type":"journal-article","created":{"date-parts":[[2021,4,16]],"date-time":"2021-04-16T06:11:18Z","timestamp":1618553478000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":5,"title":["Crawler by Contextual Inference"],"prefix":"10.1007","volume":"2","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-8705-759X","authenticated-orcid":false,"given":"Prakash","family":"Hegade","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nikhil","family":"Lingadhal","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sourabh","family":"Jain","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Usman","family":"Khan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"K. L.","family":"Vijeth","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,4,16]]},"reference":[{"issue":"18","key":"574_CR1","doi-asserted-by":"publisher","first-page":"3859","DOI":"10.1016\/j.comnet.2012.10.004","volume":"56","author":"W Hall","year":"2012","unstructured":"Hall W, Tiropanis T. Web evolution and Web science. Comput Netw. 2012;56(18):3859\u201365.","journal-title":"Comput Netw"},{"issue":"2","key":"574_CR2","doi-asserted-by":"publisher","first-page":"213","DOI":"10.1002\/spe.577","volume":"34","author":"D Fetterly","year":"2004","unstructured":"Fetterly D, Manasse M, Najork M, Wiener JL. A large-scale study of the evolution of Web pages. Softw Pract Exp. 2004;34(2):213\u201337.","journal-title":"Softw Pract Exp"},{"key":"574_CR3","volume-title":"The tangled web: a guide to securing modern web applications","author":"M Zalewski","year":"2012","unstructured":"Zalewski M. The tangled web: a guide to securing modern web applications. San Francisco: No Starch Press; 2012."},{"key":"574_CR4","doi-asserted-by":"publisher","first-page":"387","DOI":"10.4135\/9781526486387.n25","volume-title":"The SAGE handbook of research methods in political science and international relations","author":"D Nyhuis","year":"2020","unstructured":"Nyhuis D. Web data collection: potentials and challenges. In: The SAGE handbook of research methods in political science and international relations. Thousand Oaks: SAGE publisher; 2020. p. 387\u2013403."},{"key":"574_CR5","unstructured":"Burrows M, inventor; Digital Equipment Corp, assignee. Method for parsing, indexing and searching world-wide-web pages. United States patent US 5,864,863; 1999."},{"issue":"4","key":"574_CR6","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1145\/3365574","volume":"13","author":"E Manica","year":"2019","unstructured":"Manica E, Dorneles CF, Galante R. Combining URL and HTML features for entity discovery in the web. ACM Trans Web (TWEB). 2019;13(4):1\u201327.","journal-title":"ACM Trans Web (TWEB)"},{"key":"574_CR7","doi-asserted-by":"publisher","first-page":"663","DOI":"10.1016\/j.asoc.2013.09.007","volume":"14","author":"Y Du","year":"2014","unstructured":"Du Y, Hai Y, Xie C, Wang X. An approach for selecting seed URLs of focused crawler based on user-interest ontology. Appl Soft Comput. 2014;14:663\u201376.","journal-title":"Appl Soft Comput"},{"key":"574_CR8","unstructured":"Zhu H, Dean J, Ghemawat S, Yang BP, Acharya A, inventors. Google LLC, assignee, Anchor tag indexing in a web crawler system, United States patent US 7,308,643; 2007."},{"key":"574_CR9","doi-asserted-by":"crossref","unstructured":"Wolf JL, Squillante MS, Yu PS, Sethuraman J, Ozsen L. Optimal crawling strategies for web search engines. In: Proceedings of the 11th international conference on World Wide Web.; 2002. pp. 136\u201347.","DOI":"10.1145\/511446.511465"},{"key":"574_CR10","doi-asserted-by":"crossref","unstructured":"Sun Y, Zhuang Z, Giles CL. A large-scale study of robots.txt. In: Proceedings of the 16th international conference on World Wide Web.; 2007. pp. 1123\u201324.","DOI":"10.1145\/1242572.1242726"},{"key":"574_CR11","unstructured":"Gray M. The World-Wide Web Wanderer Internet Statistics MIT. 2004 http:\/\/www.mit.edu\/people\/mkgray\/net\/. Accessed  5 Jan 2021"},{"key":"574_CR12","doi-asserted-by":"crossref","unstructured":"Sharma S, Gupta P. The anatomy of web crawlers. In: Proceedings of the international conference on computing, communication & automation Noida India; 2015. pp. 849\u201353.","DOI":"10.1109\/CCAA.2015.7148493"},{"key":"574_CR13","doi-asserted-by":"crossref","unstructured":"Gupta P, Johari K. Implementation of Web Crawler. In: Proceedings of the second international conference on emerging trends in engineering & technology Nagpur India; 2009. pp 838\u201343.","DOI":"10.1109\/ICETET.2009.124"},{"key":"574_CR14","doi-asserted-by":"crossref","unstructured":"Najork M, Wiener JL. Breadth-first crawling yields high-quality pages. In: Proceedings of the 10th international conference on World Wide Web New York USA. Association for Computing Machinery. 2001.","DOI":"10.1145\/371920.371965"},{"key":"574_CR15","doi-asserted-by":"publisher","first-page":"153","DOI":"10.1007\/978-3-662-10874-1_7","volume-title":"Web dynamics","author":"G Pant","year":"2004","unstructured":"Pant G, Srinivasan P, Menczer F. Crawling the web. In: Web dynamics. Berlin: Springer; 2004. p. 153\u201377."},{"key":"574_CR16","first-page":"146","volume-title":"Encyclopedia of database systems","author":"M Najork","year":"2009","unstructured":"Najork M. Web Crawler architecture. In: Encyclopedia of database systems. Berlin: Springer; 2009. p. 146\u201354."},{"key":"574_CR17","doi-asserted-by":"crossref","unstructured":"Ma W, Chen X, W. Shang. Advanced Deep Web Crawler based on dom. In: Proceedings of the fifth international joint conference on computational sciences and optimization; 2012, pp. 605\u20139.","DOI":"10.1109\/CSO.2012.138"},{"key":"574_CR18","doi-asserted-by":"crossref","unstructured":"Menczer F, Pant G, Srinivasan P, Ruiz ME. Evaluating topic-driven web crawlers. In: Proceedings of the 24th annual international ACM SIGIR conference on research and development in information retrieval; 2001, pp. 241\u20139.","DOI":"10.1145\/383952.383995"},{"issue":"4","key":"574_CR19","doi-asserted-by":"publisher","first-page":"378","DOI":"10.1145\/1031114.1031117","volume":"4","author":"F Menczer","year":"2004","unstructured":"Menczer F, Pant G, Srinivasan P. Topical web crawlers: evaluating adaptive algorithms. ACM Trans Internet Technol. 2004;4(4):378\u2013419.","journal-title":"ACM Trans Internet Technol"},{"key":"574_CR20","doi-asserted-by":"publisher","first-page":"119","DOI":"10.1016\/S0169-7552(98)00064-6","volume":"30","author":"RC Miller","year":"1998","unstructured":"Miller RC, Bharat K. SPHINX: a framework for creating personal, site-specific web crawlers. Comput Netw ISDN Syst. 1998;30:119\u201330.","journal-title":"Comput Netw ISDN Syst."},{"issue":"4","key":"574_CR21","doi-asserted-by":"publisher","first-page":"390","DOI":"10.1145\/958942.958945","volume":"28","author":"J Cho","year":"2003","unstructured":"Cho J, Garcia-Molina H. Effective page refresh policies for web crawlers. ACM Trans Database Syst. 2003;28(4):390\u2013426.","journal-title":"ACM Trans Database Syst"},{"issue":"1","key":"574_CR22","doi-asserted-by":"publisher","first-page":"1","DOI":"10.9790\/0661-16160105","volume":"6","author":"TV Udapure","year":"2014","unstructured":"Udapure TV, Kale RD, Dharmik RC. Study of web crawler and its different types. IOSR J Comput Eng. 2014;6(1):1\u20135.","journal-title":"IOSR J Comput Eng"},{"issue":"10","key":"574_CR23","doi-asserted-by":"publisher","first-page":"1001","DOI":"10.1016\/j.datak.2009.04.002","volume":"68","author":"S Batsakis","year":"2009","unstructured":"Batsakis S, Petrakis EG, Milios E. Improving the performance of focused web crawlers. Data Knowl Eng. 2009;68(10):1001\u201313.","journal-title":"Data Knowl Eng"},{"key":"574_CR24","doi-asserted-by":"crossref","unstructured":"Gupta A, Anand P. Focused web crawlers and its approaches. In: Proceedings of the international conference on futuristic trends on computational analysis and knowledge management (ABLAZE) Noida; 2015. pp. 619\u201322.","DOI":"10.1109\/ABLAZE.2015.7154936"},{"key":"574_CR25","first-page":"55","volume":"5558","author":"B Novak","year":"2004","unstructured":"Novak B. A survey of focused web crawling algorithms. Data Min Wareh. 2004;5558:55\u20138.","journal-title":"Data Min Wareh."},{"key":"574_CR26","doi-asserted-by":"crossref","unstructured":"Safran MS, Althagafi A, Che D. Improving relevance prediction for focused web crawlers. In: Proceedings of the IEEE\/ACIS 11th international conference on computer and information science. Shanghai; 2012, pp. 161\u201366.","DOI":"10.1109\/ICIS.2012.61"},{"key":"574_CR27","doi-asserted-by":"crossref","unstructured":"Agre GH, Mahajan NV. Keyword focused web crawler. In: Proceedings of the 2nd international conference on electronics and communication systems (ICECS) Coimbatore; 2015. pp. 1089\u201392.","DOI":"10.1109\/ECS.2015.7124749"},{"key":"574_CR28","unstructured":"Johnson J, Tsioutsiouliklis K, Giles CL. Evolving strategies for focused web crawling. In: Proceedings of the 20th international conference on machine learning (ICML-03); 2003."},{"key":"574_CR29","unstructured":"Hirsch MC. Method and apparatus for semantic serializing. U.S. Patent 11,778,513; 2009."},{"issue":"1","key":"574_CR30","doi-asserted-by":"publisher","first-page":"524","DOI":"10.1016\/j.eswa.2007.07.054","volume":"35","author":"A Batzios","year":"2008","unstructured":"Batzios A, Dimou C, Symeonidis AL, Mitkas PA. BioCrawler: an intelligent crawler for the semantic web. Expert Syst Appl. 2008;35(1):524\u201330.","journal-title":"Expert Syst Appl"},{"key":"574_CR31","unstructured":"Hu K, Wong WS. A probabilistic model for intelligent Web crawlers. In: Proceedings of the 27th annual international computer software and applications conference (COMPAC 2003) Dallas TX USA; 2003. pp. 278\u201382."},{"key":"574_CR32","doi-asserted-by":"crossref","unstructured":"Cai R, Yang JM, Lai W, Wang Y, Zhang L. iRobot: an intelligent crawler for web forums. In: Proceedings of the 17th international conference on World Wide Web WWW 2008 Beijing China; 2008, pp. 447\u201356.","DOI":"10.1145\/1367497.1367558"},{"issue":"2","key":"574_CR33","doi-asserted-by":"publisher","first-page":"490","DOI":"10.1016\/j.asoc.2009.08.017","volume":"10","author":"H Zhang","year":"2010","unstructured":"Zhang H, Lu J. An online semi-supervised clustering approach to topical web crawlers. Appl Soft Comput. 2010;10(2):490\u20135.","journal-title":"Appl Soft Comput"},{"key":"574_CR34","doi-asserted-by":"crossref","unstructured":"Zhu W, Gao H, He Z, Qin J, Han B. A hybrid approach for recognizing web crawlers. In: Proceedings of the international conference on wireless algorithms, systems, and applications (WASA 2019). Honolulu. HI. USA. Springer; 2019. pp. 507\u201319.","DOI":"10.1007\/978-3-030-23597-0_41"},{"key":"574_CR35","doi-asserted-by":"publisher","unstructured":"Kumar M, Bhatia R, Rattan D. A survey of Web crawlers for information retrieval. Data Min Knowl Discov 2017; 7(6):p.e1218. https:\/\/doi.org\/10.1002\/widm.1218","DOI":"10.1002\/widm.1218"},{"issue":"2","key":"574_CR36","first-page":"231","volume":"37","author":"J Yu","year":"2015","unstructured":"Yu J, Liu Q. Survey on topic-focused crawlers. Comput Eng Sci. 2015;37(2):231\u20137.","journal-title":"Comput Eng Sci"},{"issue":"2","key":"574_CR37","first-page":"129","volume":"16","author":"YB Yu","year":"2018","unstructured":"Yu YB, Huang SL, Tashi N, Zhang H, Lei F, Wu LY. A survey about algorithms utilized by focused web crawler. J Electron Sci Technol. 2018;16(2):129\u201338.","journal-title":"J Electron Sci Technol"},{"key":"574_CR38","doi-asserted-by":"crossref","unstructured":"R\u00f6der M, de Souza G, Kuchelev D, Desouki AA, Ngomo AC. ORCA-a benchmark for data web crawlers. In 2021 IEEE 15th International Conference on Semantic Computing (ICSC), pp. 272\u2013279.","DOI":"10.1109\/ICSC50631.2021.00054"},{"key":"574_CR39","doi-asserted-by":"crossref","unstructured":"Sharma AK, Shrivastava V, Singh H. Experimental performance analysis of web crawlers using single and Multi-Threaded web crawling and indexing algorithm for the application of smart web contents. Materials Today: Proceedings. 2020.","DOI":"10.1016\/j.matpr.2020.06.596"},{"key":"574_CR40","doi-asserted-by":"publisher","first-page":"371","DOI":"10.1007\/978-1-4842-6576-5_8","volume-title":"Getting structured data from the internet","author":"JM Patel","year":"2020","unstructured":"Patel JM. Advanced web crawlers. In: Getting structured data from the internet. Berkeley: Apress; 2020. p. 371\u201393."},{"key":"574_CR41","doi-asserted-by":"crossref","unstructured":"Yang D, Thiengburanathum P. A comparison of open source web crawlers for E-commerce websites. In: Proceedings of the joint international conference on digital arts, media and technology with ECTI northern section conference on electrical, electronics, computer and telecommunications engineering (ECTI DAMT & NCON). IEEE. 2020. pp. 200\u20135","DOI":"10.1109\/ECTIDAMTNCON48261.2020.9090772"},{"issue":"14","key":"574_CR42","first-page":"42","volume":"64","author":"KF Bharti","year":"2013","unstructured":"Bharti KF, Premchand P, Govardhan A. Web crawlers for searching hidden pages: a survey. Int J Comput Appl. 2013;64(14):42\u20138.","journal-title":"Int J Comput Appl"},{"key":"574_CR43","doi-asserted-by":"publisher","first-page":"781","DOI":"10.1007\/978-981-13-8618-3_80","volume-title":"Intelligent communication, control and devices","author":"P Chaitra","year":"2020","unstructured":"Chaitra P, Deepthi V, Vidyashree K, Rajini S. A study on different types of web crawlers. In: Intelligent communication, control and devices. Singapore: Springer; 2020. p. 781\u20139."},{"key":"574_CR44","doi-asserted-by":"crossref","unstructured":"Panum T, Hansen R, Pedersen J. Kraaler: A user-perspective web crawler. In: 2019 Network Traffic Measurement and Analysis Conference (TMA). IEEE; 2019.","DOI":"10.23919\/TMA.2019.8784660"},{"issue":"3","key":"574_CR45","doi-asserted-by":"publisher","first-page":"380","DOI":"10.1093\/sysbio\/45.3.380","volume":"45","author":"R Real","year":"1996","unstructured":"Real R, Vargas JM. The Probabilistic Basis of Jaccard\u2019s index of similarity. Syst Biol. 1996;45(3):380\u20135.","journal-title":"Syst Biol"},{"key":"574_CR46","doi-asserted-by":"publisher","first-page":"36","DOI":"10.1016\/j.patrec.2018.12.007","volume":"120","author":"S Kosub","year":"2019","unstructured":"Kosub S, Note A. on the triangle inequality for the Jaccard distance. Pattern Recogn Lett. 2019;120:36\u20138.","journal-title":"Pattern Recogn Lett"},{"key":"574_CR47","unstructured":"Rahutomo F, Kitasuka T, Aritsugi M. Semantic cosine similarity. In: Proceedings of the 7th international student conference on advanced science and technology (ICAST 2012). Seoul, South Korea; 2012"},{"key":"574_CR48","doi-asserted-by":"crossref","unstructured":"Banerjee S, Pedersen T. An adapted Lesk algorithm for word sense disambiguation using WordNet. In: Proceedings of the international conference on intelligent text processing and computational linguistics. Springer, Berlin; 2002. pp. 136\u201345","DOI":"10.1007\/3-540-45715-1_11"},{"key":"574_CR49","doi-asserted-by":"publisher","unstructured":"Loper E, Bird S. NLTK: the natural language toolkit; 2002. https:\/\/doi.org\/10.3115\/1118108.1118117","DOI":"10.3115\/1118108.1118117"},{"key":"574_CR50","doi-asserted-by":"crossref","unstructured":"Hegade P, Shilpa R, Aigal P, Pai S, Shejekar P. Crawler by inference. In: Proceedings of the Indo-Taiwan 2nd international conference on computing, analytics and networks (Indo-Taiwan ICAN). Rajpura, Punjab, India; 2020. pp. 108\u201312.","DOI":"10.1109\/Indo-TaiwanICAN48429.2020.9181364"}],"container-title":["SN Computer Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-021-00574-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s42979-021-00574-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s42979-021-00574-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,5,13]],"date-time":"2021-05-13T17:46:43Z","timestamp":1620928003000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s42979-021-00574-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,4,16]]},"references-count":50,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2021,5]]}},"alternative-id":["574"],"URL":"https:\/\/doi.org\/10.1007\/s42979-021-00574-z","relation":{},"ISSN":["2662-995X","2661-8907"],"issn-type":[{"value":"2662-995X","type":"print"},{"value":"2661-8907","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,4,16]]},"assertion":[{"value":"27 November 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"9 March 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"16 April 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"On behalf of all authors, the corresponding author states that there is no conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"216"}}