@Article{jbd.2019.07235,
AUTHOR = {Guang  Sun, Huanxin  Xiang, Shuanghu  Li},
TITLE = {On Multi-Thread Crawler Optimization for Scalable Text Searching},
JOURNAL = {Journal on Big Data},
VOLUME = {1},
YEAR = {2019},
NUMBER = {2},
PAGES = {89--106},
URL = {http://www.techscience.com/jbd/v1n2/29019},
ISSN = {2579-0056},
ABSTRACT = {Web crawlers are an important part of modern search engines. With the development of the times, data has exploded and humans have entered a “big data era”. For example, Wikipedia carries the knowledge from all over the world, records the real-time news that occurs every day, and provides users with a good database of data, but because of the large amount of data, it puts a lot of pressure on users to search. At present, single-threaded crawling data can no longer meet the requirements of text crawling. In order to improve the performance and program versatility of single-threaded crawlers, a high-speed multi-threaded web crawler is designed to crawl the network hyper-scale text database. Multi-threaded crawling uses multiple threads to process web pages in parallel, combining breadth-first and depth-first algorithms to control web crawling. The practice project is based on the Python language to achieve multi-threaded optimization network hyper-large-scale text database-Wikipedia book crawling method, the project is inspired by the article on the Wikipedia article in the Big Data Digest public number.},
DOI = {10.32604/jbd.2019.07235}
}