Abstract:
Distributed crawling has shown that it can overcome important limitations
of the centralized crawling paradigm. However, the distributed nature of
current distributed crawlers is currently not fully utilized. The optimal benefits
of this approach are usually limited to the sites hosting the crawler. In this work
we propose IPMicra, a distributed location aware web crawler that utilizes an IP
address hierarchy and allows crawling of links in a near optimal location aware
manner.
Keywords: Web Crawling, Distributed Web Crawling, Mobile Agents, IPMicra, UCYMicra
@inproceedings{papapetrou:adbis04,
author = {Odysseas Papapetrou and George Samaras},
title = {IPMicra: Toward a Distributed and Adaptable Location Aware Web Crawler.},
booktitle = {ADBIS (Local Proceedings)},
year = {2004},
abstract-url={http://www2.cs.ucy.ac.cy/~cspapap/abstracts/adbis04.html},
url={http://www2.cs.ucy.ac.cy/~cspapap/publications/adbis04.pdf}
}