Abstract:
Due to the tremendous increase rate and the high change frequency
of Web documents, maintaining an up-to-date index for searching purposes
(search engines) is becoming a challenge. The traditional crawling methods are
no longer able to catch up with the constantly updating and growing Web. Realizing
the problem, in this paper we suggest an alternative distributed crawling
method with the use of mobile agents. Our goal is a scalable crawling scheme
that minimizes network utilization, keeps up with document changes, employs
time realization, and is easily upgradeable.
Keywords: distributed web crawling, mobile agents
@inproceedings{papapetrou:adbis03,
author = {Odysseas Papapetrou and Stavros Papastavrou and George Samaras},
title = {UCYMICRA: Distributed Indexing of the Web Using Migrating Crawlers.},
booktitle = {ADBIS},
year = {2003},
pages = {133-147},
publisher={Springer},
series= {Lecture Notes in Computer Science},
abstract-url={http://www2.cs.ucy.ac.cy/~cspapap/abstracts/adbis03.html},
url={http://springerlink.metapress.com/openurl.asp?genre=article{\&}issn=0302-9743{\&}volume=2798{\&}spage=133},
publisher-url={http://www.springerlink.com/}
}