Skip to content
Snippets Groups Projects
Commit 1a4a3149 authored by jsclose's avatar jsclose
Browse files

trying with 3 spiders

parent 0efd4c02
No related branches found
No related tags found
No related merge requests found
......@@ -29,6 +29,8 @@ void SocketReader::fillBuffer()
// Send a GET message for the desired page.
cout << "Socket Reader is pulling from the web" << endl;
string getMessage = "GET ";
getMessage += url.CompleteUrl;
getMessage += " HTTP/1.1\r\nHost: ";
......@@ -43,8 +45,10 @@ void SocketReader::fillBuffer()
char buffer[ 10240 ];
int bytes;
while ( ( bytes = recv( s, buffer, sizeof( buffer ), 0 ) ) > 0 )
write( 1, buffer, bytes );
close( s );
return;
}
\ No newline at end of file
......@@ -47,9 +47,7 @@ void Spider::FuncToRun()
{
if ( cond )
{
// markURLSeen( currentUrl );
//StreamReader* reader = request( currentUrl );
StreamReader *reader = request( currentUrl );
......
No preview for this file type
......@@ -84,7 +84,7 @@ unordered_map < string, int > *docMapLookUp = new unordered_map < string, int >(
Crawler crawler( mode, &urlFrontier );
crawler.SpawnSpiders(1 , docMapLookUp);
crawler.SpawnSpiders(3 , docMapLookUp);
crawler.
......
http://www.dailymail.co.uk/ushome/index.html
http://www.bbc.com/
http://www.bbc.co.uk/news/business-42959138
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment