added PageToString functions

crawler/Readers/HttpReader.cpp

@@ -47,6 +47,32 @@ bool HttpReader::fillBuffer(char * buf, size_t buf_size)
 	return (recv( sock, buf, buf_size, 0 ) == buf_size);
 	}
 
+string HttpReader::PageToString()
+	{
+	int total_size = 0;
+	int buf_size = 10240;
+	int current_size = buf_size;
+	char* http_buff = new char[buf_size];
+	char* front = http_buff;
+	int bytes;
+
+	while ( ( bytes = recv( sock, front, buf_size, 0 ) ) > 0 )
+	{
+		total_size += bytes;
+		current_size  += buf_size;
+		char *temp = new char[current_size];
+		strcpy(temp, http_buff);
+
+		front = temp + strlen(http_buff);
+		delete[] http_buff;
+		http_buff = temp;
+	}
+
+	return string(http_buff, total_size);
+	}
+
+
+
 void HttpReader::closeReader()
 	{
 	close( sock );

crawler/Readers/HttpReader.h

@@ -13,6 +13,7 @@ public:
 
 		void request();
 		bool fillBuffer(char * buf, size_t buf_size);
+		string PageToString();
 		void closeReader();
 
 

crawler/Readers/HttpsReader.cpp

@@ -62,10 +62,36 @@ bool HttpsReader::fillBuffer(char * buf, size_t buf_size)
 	return (SSL_read( ssl, buf, buf_size ) == buf_size);
 	}
 
+string HttpsReader::PageToString()
+	{
+	int total_size = 0;
+	int buf_size = 10240;
+	int current_size = buf_size;
+	char *ssl_buffer = new char[buf_size];
+	char *front = ssl_buffer;
+	int bytes;
+
+	while ((bytes = SSL_read(ssl, front, buf_size)) > 0) {
+
+		total_size += bytes;
+		current_size += buf_size;
+		char *temp = new char[current_size];
+		strcpy(temp, ssl_buffer);
+
+		front = temp + strlen(ssl_buffer);
+		delete[] ssl_buffer;
+		ssl_buffer = temp;
+	}
+
+	return string(ssl_buffer, total_size);
+
+	}
+
 void HttpsReader::closeReader()
 	{
-	SSL_shutdown( ssl );
-	SSL_free( ssl );
-	SSL_CTX_free( ctx );
-	close( sock );
+	SSL_shutdown(ssl);
+	SSL_free(ssl);
+	SSL_CTX_free(ctx);
+	close(sock);
 	}
+

crawler/Readers/HttpsReader.h

@@ -14,6 +14,7 @@ public:
 
 	void request();
 	bool fillBuffer(char * buf, size_t buf_size);
+	string PageToString();
 	void closeReader();
 
 

crawler/Readers/LocalReader.cpp

@@ -19,6 +19,14 @@ bool LocalReader::fillBuffer(char * buf, size_t buf_size){
 
 	}
 
+string LocalReader::PageToString()
+	{
+	//FIXME
+	string s("fix me");
+	return s;
+	}
+
+
 void LocalReader::closeReader()
 	{
 	//FIXME

crawler/Readers/LocalReader.h

@@ -14,6 +14,7 @@ public:
 
 	void request();
 	bool fillBuffer(char * buf, size_t buf_size);
+	string PageToString();
 	void closeReader();
 
 private:

crawler/Readers/StreamReader.h

@@ -16,6 +16,8 @@
 #include <cassert>
 #include <openssl/ssl.h>
 
+using namespace std;
+
 
 class StreamReader
 	{
@@ -24,6 +26,8 @@ public:
 
 	virtual void request() = 0;
 	virtual bool fillBuffer(char * buf, size_t buf_size) = 0;
+	virtual string PageToString() = 0;
 	virtual void closeReader() = 0;
 
+
 	};