made simple test for parser with new style

1ab1c97d · benbergk · 9d465238 · 1ab1c97d · 1ab1c97d · 1ab1c97d
Commit 1ab1c97d authored 7 years ago by benbergk
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,6 +7,20 @@ add_executable(QueueTest
        shared/ProducerConsumerQueue.h
        shared/ProducerConsumerQueue_test.cpp)

+add_executable(Parser2Test
+        parser/Parser.cpp
+        shared/Document.cpp
+        shared/url.h
+        util/util.cpp
+        util/Tokenizer.cpp
+        shared/ProducerConsumerQueue.h
+        util/stringProcessing.cpp
+        util/Stemmer.cpp
+        parser/tests/parserTest2.cpp
+        crawler/Readers/StreamReader.h
+        crawler/Readers/LocalReader.cpp
+        )
+
 add_executable(crawler-parser-test
        main.cpp
        shared/ProducerConsumerQueue.h
@@ -113,6 +127,9 @@ find_package(OpenSSL REQUIRED)

 target_link_libraries(ParserTest OpenSSL::SSL)

+target_link_libraries(Parser2Test OpenSSL::SSL)
+
+
 target_link_libraries(isolated-integration OpenSSL::SSL pthread)

 target_link_libraries(crawler-parser-test OpenSSL::SSL pthread)

--- a/crawler/Readers/LocalReader.h
+++ b/crawler/Readers/LocalReader.h
@@ -10,7 +10,7 @@ class LocalReader : public StreamReader
 	{
 public:

-	LocalReader ( string filename ) : fileName( filename), test_url(filename)
+	LocalReader ( string filename ) : fileName( filename)
 		{ }

 	bool request ( );

--- a/parser/tests/ParserTest2.cpp
+++ b/parser/tests/ParserTest2.cpp
+//
+// Created by Ben Bergkamp on 3/21/18.
+//
+
+#include <string>
+#include <cstring>
+#include <cassert>
+#include <iostream>
+#include "../Parser.h"
+#include "../../shared/Document.h"
+#include "../../shared/ProducerConsumerQueue.h"
+#include "../../crawler/Readers/LocalReader.h"
+#include "../../util/util.h"
+
+using namespace std;
+
+void TestSimple()
+	{
+	ProducerConsumerQueue< ParsedUrl > urlFrontierTest;
+
+	Parser parser( &urlFrontierTest );
+
+
+	ParsedUrl fake_url = ParsedUrl("http://www.cats.com");
+
+	string filepath = util::GetCurrentWorkingDir() + "/tests/plaintext.txt";
+
+	LocalReader reader(filepath);
+	reader.setUrl(fake_url);
+	auto success = reader.request();
+	if(!success)
+	{
+		cerr << "Couldnt open file\n";
+		exit(1);
+	}
+
+	auto dictionary = parser.execute( &reader );
+
+	for ( auto it = dictionary->begin( ); it != dictionary->end( ); it++ )
+	{
+		cout << it->first << ':';
+		for ( int i = 0; i < it->second.size( ); ++i )
+		{
+			cout << it->second[ i ] << " ";
+		}
+		cout << std::endl;
+	}
+
+	}
+
+int main()
+	{
+		TestSimple();
+	}
\ No newline at end of file
--- a/shared/url.h
+++ b/shared/url.h
@@ -29,6 +29,9 @@ private:
 	double Score;

 public:
+
+	ParsedUrl() {}
+
 	ParsedUrl ( string input_url )
 		{
 		// Assumes url points to static text but

--- a/tests/plaintext.txt
+++ b/tests/plaintext.txt
-<title> This is the title </title>
-<body> This is the body </body>
\ No newline at end of file
+<title>
+This is the title test test
+</title>
+<body>
+This is the body test test
+</body>
\ No newline at end of file