modified seed url set up, working pull from the web

.gitignore

@@ -3,3 +3,9 @@
 CMakeLists.txt
 cmake-build-debug/*
 Vagrantfile
+CMakeCache.txt
+CMakeFiles/*
+cmake_install.cmake
+eecs398_search.cbp
+main
+

crawler/spider.cpp

@@ -123,7 +123,7 @@ bool Spider::shouldURLbeCrawled( string url )
 				std::cout << it->first << " => " << it->second << '\n';
 
 			close( file );
-
+			return true;
 			}
 		}
 

main.cpp

@@ -25,67 +25,74 @@
 using namespace std;
 
 
-int main(int argc, const char * argv[])
-{
-    /*
-     *
-     * Settings Flags to control program on start up
-     * to be read in via command line with default settings
-     *
-     * string :  Mode : Getting content from the web vs local
-     *
-     * string : Seed : filename of list of starting urls
-     *
-     * int  : numberOfSpiders: # of spiders crawler spawns
-     *
-     * int  : numberOfParsers:  # of parsers  spawned
-     *
-     * bool : restoreFromLog: bool represeting if the program should load from saved state
-     */
+int main( int argc, const char *argv[] )
+	{
+	/*
+	 *
+	 * Settings Flags to control program on start up
+	 * to be read in via command line with default settings
+	 *
+	 * string :  Mode : Getting content from the web vs local
+	 *
+	 * string : Seed : filename of list of starting urls
+	 *
+	 * int  : numberOfSpiders: # of spiders crawler spawns
+	 *
+	 * int  : numberOfParsers:  # of parsers  spawned
+	 *
+	 * bool : restoreFromLog: bool represeting if the program should load from saved state
+	 */
 
-    //
-    string mode = "local";
-    // Seed urls?
-    string seed;
-    //
-    int numberOfSpiders;
-    int numberOfParsers;
-    bool restoreFromLog;
+	//
+	string mode = "web";
+	// Seed urls?
+	string seed;
+	//
+	int numberOfSpiders;
+	int numberOfParsers;
+	bool restoreFromLog;
 
 
-    ProducerConsumerQueue<string> urlFrontier;
-    ProducerConsumerQueue<int> fileQueue;
+	ProducerConsumerQueue < string > urlFrontier;
 
-    cout << "Pushed File\n";
-    urlFrontier.Push("tests/cats.html");
-    urlFrontier.Push("tests/store.html");
+	cout << "Pushed File\n";
+	char *seeds;
+	if ( mode == "local" )
+		seeds = util::getFileMap( "/tests/localSeed.txt" );
+	else
+		seeds = util::getFileMap( "/tests/webSeed.txt" );
 
+	string testFile;
+	while ( *seeds )
+		{
+		if ( *seeds == '\n')
+			{
+			urlFrontier.Push(testFile);
+			testFile = "";
+			}
 
-   unordered_map<string, int>* docMapLookUp = new unordered_map<string, int>();
+		else
+			testFile.push_back(*seeds);
+		++seeds;
+	}
+	urlFrontier.Push(testFile);
+//urlFrontier.Push("tests/store.html");
 
 
+unordered_map < string, int > *docMapLookUp = new unordered_map < string, int >( );
 
-    Crawler crawler(mode, &urlFrontier);
 
-    crawler.SpawnSpiders(1 , docMapLookUp);
+Crawler crawler( mode, &urlFrontier );
 
-    crawler.WaitOnAllSpiders();
+crawler.SpawnSpiders(1 , docMapLookUp);
 
+crawler.
 
+WaitOnAllSpiders();
 
-    //This part is a work in progress I was just trying to simulate the
-    // parser and see if they could open and read the file
 
-    cout << "Done Waiting\nQueue Size is: " << fileQueue.Size();
-   auto top = fileQueue.Pop();
-    char buf[100];
-    auto ret = read(top, buf, 100);
 
-    cout << "read val: " << ret;
-    for(int i = 0; i < 100; i++){
+//This part is a work in progress I was just trying to simulate the
+// parser and see if they could open and read the file
 
-        cout << buf[i];
-    }
-
-	
 }
\ No newline at end of file

makefile

-all:
-	g++ -std=c++11 main.cpp crawler/crawler.cpp crawler/spider.cpp crawler/CrawlerStatistics.cpp -o crawler.exe -lpthread
\ No newline at end of file
+# CMAKE generated file: DO NOT EDIT!
+# Generated by "Unix Makefiles" Generator, CMake Version 3.9
+
+# Default target executed when no arguments are given to make.
+default_target: all
+
+.PHONY : default_target
+
+# Allow only one "make -f Makefile2" at a time, but pass parallelism.
+.NOTPARALLEL:
+
+
+#=============================================================================
+# Special targets provided by cmake.
+
+# Disable implicit rules so canonical targets will work.
+.SUFFIXES:
+
+
+# Remove some rules from gmake that .SUFFIXES does not remove.
+SUFFIXES =
+
+.SUFFIXES: .hpux_make_needs_suffix_list
+
+
+# Suppress display of executed commands.
+$(VERBOSE).SILENT:
+
+
+# A target that is always out of date.
+cmake_force:
+
+.PHONY : cmake_force
+
+#=============================================================================
+# Set environment variables for the build.
+
+# The shell in which to execute make rules.
+SHELL = /bin/sh
+
+# The CMake executable.
+CMAKE_COMMAND = /Applications/CLion.app/Contents/bin/cmake/bin/cmake
+
+# The command to remove a file.
+RM = /Applications/CLion.app/Contents/bin/cmake/bin/cmake -E remove -f
+
+# Escaping for special characters.
+EQUALS = =
+
+# The top-level source directory on which CMake was run.
+CMAKE_SOURCE_DIR = /Users/jakeclose/Desktop/398/project/eecs398-search
+
+# The top-level build directory on which CMake was run.
+CMAKE_BINARY_DIR = /Users/jakeclose/Desktop/398/project/eecs398-search
+
+#=============================================================================
+# Targets provided globally by CMake.
+
+# Special rule for the target rebuild_cache
+rebuild_cache:
+	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake to regenerate build system..."
+	/Applications/CLion.app/Contents/bin/cmake/bin/cmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
+.PHONY : rebuild_cache
+
+# Special rule for the target rebuild_cache
+rebuild_cache/fast: rebuild_cache
+
+.PHONY : rebuild_cache/fast
+
+# Special rule for the target edit_cache
+edit_cache:
+	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "No interactive CMake dialog available..."
+	/Applications/CLion.app/Contents/bin/cmake/bin/cmake -E echo No\ interactive\ CMake\ dialog\ available.
+.PHONY : edit_cache
+
+# Special rule for the target edit_cache
+edit_cache/fast: edit_cache
+
+.PHONY : edit_cache/fast
+
+# The main all target
+all: cmake_check_build_system
+	$(CMAKE_COMMAND) -E cmake_progress_start /Users/jakeclose/Desktop/398/project/eecs398-search/CMakeFiles /Users/jakeclose/Desktop/398/project/eecs398-search/CMakeFiles/progress.marks
+	$(MAKE) -f CMakeFiles/Makefile2 all
+	$(CMAKE_COMMAND) -E cmake_progress_start /Users/jakeclose/Desktop/398/project/eecs398-search/CMakeFiles 0
+.PHONY : all
+
+# The main clean target
+clean:
+	$(MAKE) -f CMakeFiles/Makefile2 clean
+.PHONY : clean
+
+# The main clean target
+clean/fast: clean
+
+.PHONY : clean/fast
+
+# Prepare targets for installation.
+preinstall: all
+	$(MAKE) -f CMakeFiles/Makefile2 preinstall
+.PHONY : preinstall
+
+# Prepare targets for installation.
+preinstall/fast:
+	$(MAKE) -f CMakeFiles/Makefile2 preinstall
+.PHONY : preinstall/fast
+
+# clear depends
+depend:
+	$(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1
+.PHONY : depend
+
+#=============================================================================
+# Target rules for targets named url_test
+
+# Build rule for target.
+url_test: cmake_check_build_system
+	$(MAKE) -f CMakeFiles/Makefile2 url_test
+.PHONY : url_test
+
+# fast build rule for target.
+url_test/fast:
+	$(MAKE) -f CMakeFiles/url_test.dir/build.make CMakeFiles/url_test.dir/build
+.PHONY : url_test/fast
+
+#=============================================================================
+# Target rules for targets named main
+
+# Build rule for target.
+main: cmake_check_build_system
+	$(MAKE) -f CMakeFiles/Makefile2 main
+.PHONY : main
+
+# fast build rule for target.
+main/fast:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/build
+.PHONY : main/fast
+
+crawler/SocketReader.o: crawler/SocketReader.cpp.o
+
+.PHONY : crawler/SocketReader.o
+
+# target to build an object file
+crawler/SocketReader.cpp.o:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/SocketReader.cpp.o
+.PHONY : crawler/SocketReader.cpp.o
+
+crawler/SocketReader.i: crawler/SocketReader.cpp.i
+
+.PHONY : crawler/SocketReader.i
+
+# target to preprocess a source file
+crawler/SocketReader.cpp.i:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/SocketReader.cpp.i
+.PHONY : crawler/SocketReader.cpp.i
+
+crawler/SocketReader.s: crawler/SocketReader.cpp.s
+
+.PHONY : crawler/SocketReader.s
+
+# target to generate assembly for a file
+crawler/SocketReader.cpp.s:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/SocketReader.cpp.s
+.PHONY : crawler/SocketReader.cpp.s
+
+crawler/crawler.o: crawler/crawler.cpp.o
+
+.PHONY : crawler/crawler.o
+
+# target to build an object file
+crawler/crawler.cpp.o:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/crawler.cpp.o
+.PHONY : crawler/crawler.cpp.o
+
+crawler/crawler.i: crawler/crawler.cpp.i
+
+.PHONY : crawler/crawler.i
+
+# target to preprocess a source file
+crawler/crawler.cpp.i:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/crawler.cpp.i
+.PHONY : crawler/crawler.cpp.i
+
+crawler/crawler.s: crawler/crawler.cpp.s
+
+.PHONY : crawler/crawler.s
+
+# target to generate assembly for a file
+crawler/crawler.cpp.s:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/crawler.cpp.s
+.PHONY : crawler/crawler.cpp.s
+
+crawler/spider.o: crawler/spider.cpp.o
+
+.PHONY : crawler/spider.o
+
+# target to build an object file
+crawler/spider.cpp.o:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/spider.cpp.o
+.PHONY : crawler/spider.cpp.o
+
+crawler/spider.i: crawler/spider.cpp.i
+
+.PHONY : crawler/spider.i
+
+# target to preprocess a source file
+crawler/spider.cpp.i:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/spider.cpp.i
+.PHONY : crawler/spider.cpp.i
+
+crawler/spider.s: crawler/spider.cpp.s
+
+.PHONY : crawler/spider.s
+
+# target to generate assembly for a file
+crawler/spider.cpp.s:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/crawler/spider.cpp.s
+.PHONY : crawler/spider.cpp.s
+
+main.o: main.cpp.o
+
+.PHONY : main.o
+
+# target to build an object file
+main.cpp.o:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/main.cpp.o
+.PHONY : main.cpp.o
+
+main.i: main.cpp.i
+
+.PHONY : main.i
+
+# target to preprocess a source file
+main.cpp.i:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/main.cpp.i
+.PHONY : main.cpp.i
+
+main.s: main.cpp.s
+
+.PHONY : main.s
+
+# target to generate assembly for a file
+main.cpp.s:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/main.cpp.s
+.PHONY : main.cpp.s
+
+shared/urlTest.o: shared/urlTest.cpp.o
+
+.PHONY : shared/urlTest.o
+
+# target to build an object file
+shared/urlTest.cpp.o:
+	$(MAKE) -f CMakeFiles/url_test.dir/build.make CMakeFiles/url_test.dir/shared/urlTest.cpp.o
+.PHONY : shared/urlTest.cpp.o
+
+shared/urlTest.i: shared/urlTest.cpp.i
+
+.PHONY : shared/urlTest.i
+
+# target to preprocess a source file
+shared/urlTest.cpp.i:
+	$(MAKE) -f CMakeFiles/url_test.dir/build.make CMakeFiles/url_test.dir/shared/urlTest.cpp.i
+.PHONY : shared/urlTest.cpp.i
+
+shared/urlTest.s: shared/urlTest.cpp.s
+
+.PHONY : shared/urlTest.s
+
+# target to generate assembly for a file
+shared/urlTest.cpp.s:
+	$(MAKE) -f CMakeFiles/url_test.dir/build.make CMakeFiles/url_test.dir/shared/urlTest.cpp.s
+.PHONY : shared/urlTest.cpp.s
+
+util/util.o: util/util.cpp.o
+
+.PHONY : util/util.o
+
+# target to build an object file
+util/util.cpp.o:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/util/util.cpp.o
+.PHONY : util/util.cpp.o
+
+util/util.i: util/util.cpp.i
+
+.PHONY : util/util.i
+
+# target to preprocess a source file
+util/util.cpp.i:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/util/util.cpp.i
+.PHONY : util/util.cpp.i
+
+util/util.s: util/util.cpp.s
+
+.PHONY : util/util.s
+
+# target to generate assembly for a file
+util/util.cpp.s:
+	$(MAKE) -f CMakeFiles/main.dir/build.make CMakeFiles/main.dir/util/util.cpp.s
+.PHONY : util/util.cpp.s
+
+# Help Target
+help:
+	@echo "The following are some of the valid targets for this Makefile:"
+	@echo "... all (the default if no target is provided)"
+	@echo "... clean"
+	@echo "... depend"
+	@echo "... rebuild_cache"
+	@echo "... edit_cache"
+	@echo "... url_test"
+	@echo "... main"
+	@echo "... crawler/SocketReader.o"
+	@echo "... crawler/SocketReader.i"
+	@echo "... crawler/SocketReader.s"
+	@echo "... crawler/crawler.o"
+	@echo "... crawler/crawler.i"
+	@echo "... crawler/crawler.s"
+	@echo "... crawler/spider.o"
+	@echo "... crawler/spider.i"
+	@echo "... crawler/spider.s"
+	@echo "... main.o"
+	@echo "... main.i"
+	@echo "... main.s"
+	@echo "... shared/urlTest.o"
+	@echo "... shared/urlTest.i"
+	@echo "... shared/urlTest.s"
+	@echo "... util/util.o"
+	@echo "... util/util.i"
+	@echo "... util/util.s"
+.PHONY : help
+
+
+
+#=============================================================================
+# Special targets to cleanup operation of make.
+
+# Special rule to run CMake to check the build system integrity.
+# No rule that depends on this can have commands that come from listfiles
+# because they might be regenerated.
+cmake_check_build_system:
+	$(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0
+.PHONY : cmake_check_build_system
+

tests/localSeed.txt

+tests/cats.html
+tests/store.html
\ No newline at end of file

tests/webSeed.txt

+http://www.dailymail.co.uk/ushome/index.html
+http://www.bbc.com/