From 82eb00c1ce67daf2b7499f5b64587dc97960e3ec Mon Sep 17 00:00:00 2001
From: bergkampben <benbergk@umich.edu>
Date: Tue, 13 Mar 2018 18:56:17 -0400
Subject: [PATCH] initial refactored commit
---
CMakeLists.txt | 62 --
cmake-build-debug/CMakeFiles/Makefile.cmake | 64 --
cmake-build-debug/CMakeFiles/Makefile2 | 367 ----------
.../CMakeFiles/TargetDirectories.txt | 10 -
cmake-build-debug/CMakeFiles/progress.marks | 1 -
cmake-build-debug/Makefile | 684 ------------------
crawler/Readers/HttpReader.cpp | 53 ++
crawler/Readers/HttpReader.h | 24 +
crawler/Readers/HttpsReader.cpp | 71 ++
crawler/Readers/HttpsReader.h | 25 +
crawler/{ => Readers}/LocalReader.h | 13 +-
crawler/Readers/StreamReader.h | 55 ++
crawler/SocketReader.cpp | 185 -----
crawler/SocketReader.h | 27 -
crawler/StreamReader.h | 30 -
crawler/spider.cpp | 49 +-
crawler/spider.h | 9 +-
17 files changed, 250 insertions(+), 1479 deletions(-)
delete mode 100644 CMakeLists.txt
delete mode 100644 cmake-build-debug/CMakeFiles/Makefile.cmake
delete mode 100644 cmake-build-debug/CMakeFiles/Makefile2
delete mode 100644 cmake-build-debug/CMakeFiles/TargetDirectories.txt
delete mode 100644 cmake-build-debug/CMakeFiles/progress.marks
delete mode 100644 cmake-build-debug/Makefile
create mode 100644 crawler/Readers/HttpReader.cpp
create mode 100644 crawler/Readers/HttpReader.h
create mode 100644 crawler/Readers/HttpsReader.cpp
create mode 100644 crawler/Readers/HttpsReader.h
rename crawler/{ => Readers}/LocalReader.h (53%)
create mode 100644 crawler/Readers/StreamReader.h
delete mode 100644 crawler/SocketReader.cpp
delete mode 100644 crawler/SocketReader.h
delete mode 100644 crawler/StreamReader.h
diff --git a/CMakeLists.txt b/CMakeLists.txt
deleted file mode 100644
index b931bfc..0000000
--- a/CMakeLists.txt
+++ /dev/null
@@ -1,62 +0,0 @@
-cmake_minimum_required(VERSION 3.9)
-project(eecs398_search)
-
-set(CMAKE_CXX_STANDARD 11)
-
-add_executable(parser-test
- main.cpp
- shared/ProducerConsumerQueue.cpp
- shared/ProducerConsumerQueue.h
- parser/Parser.h
- util/Tokenizer.h
- util/stringProcessing.h
- util/Stemmer.h)
-
-
-
-add_executable(crawl
- main.cpp
- shared/ProducerConsumerQueue.h
- shared/ThreadClass.h
- shared/url.h
- crawler/crawler.cpp
- crawler/SocketReader.cpp
- crawler/StreamReader.h
- crawler/spider.cpp
- util/util.cpp
- crawler/LocalReader.h
- shared/Document.cpp
- parser/Parser.cpp
- util/stringProcessing.cpp indexStreamReader/ISR.cpp indexStreamReader/ISR.h)
-
-
-
-add_executable(StringProcessingTest
- util/stringProcessing.h
- util/Stemmer.h
- util/tests/stringProcessingTest.cpp)
-
-add_executable(TokenizerTest
- util/Tokenizer.h
- util/tests/tokenizerTest.cpp)
-
-add_executable(StemmerTest
- util/Stemmer.h
- util/tests/stemmerTest.cpp)
-
-add_executable(ParserEndToEndTest
- parser/Parser.h
- shared/Document.h
- shared/url.h
- parser/tests/parserTest.cpp)
-
-
-add_executable(URLTEST shared/url.h shared/urlTest.cpp)
-
-
-add_executable(search-engine search.cpp query/Query.cpp)
-
-find_package(OpenSSL REQUIRED)
-
-target_link_libraries(crawl OpenSSL::SSL pthread)
-
diff --git a/cmake-build-debug/CMakeFiles/Makefile.cmake b/cmake-build-debug/CMakeFiles/Makefile.cmake
deleted file mode 100644
index 057e5ad..0000000
--- a/cmake-build-debug/CMakeFiles/Makefile.cmake
+++ /dev/null
@@ -1,64 +0,0 @@
-# CMAKE generated file: DO NOT EDIT!
-# Generated by "Unix Makefiles" Generator, CMake Version 3.9
-
-# The generator used is:
-set(CMAKE_DEPENDS_GENERATOR "Unix Makefiles")
-
-# The top level Makefile was generated from the following files:
-set(CMAKE_MAKEFILE_DEPENDS
- "CMakeCache.txt"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeCInformation.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeCXXInformation.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeCommonLanguageInclude.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeExtraGeneratorDetermineCompilerMacrosAndIncludeDirs.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeFindCodeBlocks.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeGenericSystem.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeLanguageInformation.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeSystemSpecificInformation.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeSystemSpecificInitialize.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/AppleClang-C.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/AppleClang-CXX.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/CMakeCommonCompilerMacros.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/Clang.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/GNU.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindOpenSSL.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindPackageHandleStandardArgs.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindPackageMessage.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindPkgConfig.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-AppleClang-C.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-AppleClang-CXX.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Clang-C.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Clang-CXX.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Clang.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Initialize.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/UnixPaths.cmake"
- "/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/ProcessorCount.cmake"
- "../CMakeLists.txt"
- "CMakeFiles/3.9.6/CMakeCCompiler.cmake"
- "CMakeFiles/3.9.6/CMakeCXXCompiler.cmake"
- "CMakeFiles/3.9.6/CMakeSystem.cmake"
- )
-
-# The corresponding makefile is:
-set(CMAKE_MAKEFILE_OUTPUTS
- "Makefile"
- "CMakeFiles/cmake.check_cache"
- )
-
-# Byproducts of CMake generate step:
-set(CMAKE_MAKEFILE_PRODUCTS
- "CMakeFiles/CMakeDirectoryInformation.cmake"
- )
-
-# Dependency information for all targets:
-set(CMAKE_DEPEND_INFO_FILES
- "CMakeFiles/URLTEST.dir/DependInfo.cmake"
- "CMakeFiles/ParserEndToEndTest.dir/DependInfo.cmake"
- "CMakeFiles/StemmerTest.dir/DependInfo.cmake"
- "CMakeFiles/StringProcessingTest.dir/DependInfo.cmake"
- "CMakeFiles/TokenizerTest.dir/DependInfo.cmake"
- "CMakeFiles/test1.dir/DependInfo.cmake"
- "CMakeFiles/crawl.dir/DependInfo.cmake"
- "CMakeFiles/search.dir/DependInfo.cmake"
- )
diff --git a/cmake-build-debug/CMakeFiles/Makefile2 b/cmake-build-debug/CMakeFiles/Makefile2
deleted file mode 100644
index 8946ce4..0000000
--- a/cmake-build-debug/CMakeFiles/Makefile2
+++ /dev/null
@@ -1,367 +0,0 @@
-# CMAKE generated file: DO NOT EDIT!
-# Generated by "Unix Makefiles" Generator, CMake Version 3.9
-
-# Default target executed when no arguments are given to make.
-default_target: all
-
-.PHONY : default_target
-
-# The main recursive all target
-all:
-
-.PHONY : all
-
-# The main recursive preinstall target
-preinstall:
-
-.PHONY : preinstall
-
-#=============================================================================
-# Special targets provided by cmake.
-
-# Disable implicit rules so canonical targets will work.
-.SUFFIXES:
-
-
-# Remove some rules from gmake that .SUFFIXES does not remove.
-SUFFIXES =
-
-.SUFFIXES: .hpux_make_needs_suffix_list
-
-
-# Suppress display of executed commands.
-$(VERBOSE).SILENT:
-
-
-# A target that is always out of date.
-cmake_force:
-
-.PHONY : cmake_force
-
-#=============================================================================
-# Set environment variables for the build.
-
-# The shell in which to execute make rules.
-SHELL = /bin/sh
-
-# The CMake executable.
-CMAKE_COMMAND = /Applications/CLion.app/Contents/bin/cmake/bin/cmake
-
-# The command to remove a file.
-RM = /Applications/CLion.app/Contents/bin/cmake/bin/cmake -E remove -f
-
-# Escaping for special characters.
-EQUALS = =
-
-# The top-level source directory on which CMake was run.
-CMAKE_SOURCE_DIR = /Users/benbergkamp/Desktop/398/eecs398-search
-
-# The top-level build directory on which CMake was run.
-CMAKE_BINARY_DIR = /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug
-
-#=============================================================================
-# Target rules for target CMakeFiles/URLTEST.dir
-
-# All Build rule for target.
-CMakeFiles/URLTEST.dir/all:
- $(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/depend
- $(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=9,10 "Built target URLTEST"
-.PHONY : CMakeFiles/URLTEST.dir/all
-
-# Include target in all.
-all: CMakeFiles/URLTEST.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/URLTEST.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/URLTEST.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/URLTEST.dir/rule
-
-# Convenience name for target.
-URLTEST: CMakeFiles/URLTEST.dir/rule
-
-.PHONY : URLTEST
-
-# clean rule for target.
-CMakeFiles/URLTEST.dir/clean:
- $(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/clean
-.PHONY : CMakeFiles/URLTEST.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/URLTEST.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Target rules for target CMakeFiles/ParserEndToEndTest.dir
-
-# All Build rule for target.
-CMakeFiles/ParserEndToEndTest.dir/all:
- $(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/depend
- $(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=1,2 "Built target ParserEndToEndTest"
-.PHONY : CMakeFiles/ParserEndToEndTest.dir/all
-
-# Include target in all.
-all: CMakeFiles/ParserEndToEndTest.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/ParserEndToEndTest.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/ParserEndToEndTest.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/ParserEndToEndTest.dir/rule
-
-# Convenience name for target.
-ParserEndToEndTest: CMakeFiles/ParserEndToEndTest.dir/rule
-
-.PHONY : ParserEndToEndTest
-
-# clean rule for target.
-CMakeFiles/ParserEndToEndTest.dir/clean:
- $(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/clean
-.PHONY : CMakeFiles/ParserEndToEndTest.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/ParserEndToEndTest.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Target rules for target CMakeFiles/StemmerTest.dir
-
-# All Build rule for target.
-CMakeFiles/StemmerTest.dir/all:
- $(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/depend
- $(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=3,4 "Built target StemmerTest"
-.PHONY : CMakeFiles/StemmerTest.dir/all
-
-# Include target in all.
-all: CMakeFiles/StemmerTest.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/StemmerTest.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/StemmerTest.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/StemmerTest.dir/rule
-
-# Convenience name for target.
-StemmerTest: CMakeFiles/StemmerTest.dir/rule
-
-.PHONY : StemmerTest
-
-# clean rule for target.
-CMakeFiles/StemmerTest.dir/clean:
- $(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/clean
-.PHONY : CMakeFiles/StemmerTest.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/StemmerTest.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Target rules for target CMakeFiles/StringProcessingTest.dir
-
-# All Build rule for target.
-CMakeFiles/StringProcessingTest.dir/all:
- $(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/depend
- $(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=5,6 "Built target StringProcessingTest"
-.PHONY : CMakeFiles/StringProcessingTest.dir/all
-
-# Include target in all.
-all: CMakeFiles/StringProcessingTest.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/StringProcessingTest.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/StringProcessingTest.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/StringProcessingTest.dir/rule
-
-# Convenience name for target.
-StringProcessingTest: CMakeFiles/StringProcessingTest.dir/rule
-
-.PHONY : StringProcessingTest
-
-# clean rule for target.
-CMakeFiles/StringProcessingTest.dir/clean:
- $(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/clean
-.PHONY : CMakeFiles/StringProcessingTest.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/StringProcessingTest.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Target rules for target CMakeFiles/TokenizerTest.dir
-
-# All Build rule for target.
-CMakeFiles/TokenizerTest.dir/all:
- $(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/depend
- $(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=7,8 "Built target TokenizerTest"
-.PHONY : CMakeFiles/TokenizerTest.dir/all
-
-# Include target in all.
-all: CMakeFiles/TokenizerTest.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/TokenizerTest.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/TokenizerTest.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/TokenizerTest.dir/rule
-
-# Convenience name for target.
-TokenizerTest: CMakeFiles/TokenizerTest.dir/rule
-
-.PHONY : TokenizerTest
-
-# clean rule for target.
-CMakeFiles/TokenizerTest.dir/clean:
- $(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/clean
-.PHONY : CMakeFiles/TokenizerTest.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/TokenizerTest.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Target rules for target CMakeFiles/test1.dir
-
-# All Build rule for target.
-CMakeFiles/test1.dir/all:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/depend
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=23,24,25,26,27,28 "Built target test1"
-.PHONY : CMakeFiles/test1.dir/all
-
-# Include target in all.
-all: CMakeFiles/test1.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/test1.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 6
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/test1.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/test1.dir/rule
-
-# Convenience name for target.
-test1: CMakeFiles/test1.dir/rule
-
-.PHONY : test1
-
-# clean rule for target.
-CMakeFiles/test1.dir/clean:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/clean
-.PHONY : CMakeFiles/test1.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/test1.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Target rules for target CMakeFiles/crawl.dir
-
-# All Build rule for target.
-CMakeFiles/crawl.dir/all:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/depend
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=11,12,13,14,15,16,17,18,19 "Built target crawl"
-.PHONY : CMakeFiles/crawl.dir/all
-
-# Include target in all.
-all: CMakeFiles/crawl.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/crawl.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 9
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/crawl.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/crawl.dir/rule
-
-# Convenience name for target.
-crawl: CMakeFiles/crawl.dir/rule
-
-.PHONY : crawl
-
-# clean rule for target.
-CMakeFiles/crawl.dir/clean:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/clean
-.PHONY : CMakeFiles/crawl.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/crawl.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Target rules for target CMakeFiles/search.dir
-
-# All Build rule for target.
-CMakeFiles/search.dir/all:
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/depend
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/build
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=20,21,22 "Built target search"
-.PHONY : CMakeFiles/search.dir/all
-
-# Include target in all.
-all: CMakeFiles/search.dir/all
-
-.PHONY : all
-
-# Build rule for subdir invocation for target.
-CMakeFiles/search.dir/rule: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 3
- $(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/search.dir/all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : CMakeFiles/search.dir/rule
-
-# Convenience name for target.
-search: CMakeFiles/search.dir/rule
-
-.PHONY : search
-
-# clean rule for target.
-CMakeFiles/search.dir/clean:
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/clean
-.PHONY : CMakeFiles/search.dir/clean
-
-# clean rule for target.
-clean: CMakeFiles/search.dir/clean
-
-.PHONY : clean
-
-#=============================================================================
-# Special targets to cleanup operation of make.
-
-# Special rule to run CMake to check the build system integrity.
-# No rule that depends on this can have commands that come from listfiles
-# because they might be regenerated.
-cmake_check_build_system:
- $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0
-.PHONY : cmake_check_build_system
-
diff --git a/cmake-build-debug/CMakeFiles/TargetDirectories.txt b/cmake-build-debug/CMakeFiles/TargetDirectories.txt
deleted file mode 100644
index 1126b31..0000000
--- a/cmake-build-debug/CMakeFiles/TargetDirectories.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/edit_cache.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/URLTEST.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/ParserEndToEndTest.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/rebuild_cache.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/StemmerTest.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/StringProcessingTest.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/TokenizerTest.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/test1.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/crawl.dir
-/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/search.dir
diff --git a/cmake-build-debug/CMakeFiles/progress.marks b/cmake-build-debug/CMakeFiles/progress.marks
deleted file mode 100644
index 9902f17..0000000
--- a/cmake-build-debug/CMakeFiles/progress.marks
+++ /dev/null
@@ -1 +0,0 @@
-28
diff --git a/cmake-build-debug/Makefile b/cmake-build-debug/Makefile
deleted file mode 100644
index e005a96..0000000
--- a/cmake-build-debug/Makefile
+++ /dev/null
@@ -1,684 +0,0 @@
-# CMAKE generated file: DO NOT EDIT!
-# Generated by "Unix Makefiles" Generator, CMake Version 3.9
-
-# Default target executed when no arguments are given to make.
-default_target: all
-
-.PHONY : default_target
-
-# Allow only one "make -f Makefile2" at a time, but pass parallelism.
-.NOTPARALLEL:
-
-
-#=============================================================================
-# Special targets provided by cmake.
-
-# Disable implicit rules so canonical targets will work.
-.SUFFIXES:
-
-
-# Remove some rules from gmake that .SUFFIXES does not remove.
-SUFFIXES =
-
-.SUFFIXES: .hpux_make_needs_suffix_list
-
-
-# Suppress display of executed commands.
-$(VERBOSE).SILENT:
-
-
-# A target that is always out of date.
-cmake_force:
-
-.PHONY : cmake_force
-
-#=============================================================================
-# Set environment variables for the build.
-
-# The shell in which to execute make rules.
-SHELL = /bin/sh
-
-# The CMake executable.
-CMAKE_COMMAND = /Applications/CLion.app/Contents/bin/cmake/bin/cmake
-
-# The command to remove a file.
-RM = /Applications/CLion.app/Contents/bin/cmake/bin/cmake -E remove -f
-
-# Escaping for special characters.
-EQUALS = =
-
-# The top-level source directory on which CMake was run.
-CMAKE_SOURCE_DIR = /Users/benbergkamp/Desktop/398/eecs398-search
-
-# The top-level build directory on which CMake was run.
-CMAKE_BINARY_DIR = /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug
-
-#=============================================================================
-# Targets provided globally by CMake.
-
-# Special rule for the target edit_cache
-edit_cache:
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "No interactive CMake dialog available..."
- /Applications/CLion.app/Contents/bin/cmake/bin/cmake -E echo No\ interactive\ CMake\ dialog\ available.
-.PHONY : edit_cache
-
-# Special rule for the target edit_cache
-edit_cache/fast: edit_cache
-
-.PHONY : edit_cache/fast
-
-# Special rule for the target rebuild_cache
-rebuild_cache:
- @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake to regenerate build system..."
- /Applications/CLion.app/Contents/bin/cmake/bin/cmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
-.PHONY : rebuild_cache
-
-# Special rule for the target rebuild_cache
-rebuild_cache/fast: rebuild_cache
-
-.PHONY : rebuild_cache/fast
-
-# The main all target
-all: cmake_check_build_system
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/progress.marks
- $(MAKE) -f CMakeFiles/Makefile2 all
- $(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
-.PHONY : all
-
-# The main clean target
-clean:
- $(MAKE) -f CMakeFiles/Makefile2 clean
-.PHONY : clean
-
-# The main clean target
-clean/fast: clean
-
-.PHONY : clean/fast
-
-# Prepare targets for installation.
-preinstall: all
- $(MAKE) -f CMakeFiles/Makefile2 preinstall
-.PHONY : preinstall
-
-# Prepare targets for installation.
-preinstall/fast:
- $(MAKE) -f CMakeFiles/Makefile2 preinstall
-.PHONY : preinstall/fast
-
-# clear depends
-depend:
- $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1
-.PHONY : depend
-
-#=============================================================================
-# Target rules for targets named URLTEST
-
-# Build rule for target.
-URLTEST: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 URLTEST
-.PHONY : URLTEST
-
-# fast build rule for target.
-URLTEST/fast:
- $(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/build
-.PHONY : URLTEST/fast
-
-#=============================================================================
-# Target rules for targets named ParserEndToEndTest
-
-# Build rule for target.
-ParserEndToEndTest: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 ParserEndToEndTest
-.PHONY : ParserEndToEndTest
-
-# fast build rule for target.
-ParserEndToEndTest/fast:
- $(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/build
-.PHONY : ParserEndToEndTest/fast
-
-#=============================================================================
-# Target rules for targets named StemmerTest
-
-# Build rule for target.
-StemmerTest: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 StemmerTest
-.PHONY : StemmerTest
-
-# fast build rule for target.
-StemmerTest/fast:
- $(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/build
-.PHONY : StemmerTest/fast
-
-#=============================================================================
-# Target rules for targets named StringProcessingTest
-
-# Build rule for target.
-StringProcessingTest: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 StringProcessingTest
-.PHONY : StringProcessingTest
-
-# fast build rule for target.
-StringProcessingTest/fast:
- $(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/build
-.PHONY : StringProcessingTest/fast
-
-#=============================================================================
-# Target rules for targets named TokenizerTest
-
-# Build rule for target.
-TokenizerTest: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 TokenizerTest
-.PHONY : TokenizerTest
-
-# fast build rule for target.
-TokenizerTest/fast:
- $(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/build
-.PHONY : TokenizerTest/fast
-
-#=============================================================================
-# Target rules for targets named test1
-
-# Build rule for target.
-test1: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 test1
-.PHONY : test1
-
-# fast build rule for target.
-test1/fast:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/build
-.PHONY : test1/fast
-
-#=============================================================================
-# Target rules for targets named crawl
-
-# Build rule for target.
-crawl: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 crawl
-.PHONY : crawl
-
-# fast build rule for target.
-crawl/fast:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/build
-.PHONY : crawl/fast
-
-#=============================================================================
-# Target rules for targets named search
-
-# Build rule for target.
-search: cmake_check_build_system
- $(MAKE) -f CMakeFiles/Makefile2 search
-.PHONY : search
-
-# fast build rule for target.
-search/fast:
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/build
-.PHONY : search/fast
-
-crawler/SocketReader.o: crawler/SocketReader.cpp.o
-
-.PHONY : crawler/SocketReader.o
-
-# target to build an object file
-crawler/SocketReader.cpp.o:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/SocketReader.cpp.o
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/SocketReader.cpp.o
-.PHONY : crawler/SocketReader.cpp.o
-
-crawler/SocketReader.i: crawler/SocketReader.cpp.i
-
-.PHONY : crawler/SocketReader.i
-
-# target to preprocess a source file
-crawler/SocketReader.cpp.i:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/SocketReader.cpp.i
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/SocketReader.cpp.i
-.PHONY : crawler/SocketReader.cpp.i
-
-crawler/SocketReader.s: crawler/SocketReader.cpp.s
-
-.PHONY : crawler/SocketReader.s
-
-# target to generate assembly for a file
-crawler/SocketReader.cpp.s:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/SocketReader.cpp.s
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/SocketReader.cpp.s
-.PHONY : crawler/SocketReader.cpp.s
-
-crawler/crawler.o: crawler/crawler.cpp.o
-
-.PHONY : crawler/crawler.o
-
-# target to build an object file
-crawler/crawler.cpp.o:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/crawler.cpp.o
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/crawler.cpp.o
-.PHONY : crawler/crawler.cpp.o
-
-crawler/crawler.i: crawler/crawler.cpp.i
-
-.PHONY : crawler/crawler.i
-
-# target to preprocess a source file
-crawler/crawler.cpp.i:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/crawler.cpp.i
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/crawler.cpp.i
-.PHONY : crawler/crawler.cpp.i
-
-crawler/crawler.s: crawler/crawler.cpp.s
-
-.PHONY : crawler/crawler.s
-
-# target to generate assembly for a file
-crawler/crawler.cpp.s:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/crawler.cpp.s
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/crawler.cpp.s
-.PHONY : crawler/crawler.cpp.s
-
-crawler/spider.o: crawler/spider.cpp.o
-
-.PHONY : crawler/spider.o
-
-# target to build an object file
-crawler/spider.cpp.o:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/spider.cpp.o
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/spider.cpp.o
-.PHONY : crawler/spider.cpp.o
-
-crawler/spider.i: crawler/spider.cpp.i
-
-.PHONY : crawler/spider.i
-
-# target to preprocess a source file
-crawler/spider.cpp.i:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/spider.cpp.i
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/spider.cpp.i
-.PHONY : crawler/spider.cpp.i
-
-crawler/spider.s: crawler/spider.cpp.s
-
-.PHONY : crawler/spider.s
-
-# target to generate assembly for a file
-crawler/spider.cpp.s:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/spider.cpp.s
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/spider.cpp.s
-.PHONY : crawler/spider.cpp.s
-
-main.o: main.cpp.o
-
-.PHONY : main.o
-
-# target to build an object file
-main.cpp.o:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/main.cpp.o
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/main.cpp.o
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/main.cpp.o
-.PHONY : main.cpp.o
-
-main.i: main.cpp.i
-
-.PHONY : main.i
-
-# target to preprocess a source file
-main.cpp.i:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/main.cpp.i
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/main.cpp.i
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/main.cpp.i
-.PHONY : main.cpp.i
-
-main.s: main.cpp.s
-
-.PHONY : main.s
-
-# target to generate assembly for a file
-main.cpp.s:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/main.cpp.s
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/main.cpp.s
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/main.cpp.s
-.PHONY : main.cpp.s
-
-parser/Parser.o: parser/Parser.cpp.o
-
-.PHONY : parser/Parser.o
-
-# target to build an object file
-parser/Parser.cpp.o:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/parser/Parser.cpp.o
-.PHONY : parser/Parser.cpp.o
-
-parser/Parser.i: parser/Parser.cpp.i
-
-.PHONY : parser/Parser.i
-
-# target to preprocess a source file
-parser/Parser.cpp.i:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/parser/Parser.cpp.i
-.PHONY : parser/Parser.cpp.i
-
-parser/Parser.s: parser/Parser.cpp.s
-
-.PHONY : parser/Parser.s
-
-# target to generate assembly for a file
-parser/Parser.cpp.s:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/parser/Parser.cpp.s
-.PHONY : parser/Parser.cpp.s
-
-parser/tests/parserTest.o: parser/tests/parserTest.cpp.o
-
-.PHONY : parser/tests/parserTest.o
-
-# target to build an object file
-parser/tests/parserTest.cpp.o:
- $(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/parser/tests/parserTest.cpp.o
-.PHONY : parser/tests/parserTest.cpp.o
-
-parser/tests/parserTest.i: parser/tests/parserTest.cpp.i
-
-.PHONY : parser/tests/parserTest.i
-
-# target to preprocess a source file
-parser/tests/parserTest.cpp.i:
- $(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/parser/tests/parserTest.cpp.i
-.PHONY : parser/tests/parserTest.cpp.i
-
-parser/tests/parserTest.s: parser/tests/parserTest.cpp.s
-
-.PHONY : parser/tests/parserTest.s
-
-# target to generate assembly for a file
-parser/tests/parserTest.cpp.s:
- $(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/parser/tests/parserTest.cpp.s
-.PHONY : parser/tests/parserTest.cpp.s
-
-shared/Document.o: shared/Document.cpp.o
-
-.PHONY : shared/Document.o
-
-# target to build an object file
-shared/Document.cpp.o:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/shared/Document.cpp.o
-.PHONY : shared/Document.cpp.o
-
-shared/Document.i: shared/Document.cpp.i
-
-.PHONY : shared/Document.i
-
-# target to preprocess a source file
-shared/Document.cpp.i:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/shared/Document.cpp.i
-.PHONY : shared/Document.cpp.i
-
-shared/Document.s: shared/Document.cpp.s
-
-.PHONY : shared/Document.s
-
-# target to generate assembly for a file
-shared/Document.cpp.s:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/shared/Document.cpp.s
-.PHONY : shared/Document.cpp.s
-
-shared/ProducerConsumerQueue.o: shared/ProducerConsumerQueue.cpp.o
-
-.PHONY : shared/ProducerConsumerQueue.o
-
-# target to build an object file
-shared/ProducerConsumerQueue.cpp.o:
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/shared/ProducerConsumerQueue.cpp.o
-.PHONY : shared/ProducerConsumerQueue.cpp.o
-
-shared/ProducerConsumerQueue.i: shared/ProducerConsumerQueue.cpp.i
-
-.PHONY : shared/ProducerConsumerQueue.i
-
-# target to preprocess a source file
-shared/ProducerConsumerQueue.cpp.i:
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/shared/ProducerConsumerQueue.cpp.i
-.PHONY : shared/ProducerConsumerQueue.cpp.i
-
-shared/ProducerConsumerQueue.s: shared/ProducerConsumerQueue.cpp.s
-
-.PHONY : shared/ProducerConsumerQueue.s
-
-# target to generate assembly for a file
-shared/ProducerConsumerQueue.cpp.s:
- $(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/shared/ProducerConsumerQueue.cpp.s
-.PHONY : shared/ProducerConsumerQueue.cpp.s
-
-shared/urlTest.o: shared/urlTest.cpp.o
-
-.PHONY : shared/urlTest.o
-
-# target to build an object file
-shared/urlTest.cpp.o:
- $(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/shared/urlTest.cpp.o
-.PHONY : shared/urlTest.cpp.o
-
-shared/urlTest.i: shared/urlTest.cpp.i
-
-.PHONY : shared/urlTest.i
-
-# target to preprocess a source file
-shared/urlTest.cpp.i:
- $(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/shared/urlTest.cpp.i
-.PHONY : shared/urlTest.cpp.i
-
-shared/urlTest.s: shared/urlTest.cpp.s
-
-.PHONY : shared/urlTest.s
-
-# target to generate assembly for a file
-shared/urlTest.cpp.s:
- $(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/shared/urlTest.cpp.s
-.PHONY : shared/urlTest.cpp.s
-
-util/stringProcessing.o: util/stringProcessing.cpp.o
-
-.PHONY : util/stringProcessing.o
-
-# target to build an object file
-util/stringProcessing.cpp.o:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/stringProcessing.cpp.o
-.PHONY : util/stringProcessing.cpp.o
-
-util/stringProcessing.i: util/stringProcessing.cpp.i
-
-.PHONY : util/stringProcessing.i
-
-# target to preprocess a source file
-util/stringProcessing.cpp.i:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/stringProcessing.cpp.i
-.PHONY : util/stringProcessing.cpp.i
-
-util/stringProcessing.s: util/stringProcessing.cpp.s
-
-.PHONY : util/stringProcessing.s
-
-# target to generate assembly for a file
-util/stringProcessing.cpp.s:
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/stringProcessing.cpp.s
-.PHONY : util/stringProcessing.cpp.s
-
-util/tests/stemmerTest.o: util/tests/stemmerTest.cpp.o
-
-.PHONY : util/tests/stemmerTest.o
-
-# target to build an object file
-util/tests/stemmerTest.cpp.o:
- $(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/util/tests/stemmerTest.cpp.o
-.PHONY : util/tests/stemmerTest.cpp.o
-
-util/tests/stemmerTest.i: util/tests/stemmerTest.cpp.i
-
-.PHONY : util/tests/stemmerTest.i
-
-# target to preprocess a source file
-util/tests/stemmerTest.cpp.i:
- $(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/util/tests/stemmerTest.cpp.i
-.PHONY : util/tests/stemmerTest.cpp.i
-
-util/tests/stemmerTest.s: util/tests/stemmerTest.cpp.s
-
-.PHONY : util/tests/stemmerTest.s
-
-# target to generate assembly for a file
-util/tests/stemmerTest.cpp.s:
- $(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/util/tests/stemmerTest.cpp.s
-.PHONY : util/tests/stemmerTest.cpp.s
-
-util/tests/stringProcessingTest.o: util/tests/stringProcessingTest.cpp.o
-
-.PHONY : util/tests/stringProcessingTest.o
-
-# target to build an object file
-util/tests/stringProcessingTest.cpp.o:
- $(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/util/tests/stringProcessingTest.cpp.o
-.PHONY : util/tests/stringProcessingTest.cpp.o
-
-util/tests/stringProcessingTest.i: util/tests/stringProcessingTest.cpp.i
-
-.PHONY : util/tests/stringProcessingTest.i
-
-# target to preprocess a source file
-util/tests/stringProcessingTest.cpp.i:
- $(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/util/tests/stringProcessingTest.cpp.i
-.PHONY : util/tests/stringProcessingTest.cpp.i
-
-util/tests/stringProcessingTest.s: util/tests/stringProcessingTest.cpp.s
-
-.PHONY : util/tests/stringProcessingTest.s
-
-# target to generate assembly for a file
-util/tests/stringProcessingTest.cpp.s:
- $(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/util/tests/stringProcessingTest.cpp.s
-.PHONY : util/tests/stringProcessingTest.cpp.s
-
-util/tests/tokenizerTest.o: util/tests/tokenizerTest.cpp.o
-
-.PHONY : util/tests/tokenizerTest.o
-
-# target to build an object file
-util/tests/tokenizerTest.cpp.o:
- $(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/util/tests/tokenizerTest.cpp.o
-.PHONY : util/tests/tokenizerTest.cpp.o
-
-util/tests/tokenizerTest.i: util/tests/tokenizerTest.cpp.i
-
-.PHONY : util/tests/tokenizerTest.i
-
-# target to preprocess a source file
-util/tests/tokenizerTest.cpp.i:
- $(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/util/tests/tokenizerTest.cpp.i
-.PHONY : util/tests/tokenizerTest.cpp.i
-
-util/tests/tokenizerTest.s: util/tests/tokenizerTest.cpp.s
-
-.PHONY : util/tests/tokenizerTest.s
-
-# target to generate assembly for a file
-util/tests/tokenizerTest.cpp.s:
- $(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/util/tests/tokenizerTest.cpp.s
-.PHONY : util/tests/tokenizerTest.cpp.s
-
-util/util.o: util/util.cpp.o
-
-.PHONY : util/util.o
-
-# target to build an object file
-util/util.cpp.o:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/util/util.cpp.o
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/util.cpp.o
-.PHONY : util/util.cpp.o
-
-util/util.i: util/util.cpp.i
-
-.PHONY : util/util.i
-
-# target to preprocess a source file
-util/util.cpp.i:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/util/util.cpp.i
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/util.cpp.i
-.PHONY : util/util.cpp.i
-
-util/util.s: util/util.cpp.s
-
-.PHONY : util/util.s
-
-# target to generate assembly for a file
-util/util.cpp.s:
- $(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/util/util.cpp.s
- $(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/util.cpp.s
-.PHONY : util/util.cpp.s
-
-# Help Target
-help:
- @echo "The following are some of the valid targets for this Makefile:"
- @echo "... all (the default if no target is provided)"
- @echo "... clean"
- @echo "... depend"
- @echo "... edit_cache"
- @echo "... URLTEST"
- @echo "... ParserEndToEndTest"
- @echo "... rebuild_cache"
- @echo "... StemmerTest"
- @echo "... StringProcessingTest"
- @echo "... TokenizerTest"
- @echo "... test1"
- @echo "... crawl"
- @echo "... search"
- @echo "... crawler/SocketReader.o"
- @echo "... crawler/SocketReader.i"
- @echo "... crawler/SocketReader.s"
- @echo "... crawler/crawler.o"
- @echo "... crawler/crawler.i"
- @echo "... crawler/crawler.s"
- @echo "... crawler/spider.o"
- @echo "... crawler/spider.i"
- @echo "... crawler/spider.s"
- @echo "... main.o"
- @echo "... main.i"
- @echo "... main.s"
- @echo "... parser/Parser.o"
- @echo "... parser/Parser.i"
- @echo "... parser/Parser.s"
- @echo "... parser/tests/parserTest.o"
- @echo "... parser/tests/parserTest.i"
- @echo "... parser/tests/parserTest.s"
- @echo "... shared/Document.o"
- @echo "... shared/Document.i"
- @echo "... shared/Document.s"
- @echo "... shared/ProducerConsumerQueue.o"
- @echo "... shared/ProducerConsumerQueue.i"
- @echo "... shared/ProducerConsumerQueue.s"
- @echo "... shared/urlTest.o"
- @echo "... shared/urlTest.i"
- @echo "... shared/urlTest.s"
- @echo "... util/stringProcessing.o"
- @echo "... util/stringProcessing.i"
- @echo "... util/stringProcessing.s"
- @echo "... util/tests/stemmerTest.o"
- @echo "... util/tests/stemmerTest.i"
- @echo "... util/tests/stemmerTest.s"
- @echo "... util/tests/stringProcessingTest.o"
- @echo "... util/tests/stringProcessingTest.i"
- @echo "... util/tests/stringProcessingTest.s"
- @echo "... util/tests/tokenizerTest.o"
- @echo "... util/tests/tokenizerTest.i"
- @echo "... util/tests/tokenizerTest.s"
- @echo "... util/util.o"
- @echo "... util/util.i"
- @echo "... util/util.s"
-.PHONY : help
-
-
-
-#=============================================================================
-# Special targets to cleanup operation of make.
-
-# Special rule to run CMake to check the build system integrity.
-# No rule that depends on this can have commands that come from listfiles
-# because they might be regenerated.
-cmake_check_build_system:
- $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0
-.PHONY : cmake_check_build_system
-
diff --git a/crawler/Readers/HttpReader.cpp b/crawler/Readers/HttpReader.cpp
new file mode 100644
index 0000000..a1b8b91
--- /dev/null
+++ b/crawler/Readers/HttpReader.cpp
@@ -0,0 +1,53 @@
+//
+// Created by Ben Bergkamp on 3/13/18.
+//
+
+#include "HttpReader.h"
+
+
+void HttpReader::request()
+ {
+ sock = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
+ assert( sock != -1 );
+
+ // Get the host address.
+
+ struct hostent *host = gethostbyname( url.Host );
+ assert( host );
+
+ struct sockaddr_in address;
+ memset( &address, 0, sizeof( address ) );
+ address.sin_family = AF_INET;
+ address.sin_port = htons( 80 );
+ memcpy( &address.sin_addr, host->h_addr, host->h_length );
+
+ // Connect to the host.
+
+ int connectResult = connect( sock, ( struct sockaddr * )&address,
+ sizeof( address ) );
+ assert( connectResult == 0 );
+
+ // Send a GET message for the desired page.
+
+ cout << "Socket Reader is pulling from the web" << endl;
+
+ string getMessage = "GET ";
+ getMessage += url.CompleteUrl;
+ getMessage += " HTTP/1.1\r\nHost: ";
+ getMessage += url.Host;
+ getMessage += "\r\nConnection: close\r\n\r\n";
+
+ cout << getMessage << endl;
+ send( sock, getMessage.c_str( ), getMessage.length( ), 0 );
+
+ }
+
+bool HttpReader::fillBuffer(char * buf, size_t buf_size)
+ {
+ return (recv( sock, buf, buf_size, 0 ) == buf_size);
+ }
+
+void HttpReader::closeReader()
+ {
+ close( sock );
+ }
\ No newline at end of file
diff --git a/crawler/Readers/HttpReader.h b/crawler/Readers/HttpReader.h
new file mode 100644
index 0000000..788b744
--- /dev/null
+++ b/crawler/Readers/HttpReader.h
@@ -0,0 +1,24 @@
+//
+// Created by Ben Bergkamp on 3/13/18.
+//
+#pragma once
+
+#include "StreamReader.h"
+
+class HttpReader : public StreamReader
+ {
+public:
+
+ HttpReader( ParsedUrl url_in ) : url( url_in ) { }
+
+ void request();
+ bool fillBuffer(char * buf, size_t buf_size);
+ void closeReader();
+
+
+private:
+
+ ParsedUrl url;
+ int sock;
+
+ };
diff --git a/crawler/Readers/HttpsReader.cpp b/crawler/Readers/HttpsReader.cpp
new file mode 100644
index 0000000..e12196a
--- /dev/null
+++ b/crawler/Readers/HttpsReader.cpp
@@ -0,0 +1,71 @@
+//
+// Created by Ben Bergkamp on 3/13/18.
+//
+
+#include "HttpsReader.h"
+
+void HttpsReader::request()
+ {
+ struct hostent *host = gethostbyname( url.Host );
+ assert( host );
+
+ struct sockaddr_in address;
+ memset( &address, 0, sizeof( address ) );
+ address.sin_family = AF_INET;
+ address.sin_port = htons( 443 );
+ memcpy( &address.sin_addr, host->h_addr, host->h_length );
+
+ // Create a TCP/IP socket.
+
+ sock = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
+ assert( sock != -1 );
+
+ // Connect the socket to the host address.
+
+ int connectResult = connect( sock, ( struct sockaddr * )&address,
+ sizeof( address ) );
+ assert( connectResult == 0 );
+
+ // Build an SSL layer and set it to read/write
+ // to the socket we've connected.
+
+
+ SSL_library_init( );
+
+ ctx = SSL_CTX_new( SSLv23_method( ) );
+ assert( ctx );
+ ssl = SSL_new( ctx );
+ assert( ssl );
+
+ SSL_set_fd( ssl, sock );
+
+ // Establish an SSL connection.
+
+ int sslConnectResult = SSL_connect( ssl );
+ assert( sslConnectResult == 1 );
+
+ // Send a GET message for the desired page through the SSL.
+
+ string getMessage = "GET ";
+ getMessage += url.CompleteUrl;
+ getMessage += " HTTP/1.1\r\nHost: ";
+ getMessage += url.Host;
+ getMessage += "\r\nConnection: close\r\n\r\n";
+
+ cout << getMessage << endl;
+ SSL_write( ssl, getMessage.c_str( ), getMessage.length( ) );
+
+ }
+
+bool HttpsReader::fillBuffer(char * buf, size_t buf_size)
+ {
+ return (SSL_read( ssl, buf, buf_size ) == buf_size);
+ }
+
+void HttpsReader::closeReader()
+ {
+ SSL_shutdown( ssl );
+ SSL_free( ssl );
+ SSL_CTX_free( ctx );
+ close( sock );
+ }
diff --git a/crawler/Readers/HttpsReader.h b/crawler/Readers/HttpsReader.h
new file mode 100644
index 0000000..1cec38c
--- /dev/null
+++ b/crawler/Readers/HttpsReader.h
@@ -0,0 +1,25 @@
+//
+// Created by Ben Bergkamp on 3/13/18.
+//
+
+#pragma once
+
+#include "StreamReader.h"
+
+class HttpsReader : public StreamReader
+ {
+public:
+
+ HttpsReader( ParsedUrl url_in ) : url( url_in ) { }
+
+ void request();
+ bool fillBuffer(char * buf, size_t buf_size);
+ void closeReader();
+
+
+private:
+ ParsedUrl url;
+ int sock;
+ SSL * ssl;
+ SSL_CTX * ctx;
+ };
diff --git a/crawler/LocalReader.h b/crawler/Readers/LocalReader.h
similarity index 53%
rename from crawler/LocalReader.h
rename to crawler/Readers/LocalReader.h
index a58ec7d..9ed110e 100644
--- a/crawler/LocalReader.h
+++ b/crawler/Readers/LocalReader.h
@@ -2,20 +2,23 @@
// Created by Ben Bergkamp on 2/14/18.
//
-#ifndef EECS398_SEARCH_LOCALREADER_H
-#define EECS398_SEARCH_LOCALREADER_H
+#pragma once
#include "StreamReader.h"
class LocalReader : public StreamReader
{
string fileName;
- void fillBuffer(){
- strcpy(buffer, util::getFileMap( fileName )) ;
+ bool fillBuffer(char * buf, size_t buf_size){
+
+ //FIXME
+ strcpy(buf, util::getFileMap( fileName )) ;
+ return true;
+ //FIXME
+
}
public:
LocalReader( string url_in ) : fileName( url_in ) { }
};
-#endif //EECS398_SEARCH_LOCALREADER_H
diff --git a/crawler/Readers/StreamReader.h b/crawler/Readers/StreamReader.h
new file mode 100644
index 0000000..e5c17a3
--- /dev/null
+++ b/crawler/Readers/StreamReader.h
@@ -0,0 +1,55 @@
+//
+// Created by Jake Close on 2/13/18.
+//
+
+#pragma once
+
+#include "../../shared/url.h"
+#include "../../util/util.h"
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <iostream>
+#include <string.h>
+#include <string>
+#include <cassert>
+#include <openssl/ssl.h>
+
+
+
+class StreamReader
+ {
+public:
+ StreamReader() {};
+
+ virtual void request() = 0;
+ virtual bool fillBuffer(char * buf, size_t buf_size) = 0;
+ virtual void closeReader() = 0;
+
+ };
+
+/*
+StreamReader* SR_factory(ParsedUrl url, string mode)
+ {
+ string localFile;
+
+ StreamReader *newReader = nullptr
+ ;
+ if ( mode == "local" )
+ {
+ newReader = new LocalReader( url.CompleteUrl );
+ }
+ else if ( mode == "web" )
+ {
+ if(url.Service == "http") {
+ newReader = new HttpReader(url);
+ }
+ else if(url.Service == "https"){
+ newReader = new HttpsReader(url);
+ }
+ }
+
+ return newReader;
+ }
+ */
\ No newline at end of file
diff --git a/crawler/SocketReader.cpp b/crawler/SocketReader.cpp
deleted file mode 100644
index 56fe0c2..0000000
--- a/crawler/SocketReader.cpp
+++ /dev/null
@@ -1,185 +0,0 @@
-//
-// Created by Ben Bergkamp on 2/14/18.
-//
-
-#include "SocketReader.h"
-
-char * GetArbitrarySizeBuffer(SSL* ssl)
- {
-
- int buf_size = 10240;
- int current_size = buf_size;
- char* ssl_buffer = new char[buf_size];
- char* front = ssl_buffer;
- int bytes;
-
- while ( ( bytes = SSL_read( ssl, front, buf_size ) ) > 0 )
- {
-
- current_size += buf_size;
- char *temp = new char[current_size];
- strcpy(temp, ssl_buffer);
-
- front = temp + strlen(ssl_buffer);
- delete[] ssl_buffer;
- ssl_buffer = temp;
- }
-
- return ssl_buffer;
- }
-
-
-char * GetArbitrarySizeBuffer(int s )
- {
-
- int buf_size = 10240;
- int current_size = buf_size;
- char* http_buff = new char[buf_size];
- char* front = http_buff;
- int bytes;
-
- while ( ( bytes = recv( s, front, buf_size, 0 ) ) > 0 )
- {
-
- current_size += buf_size;
- char *temp = new char[current_size];
- strcpy(temp, http_buff);
-
- front = temp + strlen(http_buff);
- delete[] http_buff;
- http_buff = temp;
- }
-
- return http_buff;
- }
-
-
-
-
-void SocketReader::httpRequest()
- {
- int s = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
- assert( s != -1 );
-
- // Get the host address.
-
- struct hostent *host = gethostbyname( url.Host );
- assert( host );
-
- struct sockaddr_in address;
- memset( &address, 0, sizeof( address ) );
- address.sin_family = AF_INET;
- address.sin_port = htons( 80 );
- memcpy( &address.sin_addr, host->h_addr, host->h_length );
-
- // Connect to the host.
-
- int connectResult = connect( s, ( struct sockaddr * )&address,
- sizeof( address ) );
- assert( connectResult == 0 );
-
- // Send a GET message for the desired page.
-
- cout << "Socket Reader is pulling from the web" << endl;
-
- string getMessage = "GET ";
- getMessage += url.CompleteUrl;
- getMessage += " HTTP/1.1\r\nHost: ";
- getMessage += url.Host;
- getMessage += "\r\nConnection: close\r\n\r\n";
-
- cout << getMessage << endl;
- send( s, getMessage.c_str( ), getMessage.length( ), 0 );
-
- // Read from the socket until there's no more data.
-
- char HTTPbuffer[ 10240 ];
- int bytes;
-
-
- while ( ( bytes = recv( s, buffer, sizeof( buffer ), 0 ) ) > 0 )
- write( 1, buffer, bytes );
-
- buffer = GetArbitrarySizeBuffer(s);
-
- close( s );
- return;
- }
-
-
-void SocketReader::httpsRequest(){
-
- struct hostent *host = gethostbyname( url.Host );
- assert( host );
-
- struct sockaddr_in address;
- memset( &address, 0, sizeof( address ) );
- address.sin_family = AF_INET;
- address.sin_port = htons( 443 );
- memcpy( &address.sin_addr, host->h_addr, host->h_length );
-
- // Create a TCP/IP socket.
-
- int s = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
- assert( s != -1 );
-
- // Connect the socket to the host address.
-
- int connectResult = connect( s, ( struct sockaddr * )&address,
- sizeof( address ) );
- assert( connectResult == 0 );
-
- // Build an SSL layer and set it to read/write
- // to the socket we've connected.
-
-
- SSL_library_init( );
-
- SSL_CTX *ctx = SSL_CTX_new( SSLv23_method( ) );
- assert( ctx );
- SSL *ssl = SSL_new( ctx );
- assert( ssl );
-
- SSL_set_fd( ssl, s );
-
- // Establish an SSL connection.
-
- int sslConnectResult = SSL_connect( ssl );
- assert( sslConnectResult == 1 );
-
- // Send a GET message for the desired page through the SSL.
-
- string getMessage = "GET ";
- getMessage += url.CompleteUrl;
- getMessage += " HTTP/1.1\r\nHost: ";
- getMessage += url.Host;
- getMessage += "\r\nConnection: close\r\n\r\n";
-
- cout << getMessage << endl;
- SSL_write( ssl, getMessage.c_str( ), getMessage.length( ) );
-
- // Read from the SSL until there's no more data.
-
- buffer = GetArbitrarySizeBuffer(ssl);
-
- SSL_shutdown( ssl );
- SSL_free( ssl );
- SSL_CTX_free( ctx );
- close( s );
-
-
- }
-
-
-
-
-
-void SocketReader::fillBuffer( )
- {
- if ( !strcmp(url.Service , "http" ) )
- httpRequest();
- else
- httpsRequest();
-
-
- }
\ No newline at end of file
diff --git a/crawler/SocketReader.h b/crawler/SocketReader.h
deleted file mode 100644
index 9c36a0e..0000000
--- a/crawler/SocketReader.h
+++ /dev/null
@@ -1,27 +0,0 @@
-//
-// Created by Ben Bergkamp on 2/14/18.
-//
-
-#ifndef EECS398_SEARCH_SOCKETREADER_H
-#define EECS398_SEARCH_SOCKETREADER_H
-
-#include "StreamReader.h"
-
-
-class SocketReader : public StreamReader
- {
-public:
- SocketReader( ParsedUrl url_in ) : url( url_in ) { }
- virtual void fillBuffer();
- void httpRequest();
- void httpsRequest();
-
-
- //virtual void fillBuffer(char ssl);
-
-private:
- ParsedUrl url;
- };
-
-
-#endif //EECS398_SEARCH_SOCKETREADER_H
diff --git a/crawler/StreamReader.h b/crawler/StreamReader.h
deleted file mode 100644
index 2b089ef..0000000
--- a/crawler/StreamReader.h
+++ /dev/null
@@ -1,30 +0,0 @@
-//
-// Created by Jake Close on 2/13/18.
-//
-
-#pragma once
-
-#include "../shared/url.h"
-#include "../util/util.h"
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <iostream>
-#include <string.h>
-#include <string>
-#include <cassert>
-#include <openssl/ssl.h>
-
-
-
-class StreamReader
- {
-public:
- StreamReader() {};
- virtual void fillBuffer() = 0;
- virtual void close() = 0;
- virtual void request();
- char *buffer;
-
- };
\ No newline at end of file
diff --git a/crawler/spider.cpp b/crawler/spider.cpp
index b87b4b5..b1316ec 100644
--- a/crawler/spider.cpp
+++ b/crawler/spider.cpp
@@ -10,15 +10,10 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
-
-#include <unistd.h>
-
-#include "LocalReader.h"
-#include "SocketReader.h"
#include "../shared/Document.h"
#include "../parser/Parser.h"
-size_t Spider::hash(const char * s){
+size_t Spider::hash(const char * s)
{
// http://www.cse.yorku.ca/~oz/hash.html
size_t h = 5381;
@@ -27,7 +22,6 @@ size_t Spider::hash(const char * s){
h = ((h << 5) + h) + c;
return h;
}
- }
string Spider::getUrl()
@@ -57,16 +51,21 @@ void Spider::FuncToRun()
if(cond)
{
+ //StreamReader *reader = SR_factory( currentUrl, this->mode );
- StreamReader *reader = request( currentUrl );
- size_t docID = hash(currentUrl.CompleteUrl);
- string pathToDisk = util::GetCurrentWorkingDir() + "/crawlerOutput/" + to_string(docID)+ ".txt";
- int fd = util::writeToNewFileToLocation( reader->buffer, pathToDisk);
+ /*
+ OLD VERSION
+ size_t docID = hash(currentUrl.CompleteUrl);
+ string pathToDisk = util::GetCurrentWorkingDir() + "/crawlerOutput/" + to_string(docID)+ ".txt";
+ int fd = util::writeToNewFileToLocation( reader->buffer, pathToDisk);
Document document ( currentUrl, reader->buffer );
auto dictionary = parser.execute ( &document );
+ */
+
+
cond = true;
}
@@ -138,7 +137,7 @@ bool Spider::shouldURLbeCrawled( ParsedUrl url )
return false;
}
-
+/*
//check if path in url is in the robots txt
bool Spider::checkRobots(ParsedUrl url)
{
@@ -181,32 +180,8 @@ int Spider::getRobots(ParsedUrl url )
- };
-
-/*
-returns true if fileMap was created, otherwise false
- Modifies the filemap to be a char* of the file of the url passed
-*/
-
-// make this become a stream reader factory
-StreamReader* Spider::request( ParsedUrl url )
- {
- string localFile;
-
- StreamReader *newReader;
- if ( this->mode == "local" )
- {
- newReader = new LocalReader( url.CompleteUrl );
- }
- else if ( this->mode == "web" )
- {
- newReader = new SocketReader( url );
- }
-
- //remove fill buffer/ change to get request
- newReader->fillBuffer( );
- return newReader;
}
+*/
//request function that handles sending over get request via socket or trying to open file
diff --git a/crawler/spider.h b/crawler/spider.h
index 6b589bf..4b3900c 100644
--- a/crawler/spider.h
+++ b/crawler/spider.h
@@ -8,7 +8,7 @@
#include "../shared/ThreadClass.h"
#include<iostream>
#include <unordered_map>
-#include "StreamReader.h"
+#include "Readers/StreamReader.h"
#include "../util/util.h"
#include "../parser/Parser.h"
@@ -35,16 +35,11 @@ public:
virtual void FuncToRun();
- //Makes request to given url
- // if successful, writes file to disk, stores location to memeber value
- // else return false and error information, retry if necessary
- StreamReader *request( ParsedUrl url );
-
bool writeDocToDisk(ParsedUrl url);
bool shouldURLbeCrawled( ParsedUrl URL );
size_t hash(const char * s);
- int getRobots(ParsedUrl url );
+ //int getRobots(ParsedUrl url );
bool checkRobots(ParsedUrl url);
--
GitLab