Skip to content
Snippets Groups Projects
Commit 82eb00c1 authored by benbergk's avatar benbergk
Browse files

initial refactored commit

parent ac7893d4
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 250 additions and 1449 deletions
CMakeLists.txt deleted 100644 → 0
+ 0
− 62
View file @ ac7893d4
cmake_minimum_required(VERSION 3.9)
project(eecs398_search)
set(CMAKE_CXX_STANDARD 11)
add_executable(parser-test
main.cpp
shared/ProducerConsumerQueue.cpp
shared/ProducerConsumerQueue.h
parser/Parser.h
util/Tokenizer.h
util/stringProcessing.h
util/Stemmer.h)
add_executable(crawl
main.cpp
shared/ProducerConsumerQueue.h
shared/ThreadClass.h
shared/url.h
crawler/crawler.cpp
crawler/SocketReader.cpp
crawler/StreamReader.h
crawler/spider.cpp
util/util.cpp
crawler/LocalReader.h
shared/Document.cpp
parser/Parser.cpp
util/stringProcessing.cpp indexStreamReader/ISR.cpp indexStreamReader/ISR.h)
add_executable(StringProcessingTest
util/stringProcessing.h
util/Stemmer.h
util/tests/stringProcessingTest.cpp)
add_executable(TokenizerTest
util/Tokenizer.h
util/tests/tokenizerTest.cpp)
add_executable(StemmerTest
util/Stemmer.h
util/tests/stemmerTest.cpp)
add_executable(ParserEndToEndTest
parser/Parser.h
shared/Document.h
shared/url.h
parser/tests/parserTest.cpp)
add_executable(URLTEST shared/url.h shared/urlTest.cpp)
add_executable(search-engine search.cpp query/Query.cpp)
find_package(OpenSSL REQUIRED)
target_link_libraries(crawl OpenSSL::SSL pthread)
cmake-build-debug/CMakeFiles/Makefile.cmake deleted 100644 → 0
+ 0
− 64
View file @ ac7893d4
# CMAKE generated file: DO NOT EDIT!
# Generated by "Unix Makefiles" Generator, CMake Version 3.9
# The generator used is:
set(CMAKE_DEPENDS_GENERATOR "Unix Makefiles")
# The top level Makefile was generated from the following files:
set(CMAKE_MAKEFILE_DEPENDS
"CMakeCache.txt"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeCInformation.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeCXXInformation.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeCommonLanguageInclude.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeExtraGeneratorDetermineCompilerMacrosAndIncludeDirs.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeFindCodeBlocks.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeGenericSystem.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeLanguageInformation.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeSystemSpecificInformation.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/CMakeSystemSpecificInitialize.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/AppleClang-C.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/AppleClang-CXX.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/CMakeCommonCompilerMacros.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/Clang.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Compiler/GNU.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindOpenSSL.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindPackageHandleStandardArgs.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindPackageMessage.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/FindPkgConfig.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-AppleClang-C.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-AppleClang-CXX.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Clang-C.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Clang-CXX.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Clang.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin-Initialize.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/Darwin.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/Platform/UnixPaths.cmake"
"/Applications/CLion.app/Contents/bin/cmake/share/cmake-3.9/Modules/ProcessorCount.cmake"
"../CMakeLists.txt"
"CMakeFiles/3.9.6/CMakeCCompiler.cmake"
"CMakeFiles/3.9.6/CMakeCXXCompiler.cmake"
"CMakeFiles/3.9.6/CMakeSystem.cmake"
)
# The corresponding makefile is:
set(CMAKE_MAKEFILE_OUTPUTS
"Makefile"
"CMakeFiles/cmake.check_cache"
)
# Byproducts of CMake generate step:
set(CMAKE_MAKEFILE_PRODUCTS
"CMakeFiles/CMakeDirectoryInformation.cmake"
)
# Dependency information for all targets:
set(CMAKE_DEPEND_INFO_FILES
"CMakeFiles/URLTEST.dir/DependInfo.cmake"
"CMakeFiles/ParserEndToEndTest.dir/DependInfo.cmake"
"CMakeFiles/StemmerTest.dir/DependInfo.cmake"
"CMakeFiles/StringProcessingTest.dir/DependInfo.cmake"
"CMakeFiles/TokenizerTest.dir/DependInfo.cmake"
"CMakeFiles/test1.dir/DependInfo.cmake"
"CMakeFiles/crawl.dir/DependInfo.cmake"
"CMakeFiles/search.dir/DependInfo.cmake"
)
cmake-build-debug/CMakeFiles/Makefile2 deleted 100644 → 0
+ 0
− 367
View file @ ac7893d4
# CMAKE generated file: DO NOT EDIT!
# Generated by "Unix Makefiles" Generator, CMake Version 3.9
# Default target executed when no arguments are given to make.
default_target: all
.PHONY : default_target
# The main recursive all target
all:
.PHONY : all
# The main recursive preinstall target
preinstall:
.PHONY : preinstall
#=============================================================================
# Special targets provided by cmake.
# Disable implicit rules so canonical targets will work.
.SUFFIXES:
# Remove some rules from gmake that .SUFFIXES does not remove.
SUFFIXES =
.SUFFIXES: .hpux_make_needs_suffix_list
# Suppress display of executed commands.
$(VERBOSE).SILENT:
# A target that is always out of date.
cmake_force:
.PHONY : cmake_force
#=============================================================================
# Set environment variables for the build.
# The shell in which to execute make rules.
SHELL = /bin/sh
# The CMake executable.
CMAKE_COMMAND = /Applications/CLion.app/Contents/bin/cmake/bin/cmake
# The command to remove a file.
RM = /Applications/CLion.app/Contents/bin/cmake/bin/cmake -E remove -f
# Escaping for special characters.
EQUALS = =
# The top-level source directory on which CMake was run.
CMAKE_SOURCE_DIR = /Users/benbergkamp/Desktop/398/eecs398-search
# The top-level build directory on which CMake was run.
CMAKE_BINARY_DIR = /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug
#=============================================================================
# Target rules for target CMakeFiles/URLTEST.dir
# All Build rule for target.
CMakeFiles/URLTEST.dir/all:
$(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/depend
$(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=9,10 "Built target URLTEST"
.PHONY : CMakeFiles/URLTEST.dir/all
# Include target in all.
all: CMakeFiles/URLTEST.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/URLTEST.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/URLTEST.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/URLTEST.dir/rule
# Convenience name for target.
URLTEST: CMakeFiles/URLTEST.dir/rule
.PHONY : URLTEST
# clean rule for target.
CMakeFiles/URLTEST.dir/clean:
$(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/clean
.PHONY : CMakeFiles/URLTEST.dir/clean
# clean rule for target.
clean: CMakeFiles/URLTEST.dir/clean
.PHONY : clean
#=============================================================================
# Target rules for target CMakeFiles/ParserEndToEndTest.dir
# All Build rule for target.
CMakeFiles/ParserEndToEndTest.dir/all:
$(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/depend
$(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=1,2 "Built target ParserEndToEndTest"
.PHONY : CMakeFiles/ParserEndToEndTest.dir/all
# Include target in all.
all: CMakeFiles/ParserEndToEndTest.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/ParserEndToEndTest.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/ParserEndToEndTest.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/ParserEndToEndTest.dir/rule
# Convenience name for target.
ParserEndToEndTest: CMakeFiles/ParserEndToEndTest.dir/rule
.PHONY : ParserEndToEndTest
# clean rule for target.
CMakeFiles/ParserEndToEndTest.dir/clean:
$(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/clean
.PHONY : CMakeFiles/ParserEndToEndTest.dir/clean
# clean rule for target.
clean: CMakeFiles/ParserEndToEndTest.dir/clean
.PHONY : clean
#=============================================================================
# Target rules for target CMakeFiles/StemmerTest.dir
# All Build rule for target.
CMakeFiles/StemmerTest.dir/all:
$(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/depend
$(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=3,4 "Built target StemmerTest"
.PHONY : CMakeFiles/StemmerTest.dir/all
# Include target in all.
all: CMakeFiles/StemmerTest.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/StemmerTest.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/StemmerTest.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/StemmerTest.dir/rule
# Convenience name for target.
StemmerTest: CMakeFiles/StemmerTest.dir/rule
.PHONY : StemmerTest
# clean rule for target.
CMakeFiles/StemmerTest.dir/clean:
$(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/clean
.PHONY : CMakeFiles/StemmerTest.dir/clean
# clean rule for target.
clean: CMakeFiles/StemmerTest.dir/clean
.PHONY : clean
#=============================================================================
# Target rules for target CMakeFiles/StringProcessingTest.dir
# All Build rule for target.
CMakeFiles/StringProcessingTest.dir/all:
$(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/depend
$(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=5,6 "Built target StringProcessingTest"
.PHONY : CMakeFiles/StringProcessingTest.dir/all
# Include target in all.
all: CMakeFiles/StringProcessingTest.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/StringProcessingTest.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/StringProcessingTest.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/StringProcessingTest.dir/rule
# Convenience name for target.
StringProcessingTest: CMakeFiles/StringProcessingTest.dir/rule
.PHONY : StringProcessingTest
# clean rule for target.
CMakeFiles/StringProcessingTest.dir/clean:
$(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/clean
.PHONY : CMakeFiles/StringProcessingTest.dir/clean
# clean rule for target.
clean: CMakeFiles/StringProcessingTest.dir/clean
.PHONY : clean
#=============================================================================
# Target rules for target CMakeFiles/TokenizerTest.dir
# All Build rule for target.
CMakeFiles/TokenizerTest.dir/all:
$(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/depend
$(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=7,8 "Built target TokenizerTest"
.PHONY : CMakeFiles/TokenizerTest.dir/all
# Include target in all.
all: CMakeFiles/TokenizerTest.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/TokenizerTest.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 2
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/TokenizerTest.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/TokenizerTest.dir/rule
# Convenience name for target.
TokenizerTest: CMakeFiles/TokenizerTest.dir/rule
.PHONY : TokenizerTest
# clean rule for target.
CMakeFiles/TokenizerTest.dir/clean:
$(MAKE) -f CMakeFiles/TokenizerTest.dir/build.make CMakeFiles/TokenizerTest.dir/clean
.PHONY : CMakeFiles/TokenizerTest.dir/clean
# clean rule for target.
clean: CMakeFiles/TokenizerTest.dir/clean
.PHONY : clean
#=============================================================================
# Target rules for target CMakeFiles/test1.dir
# All Build rule for target.
CMakeFiles/test1.dir/all:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/depend
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=23,24,25,26,27,28 "Built target test1"
.PHONY : CMakeFiles/test1.dir/all
# Include target in all.
all: CMakeFiles/test1.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/test1.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 6
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/test1.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/test1.dir/rule
# Convenience name for target.
test1: CMakeFiles/test1.dir/rule
.PHONY : test1
# clean rule for target.
CMakeFiles/test1.dir/clean:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/clean
.PHONY : CMakeFiles/test1.dir/clean
# clean rule for target.
clean: CMakeFiles/test1.dir/clean
.PHONY : clean
#=============================================================================
# Target rules for target CMakeFiles/crawl.dir
# All Build rule for target.
CMakeFiles/crawl.dir/all:
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/depend
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=11,12,13,14,15,16,17,18,19 "Built target crawl"
.PHONY : CMakeFiles/crawl.dir/all
# Include target in all.
all: CMakeFiles/crawl.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/crawl.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 9
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/crawl.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/crawl.dir/rule
# Convenience name for target.
crawl: CMakeFiles/crawl.dir/rule
.PHONY : crawl
# clean rule for target.
CMakeFiles/crawl.dir/clean:
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/clean
.PHONY : CMakeFiles/crawl.dir/clean
# clean rule for target.
clean: CMakeFiles/crawl.dir/clean
.PHONY : clean
#=============================================================================
# Target rules for target CMakeFiles/search.dir
# All Build rule for target.
CMakeFiles/search.dir/all:
$(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/depend
$(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/build
@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --progress-dir=/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles --progress-num=20,21,22 "Built target search"
.PHONY : CMakeFiles/search.dir/all
# Include target in all.
all: CMakeFiles/search.dir/all
.PHONY : all
# Build rule for subdir invocation for target.
CMakeFiles/search.dir/rule: cmake_check_build_system
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 3
$(MAKE) -f CMakeFiles/Makefile2 CMakeFiles/search.dir/all
$(CMAKE_COMMAND) -E cmake_progress_start /Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles 0
.PHONY : CMakeFiles/search.dir/rule
# Convenience name for target.
search: CMakeFiles/search.dir/rule
.PHONY : search
# clean rule for target.
CMakeFiles/search.dir/clean:
$(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/clean
.PHONY : CMakeFiles/search.dir/clean
# clean rule for target.
clean: CMakeFiles/search.dir/clean
.PHONY : clean
#=============================================================================
# Special targets to cleanup operation of make.
# Special rule to run CMake to check the build system integrity.
# No rule that depends on this can have commands that come from listfiles
# because they might be regenerated.
cmake_check_build_system:
$(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0
.PHONY : cmake_check_build_system
cmake-build-debug/CMakeFiles/TargetDirectories.txt deleted 100644 → 0
+ 0
− 10
View file @ ac7893d4
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/edit_cache.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/URLTEST.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/ParserEndToEndTest.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/rebuild_cache.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/StemmerTest.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/StringProcessingTest.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/TokenizerTest.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/test1.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/crawl.dir
/Users/benbergkamp/Desktop/398/eecs398-search/cmake-build-debug/CMakeFiles/search.dir
cmake-build-debug/CMakeFiles/progress.marks deleted 100644 → 0
+ 0
− 1
View file @ ac7893d4
28
cmake-build-debug/Makefile deleted 100644 → 0
+ 0
− 684
View file @ ac7893d4
This diff is collapsed.
crawler/Readers/HttpReader.cpp 0 → 100644
+ 53
− 0
View file @ 82eb00c1
//
// Created by Ben Bergkamp on 3/13/18.
//
#include "HttpReader.h"
void HttpReader::request()
{
sock = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
assert( sock != -1 );
// Get the host address.
struct hostent *host = gethostbyname( url.Host );
assert( host );
struct sockaddr_in address;
memset( &address, 0, sizeof( address ) );
address.sin_family = AF_INET;
address.sin_port = htons( 80 );
memcpy( &address.sin_addr, host->h_addr, host->h_length );
// Connect to the host.
int connectResult = connect( sock, ( struct sockaddr * )&address,
sizeof( address ) );
assert( connectResult == 0 );
// Send a GET message for the desired page.
cout << "Socket Reader is pulling from the web" << endl;
string getMessage = "GET ";
getMessage += url.CompleteUrl;
getMessage += " HTTP/1.1\r\nHost: ";
getMessage += url.Host;
getMessage += "\r\nConnection: close\r\n\r\n";
cout << getMessage << endl;
send( sock, getMessage.c_str( ), getMessage.length( ), 0 );
}
bool HttpReader::fillBuffer(char * buf, size_t buf_size)
{
return (recv( sock, buf, buf_size, 0 ) == buf_size);
}
void HttpReader::closeReader()
{
close( sock );
}
\ No newline at end of file
crawler/Readers/HttpReader.h 0 → 100644
+ 24
− 0
View file @ 82eb00c1
//
// Created by Ben Bergkamp on 3/13/18.
//
#pragma once
#include "StreamReader.h"
class HttpReader : public StreamReader
{
public:
HttpReader( ParsedUrl url_in ) : url( url_in ) { }
void request();
bool fillBuffer(char * buf, size_t buf_size);
void closeReader();
private:
ParsedUrl url;
int sock;
};
crawler/Readers/HttpsReader.cpp 0 → 100644
+ 71
− 0
View file @ 82eb00c1
//
// Created by Ben Bergkamp on 3/13/18.
//
#include "HttpsReader.h"
void HttpsReader::request()
{
struct hostent *host = gethostbyname( url.Host );
assert( host );
struct sockaddr_in address;
memset( &address, 0, sizeof( address ) );
address.sin_family = AF_INET;
address.sin_port = htons( 443 );
memcpy( &address.sin_addr, host->h_addr, host->h_length );
// Create a TCP/IP socket.
sock = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
assert( sock != -1 );
// Connect the socket to the host address.
int connectResult = connect( sock, ( struct sockaddr * )&address,
sizeof( address ) );
assert( connectResult == 0 );
// Build an SSL layer and set it to read/write
// to the socket we've connected.
SSL_library_init( );
ctx = SSL_CTX_new( SSLv23_method( ) );
assert( ctx );
ssl = SSL_new( ctx );
assert( ssl );
SSL_set_fd( ssl, sock );
// Establish an SSL connection.
int sslConnectResult = SSL_connect( ssl );
assert( sslConnectResult == 1 );
// Send a GET message for the desired page through the SSL.
string getMessage = "GET ";
getMessage += url.CompleteUrl;
getMessage += " HTTP/1.1\r\nHost: ";
getMessage += url.Host;
getMessage += "\r\nConnection: close\r\n\r\n";
cout << getMessage << endl;
SSL_write( ssl, getMessage.c_str( ), getMessage.length( ) );
}
bool HttpsReader::fillBuffer(char * buf, size_t buf_size)
{
return (SSL_read( ssl, buf, buf_size ) == buf_size);
}
void HttpsReader::closeReader()
{
SSL_shutdown( ssl );
SSL_free( ssl );
SSL_CTX_free( ctx );
close( sock );
}
crawler/Readers/HttpsReader.h 0 → 100644
+ 25
− 0
View file @ 82eb00c1
//
// Created by Ben Bergkamp on 3/13/18.
//
#pragma once
#include "StreamReader.h"
class HttpsReader : public StreamReader
{
public:
HttpsReader( ParsedUrl url_in ) : url( url_in ) { }
void request();
bool fillBuffer(char * buf, size_t buf_size);
void closeReader();
private:
ParsedUrl url;
int sock;
SSL * ssl;
SSL_CTX * ctx;
};
crawler/LocalReader.h → crawler/Readers/LocalReader.h
+ 8
− 5
View file @ 82eb00c1
...@@ -2,20 +2,23 @@ ...@@ -2,20 +2,23 @@
// Created by Ben Bergkamp on 2/14/18. // Created by Ben Bergkamp on 2/14/18.
// //
#ifndef EECS398_SEARCH_LOCALREADER_H #pragma once
#define EECS398_SEARCH_LOCALREADER_H
#include "StreamReader.h" #include "StreamReader.h"
class LocalReader : public StreamReader class LocalReader : public StreamReader
{ {
string fileName; string fileName;
void fillBuffer(){ bool fillBuffer(char * buf, size_t buf_size){
strcpy(buffer, util::getFileMap( fileName )) ;
//FIXME
strcpy(buf, util::getFileMap( fileName )) ;
return true;
//FIXME
} }
public: public:
LocalReader( string url_in ) : fileName( url_in ) { } LocalReader( string url_in ) : fileName( url_in ) { }
}; };
#endif //EECS398_SEARCH_LOCALREADER_H
crawler/StreamReader.h → crawler/Readers/StreamReader.h
+ 55
− 0
View file @ 82eb00c1
...@@ -4,8 +4,8 @@ ...@@ -4,8 +4,8 @@
#pragma once #pragma once
#include "../shared/url.h" #include "../../shared/url.h"
#include "../util/util.h" #include "../../util/util.h"
#include <unistd.h> #include <unistd.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/socket.h> #include <sys/socket.h>
...@@ -22,9 +22,34 @@ class StreamReader ...@@ -22,9 +22,34 @@ class StreamReader
{ {
public: public:
StreamReader() {}; StreamReader() {};
virtual void fillBuffer() = 0;
virtual void close() = 0;
virtual void request();
char *buffer;
}; virtual void request() = 0;
\ No newline at end of file virtual bool fillBuffer(char * buf, size_t buf_size) = 0;
virtual void closeReader() = 0;
};
/*
StreamReader* SR_factory(ParsedUrl url, string mode)
{
string localFile;
StreamReader *newReader = nullptr
;
if ( mode == "local" )
{
newReader = new LocalReader( url.CompleteUrl );
}
else if ( mode == "web" )
{
if(url.Service == "http") {
newReader = new HttpReader(url);
}
else if(url.Service == "https"){
newReader = new HttpsReader(url);
}
}
return newReader;
}
*/
\ No newline at end of file
crawler/SocketReader.cpp deleted 100644 → 0
+ 0
− 185
View file @ ac7893d4
//
// Created by Ben Bergkamp on 2/14/18.
//
#include "SocketReader.h"
char * GetArbitrarySizeBuffer(SSL* ssl)
{
int buf_size = 10240;
int current_size = buf_size;
char* ssl_buffer = new char[buf_size];
char* front = ssl_buffer;
int bytes;
while ( ( bytes = SSL_read( ssl, front, buf_size ) ) > 0 )
{
current_size += buf_size;
char *temp = new char[current_size];
strcpy(temp, ssl_buffer);
front = temp + strlen(ssl_buffer);
delete[] ssl_buffer;
ssl_buffer = temp;
}
return ssl_buffer;
}
char * GetArbitrarySizeBuffer(int s )
{
int buf_size = 10240;
int current_size = buf_size;
char* http_buff = new char[buf_size];
char* front = http_buff;
int bytes;
while ( ( bytes = recv( s, front, buf_size, 0 ) ) > 0 )
{
current_size += buf_size;
char *temp = new char[current_size];
strcpy(temp, http_buff);
front = temp + strlen(http_buff);
delete[] http_buff;
http_buff = temp;
}
return http_buff;
}
void SocketReader::httpRequest()
{
int s = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
assert( s != -1 );
// Get the host address.
struct hostent *host = gethostbyname( url.Host );
assert( host );
struct sockaddr_in address;
memset( &address, 0, sizeof( address ) );
address.sin_family = AF_INET;
address.sin_port = htons( 80 );
memcpy( &address.sin_addr, host->h_addr, host->h_length );
// Connect to the host.
int connectResult = connect( s, ( struct sockaddr * )&address,
sizeof( address ) );
assert( connectResult == 0 );
// Send a GET message for the desired page.
cout << "Socket Reader is pulling from the web" << endl;
string getMessage = "GET ";
getMessage += url.CompleteUrl;
getMessage += " HTTP/1.1\r\nHost: ";
getMessage += url.Host;
getMessage += "\r\nConnection: close\r\n\r\n";
cout << getMessage << endl;
send( s, getMessage.c_str( ), getMessage.length( ), 0 );
// Read from the socket until there's no more data.
char HTTPbuffer[ 10240 ];
int bytes;
while ( ( bytes = recv( s, buffer, sizeof( buffer ), 0 ) ) > 0 )
write( 1, buffer, bytes );
buffer = GetArbitrarySizeBuffer(s);
close( s );
return;
}
void SocketReader::httpsRequest(){
struct hostent *host = gethostbyname( url.Host );
assert( host );
struct sockaddr_in address;
memset( &address, 0, sizeof( address ) );
address.sin_family = AF_INET;
address.sin_port = htons( 443 );
memcpy( &address.sin_addr, host->h_addr, host->h_length );
// Create a TCP/IP socket.
int s = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
assert( s != -1 );
// Connect the socket to the host address.
int connectResult = connect( s, ( struct sockaddr * )&address,
sizeof( address ) );
assert( connectResult == 0 );
// Build an SSL layer and set it to read/write
// to the socket we've connected.
SSL_library_init( );
SSL_CTX *ctx = SSL_CTX_new( SSLv23_method( ) );
assert( ctx );
SSL *ssl = SSL_new( ctx );
assert( ssl );
SSL_set_fd( ssl, s );
// Establish an SSL connection.
int sslConnectResult = SSL_connect( ssl );
assert( sslConnectResult == 1 );
// Send a GET message for the desired page through the SSL.
string getMessage = "GET ";
getMessage += url.CompleteUrl;
getMessage += " HTTP/1.1\r\nHost: ";
getMessage += url.Host;
getMessage += "\r\nConnection: close\r\n\r\n";
cout << getMessage << endl;
SSL_write( ssl, getMessage.c_str( ), getMessage.length( ) );
// Read from the SSL until there's no more data.
buffer = GetArbitrarySizeBuffer(ssl);
SSL_shutdown( ssl );
SSL_free( ssl );
SSL_CTX_free( ctx );
close( s );
}
void SocketReader::fillBuffer( )
{
if ( !strcmp(url.Service , "http" ) )
httpRequest();
else
httpsRequest();
}
\ No newline at end of file
crawler/SocketReader.h deleted 100644 → 0
+ 0
− 27
View file @ ac7893d4
//
// Created by Ben Bergkamp on 2/14/18.
//
#ifndef EECS398_SEARCH_SOCKETREADER_H
#define EECS398_SEARCH_SOCKETREADER_H
#include "StreamReader.h"
class SocketReader : public StreamReader
{
public:
SocketReader( ParsedUrl url_in ) : url( url_in ) { }
virtual void fillBuffer();
void httpRequest();
void httpsRequest();
//virtual void fillBuffer(char ssl);
private:
ParsedUrl url;
};
#endif //EECS398_SEARCH_SOCKETREADER_H
crawler/spider.cpp
+ 12
− 37
View file @ 82eb00c1
...@@ -10,15 +10,10 @@ ...@@ -10,15 +10,10 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <fcntl.h> #include <fcntl.h>
#include <unistd.h> #include <unistd.h>
#include <unistd.h>
#include "LocalReader.h"
#include "SocketReader.h"
#include "../shared/Document.h" #include "../shared/Document.h"
#include "../parser/Parser.h" #include "../parser/Parser.h"
size_t Spider::hash(const char * s){ size_t Spider::hash(const char * s)
{ {
// http://www.cse.yorku.ca/~oz/hash.html // http://www.cse.yorku.ca/~oz/hash.html
size_t h = 5381; size_t h = 5381;
...@@ -27,7 +22,6 @@ size_t Spider::hash(const char * s){ ...@@ -27,7 +22,6 @@ size_t Spider::hash(const char * s){
h = ((h << 5) + h) + c; h = ((h << 5) + h) + c;
return h; return h;
} }
}
string Spider::getUrl() string Spider::getUrl()
...@@ -57,16 +51,21 @@ void Spider::FuncToRun() ...@@ -57,16 +51,21 @@ void Spider::FuncToRun()
if(cond) if(cond)
{ {
//StreamReader *reader = SR_factory( currentUrl, this->mode );
StreamReader *reader = request( currentUrl );
size_t docID = hash(currentUrl.CompleteUrl);
string pathToDisk = util::GetCurrentWorkingDir() + "/crawlerOutput/" + to_string(docID)+ ".txt";
int fd = util::writeToNewFileToLocation( reader->buffer, pathToDisk);
/*
OLD VERSION
size_t docID = hash(currentUrl.CompleteUrl);
string pathToDisk = util::GetCurrentWorkingDir() + "/crawlerOutput/" + to_string(docID)+ ".txt";
int fd = util::writeToNewFileToLocation( reader->buffer, pathToDisk);
Document document ( currentUrl, reader->buffer ); Document document ( currentUrl, reader->buffer );
auto dictionary = parser.execute ( &document ); auto dictionary = parser.execute ( &document );
*/
cond = true; cond = true;
} }
...@@ -138,7 +137,7 @@ bool Spider::shouldURLbeCrawled( ParsedUrl url ) ...@@ -138,7 +137,7 @@ bool Spider::shouldURLbeCrawled( ParsedUrl url )
return false; return false;
} }
/*
//check if path in url is in the robots txt //check if path in url is in the robots txt
bool Spider::checkRobots(ParsedUrl url) bool Spider::checkRobots(ParsedUrl url)
{ {
...@@ -181,32 +180,8 @@ int Spider::getRobots(ParsedUrl url ) ...@@ -181,32 +180,8 @@ int Spider::getRobots(ParsedUrl url )
};
/*
returns true if fileMap was created, otherwise false
Modifies the filemap to be a char* of the file of the url passed
*/
// make this become a stream reader factory
StreamReader* Spider::request( ParsedUrl url )
{
string localFile;
StreamReader *newReader;
if ( this->mode == "local" )
{
newReader = new LocalReader( url.CompleteUrl );
}
else if ( this->mode == "web" )
{
newReader = new SocketReader( url );
}
//remove fill buffer/ change to get request
newReader->fillBuffer( );
return newReader;
} }
*/
//request function that handles sending over get request via socket or trying to open file //request function that handles sending over get request via socket or trying to open file
......
crawler/spider.h
+ 2
− 7
View file @ 82eb00c1
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
#include "../shared/ThreadClass.h" #include "../shared/ThreadClass.h"
#include<iostream> #include<iostream>
#include <unordered_map> #include <unordered_map>
#include "StreamReader.h" #include "Readers/StreamReader.h"
#include "../util/util.h" #include "../util/util.h"
#include "../parser/Parser.h" #include "../parser/Parser.h"
...@@ -35,16 +35,11 @@ public: ...@@ -35,16 +35,11 @@ public:
virtual void FuncToRun(); virtual void FuncToRun();
//Makes request to given url
// if successful, writes file to disk, stores location to memeber value
// else return false and error information, retry if necessary
StreamReader *request( ParsedUrl url );
bool writeDocToDisk(ParsedUrl url); bool writeDocToDisk(ParsedUrl url);
bool shouldURLbeCrawled( ParsedUrl URL ); bool shouldURLbeCrawled( ParsedUrl URL );
size_t hash(const char * s); size_t hash(const char * s);
int getRobots(ParsedUrl url ); //int getRobots(ParsedUrl url );
bool checkRobots(ParsedUrl url); bool checkRobots(ParsedUrl url);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment