Skip to content
Snippets Groups Projects
Commit 71e0bafc authored by jsclose's avatar jsclose
Browse files

stable crawler with parse

parent 121f98ed
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
CMakeLists.txt
+ 0
− 3
View file @ 71e0bafc
......@@ -30,8 +30,6 @@ add_executable(crawl
util/stringProcessing.cpp)
add_executable(test1 main.cpp shared/ProducerConsumerQueue.h
shared/ThreadClass.h crawler/crawler.cpp crawler/spider.cpp shared/url.h crawler/StreamReader.h util/util.cpp crawler/SocketReader.cpp crawler/SocketReader.h crawler/LocalReader.h )
add_executable(StringProcessingTest
util/stringProcessing.h
......@@ -60,4 +58,3 @@ find_package(OpenSSL REQUIRED)
target_link_libraries(crawl OpenSSL::SSL pthread)
target_link_libraries(test1 OpenSSL::SSL)
\ No newline at end of file
ParserEndToEndTest deleted 100755 → 0
+ 0
− 0
View file @ 121f98ed
File deleted
crawl 0 → 100755
+ 0
− 0
View file @ 71e0bafc
File added
crawler/spider.cpp
+ 5
− 4
View file @ 71e0bafc
......@@ -52,8 +52,9 @@ void Spider::FuncToRun()
//url has not seen before or time since seen is past certain criteria
if ( shouldURLbeCrawled( currentUrl ))
{
bool success = writeDocToDisk(currentUrl);
if ( success && cond )
//bool success = writeDocToDisk(currentUrl);
//if ( success && cond )
if(cond)
{
......@@ -63,10 +64,10 @@ void Spider::FuncToRun()
int fd = util::writeToNewFileToLocation( reader->buffer, pathToDisk);
/*
Document document ( currentUrl, reader->buffer );
auto dictionary = parser.execute ( &document );
*/
cond = true;
}
else
......
crawler/spider.h
+ 2
− 2
View file @ 71e0bafc
......@@ -25,7 +25,7 @@ public:
Spider( string mode_in, ProducerConsumerQueue < string > *url_q_in,
unordered_map < string, int > *doc_map_lookup_in )
: mode( mode_in ), urlFrontier( url_q_in ), docMapLookup( doc_map_lookup_in )
: mode( mode_in ), urlFrontier( url_q_in ), docMapLookup( doc_map_lookup_in ), parser( url_q_in)
{
};
......@@ -54,6 +54,6 @@ private:
ProducerConsumerQueue < string > *urlFrontier;
string mode;
unordered_map < string, int > *docMapLookup;
Parser parser;
};
\ No newline at end of file
crawlerOutput/en.wikipedia.org.txt deleted 100755 → 0
+ 0
− 1035
View file @ 121f98ed
This diff is collapsed.
crawlerOutput/example.txt deleted 100644 → 0
+ 0
− 64
View file @ 121f98ed
<!DOCTYPE
html>
<html>
<head>
<!--
HTML
Codes
by
Quackit.com
-->
<title>
Food
store
is
here</title>
<meta
name="viewport"
content="width=device-width,
initial-scale=1">
<meta
name="keywords"
content="store
food
dinner
lunch">
<meta
name="description"
content="The
food
store
sells
cat
food
for
dinner,
lunch,
and
breakfast.">
<style>
body
{background-color:#ffffff;background-repeat:no-repeat;background-position:top
left;background-attachment:fixed;}
h1{font-family:Arial,
sans-serif;color:#000000;background-color:#ffffff;}
p
{font-family:Georgia,
serif;font-size:14px;font-style:normal;font-weight:normal;color:#000000;background-color:#ffffff;}
</style>
</head>
<body>
<h1>COme
shop
Come
shop
at
our
Store</h1>
<p>Please
come
to
our
store!</p>
</body>
</html>
crawlerOutput/www.nytimes.com.txt deleted 100755 → 0
+ 0
− 5774
View file @ 121f98ed
This diff is collapsed.
makefile
+ 56
− 52
View file @ 71e0bafc
......@@ -136,32 +136,6 @@ ParserEndToEndTest/fast:
$(MAKE) -f CMakeFiles/ParserEndToEndTest.dir/build.make CMakeFiles/ParserEndToEndTest.dir/build
.PHONY : ParserEndToEndTest/fast
#=============================================================================
# Target rules for targets named StemmerTest
# Build rule for target.
StemmerTest: cmake_check_build_system
$(MAKE) -f CMakeFiles/Makefile2 StemmerTest
.PHONY : StemmerTest
# fast build rule for target.
StemmerTest/fast:
$(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/build
.PHONY : StemmerTest/fast
#=============================================================================
# Target rules for targets named StringProcessingTest
# Build rule for target.
StringProcessingTest: cmake_check_build_system
$(MAKE) -f CMakeFiles/Makefile2 StringProcessingTest
.PHONY : StringProcessingTest
# fast build rule for target.
StringProcessingTest/fast:
$(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/build
.PHONY : StringProcessingTest/fast
#=============================================================================
# Target rules for targets named TokenizerTest
......@@ -176,17 +150,17 @@ TokenizerTest/fast:
.PHONY : TokenizerTest/fast
#=============================================================================
# Target rules for targets named test1
# Target rules for targets named StringProcessingTest
# Build rule for target.
test1: cmake_check_build_system
$(MAKE) -f CMakeFiles/Makefile2 test1
.PHONY : test1
StringProcessingTest: cmake_check_build_system
$(MAKE) -f CMakeFiles/Makefile2 StringProcessingTest
.PHONY : StringProcessingTest
# fast build rule for target.
test1/fast:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/build
.PHONY : test1/fast
StringProcessingTest/fast:
$(MAKE) -f CMakeFiles/StringProcessingTest.dir/build.make CMakeFiles/StringProcessingTest.dir/build
.PHONY : StringProcessingTest/fast
#=============================================================================
# Target rules for targets named crawl
......@@ -201,6 +175,19 @@ crawl/fast:
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/build
.PHONY : crawl/fast
#=============================================================================
# Target rules for targets named StemmerTest
# Build rule for target.
StemmerTest: cmake_check_build_system
$(MAKE) -f CMakeFiles/Makefile2 StemmerTest
.PHONY : StemmerTest
# fast build rule for target.
StemmerTest/fast:
$(MAKE) -f CMakeFiles/StemmerTest.dir/build.make CMakeFiles/StemmerTest.dir/build
.PHONY : StemmerTest/fast
#=============================================================================
# Target rules for targets named search
......@@ -220,7 +207,6 @@ crawler/SocketReader.o: crawler/SocketReader.cpp.o
# target to build an object file
crawler/SocketReader.cpp.o:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/SocketReader.cpp.o
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/SocketReader.cpp.o
.PHONY : crawler/SocketReader.cpp.o
......@@ -230,7 +216,6 @@ crawler/SocketReader.i: crawler/SocketReader.cpp.i
# target to preprocess a source file
crawler/SocketReader.cpp.i:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/SocketReader.cpp.i
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/SocketReader.cpp.i
.PHONY : crawler/SocketReader.cpp.i
......@@ -240,7 +225,6 @@ crawler/SocketReader.s: crawler/SocketReader.cpp.s
# target to generate assembly for a file
crawler/SocketReader.cpp.s:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/SocketReader.cpp.s
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/SocketReader.cpp.s
.PHONY : crawler/SocketReader.cpp.s
......@@ -250,7 +234,6 @@ crawler/crawler.o: crawler/crawler.cpp.o
# target to build an object file
crawler/crawler.cpp.o:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/crawler.cpp.o
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/crawler.cpp.o
.PHONY : crawler/crawler.cpp.o
......@@ -260,7 +243,6 @@ crawler/crawler.i: crawler/crawler.cpp.i
# target to preprocess a source file
crawler/crawler.cpp.i:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/crawler.cpp.i
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/crawler.cpp.i
.PHONY : crawler/crawler.cpp.i
......@@ -270,7 +252,6 @@ crawler/crawler.s: crawler/crawler.cpp.s
# target to generate assembly for a file
crawler/crawler.cpp.s:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/crawler.cpp.s
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/crawler.cpp.s
.PHONY : crawler/crawler.cpp.s
......@@ -280,7 +261,6 @@ crawler/spider.o: crawler/spider.cpp.o
# target to build an object file
crawler/spider.cpp.o:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/spider.cpp.o
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/spider.cpp.o
.PHONY : crawler/spider.cpp.o
......@@ -290,7 +270,6 @@ crawler/spider.i: crawler/spider.cpp.i
# target to preprocess a source file
crawler/spider.cpp.i:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/spider.cpp.i
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/spider.cpp.i
.PHONY : crawler/spider.cpp.i
......@@ -300,7 +279,6 @@ crawler/spider.s: crawler/spider.cpp.s
# target to generate assembly for a file
crawler/spider.cpp.s:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/crawler/spider.cpp.s
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/crawler/spider.cpp.s
.PHONY : crawler/spider.cpp.s
......@@ -310,7 +288,6 @@ main.o: main.cpp.o
# target to build an object file
main.cpp.o:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/main.cpp.o
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/main.cpp.o
$(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/main.cpp.o
.PHONY : main.cpp.o
......@@ -321,7 +298,6 @@ main.i: main.cpp.i
# target to preprocess a source file
main.cpp.i:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/main.cpp.i
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/main.cpp.i
$(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/main.cpp.i
.PHONY : main.cpp.i
......@@ -332,7 +308,6 @@ main.s: main.cpp.s
# target to generate assembly for a file
main.cpp.s:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/main.cpp.s
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/main.cpp.s
$(MAKE) -f CMakeFiles/search.dir/build.make CMakeFiles/search.dir/main.cpp.s
.PHONY : main.cpp.s
......@@ -472,6 +447,33 @@ shared/urlTest.cpp.s:
$(MAKE) -f CMakeFiles/URLTEST.dir/build.make CMakeFiles/URLTEST.dir/shared/urlTest.cpp.s
.PHONY : shared/urlTest.cpp.s
util/stringProcessing.o: util/stringProcessing.cpp.o
.PHONY : util/stringProcessing.o
# target to build an object file
util/stringProcessing.cpp.o:
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/stringProcessing.cpp.o
.PHONY : util/stringProcessing.cpp.o
util/stringProcessing.i: util/stringProcessing.cpp.i
.PHONY : util/stringProcessing.i
# target to preprocess a source file
util/stringProcessing.cpp.i:
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/stringProcessing.cpp.i
.PHONY : util/stringProcessing.cpp.i
util/stringProcessing.s: util/stringProcessing.cpp.s
.PHONY : util/stringProcessing.s
# target to generate assembly for a file
util/stringProcessing.cpp.s:
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/stringProcessing.cpp.s
.PHONY : util/stringProcessing.cpp.s
util/tests/stemmerTest.o: util/tests/stemmerTest.cpp.o
.PHONY : util/tests/stemmerTest.o
......@@ -559,7 +561,7 @@ util/util.o: util/util.cpp.o
# target to build an object file
util/util.cpp.o:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/util/util.cpp.o
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/util.cpp.o
.PHONY : util/util.cpp.o
util/util.i: util/util.cpp.i
......@@ -568,7 +570,7 @@ util/util.i: util/util.cpp.i
# target to preprocess a source file
util/util.cpp.i:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/util/util.cpp.i
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/util.cpp.i
.PHONY : util/util.cpp.i
util/util.s: util/util.cpp.s
......@@ -577,7 +579,7 @@ util/util.s: util/util.cpp.s
# target to generate assembly for a file
util/util.cpp.s:
$(MAKE) -f CMakeFiles/test1.dir/build.make CMakeFiles/test1.dir/util/util.cpp.s
$(MAKE) -f CMakeFiles/crawl.dir/build.make CMakeFiles/crawl.dir/util/util.cpp.s
.PHONY : util/util.cpp.s
# Help Target
......@@ -589,12 +591,11 @@ help:
@echo "... edit_cache"
@echo "... URLTEST"
@echo "... ParserEndToEndTest"
@echo "... rebuild_cache"
@echo "... StemmerTest"
@echo "... StringProcessingTest"
@echo "... TokenizerTest"
@echo "... test1"
@echo "... StringProcessingTest"
@echo "... crawl"
@echo "... rebuild_cache"
@echo "... StemmerTest"
@echo "... search"
@echo "... crawler/SocketReader.o"
@echo "... crawler/SocketReader.i"
......@@ -623,6 +624,9 @@ help:
@echo "... shared/urlTest.o"
@echo "... shared/urlTest.i"
@echo "... shared/urlTest.s"
@echo "... util/stringProcessing.o"
@echo "... util/stringProcessing.i"
@echo "... util/stringProcessing.s"
@echo "... util/tests/stemmerTest.o"
@echo "... util/tests/stemmerTest.i"
@echo "... util/tests/stemmerTest.s"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment