Skip to content
Snippets Groups Projects
Commit 0fbf4f3c authored by Nicholas Yang's avatar Nicholas Yang
Browse files

word seek mmdhts in

parent f16b76a2
No related branches found
No related tags found
1 merge request!3Indexer
No preview for this file type
No preview for this file type
......@@ -43,11 +43,13 @@ void Indexer::run ( )
if(currentBlockNumberWords >= 20000) {
save();
saveWordSeek();
reset();
}
}
save();
saveWordSeek();
reset();
saveChunkDictionary();
}
......@@ -146,12 +148,11 @@ void Indexer::save ( )
}
close( file );
currentFile++;
}
void Indexer::saveChunkDictionary ( )
{
MMDiskHashTable dhtChunk = MMDiskHashTable(util::GetCurrentWorkingDir() + "/indexer/output/index-master.txt", 30, 168);
MMDiskHashTable dhtChunk = MMDiskHashTable(util::GetCurrentWorkingDir() + "/indexer/output/master.txt", 30, 168);
for ( auto word : chunkDictionary )
{
string key = word.first;
......@@ -167,6 +168,22 @@ void Indexer::saveChunkDictionary ( )
}
}
void Indexer::saveWordSeek() {
MMDiskHashTable wordSeek = MMDiskHashTable(
util::GetCurrentWorkingDir() + "/indexer/output/" + to_string(currentFile) + "-wordseek.txt", 30, 168);
for (auto word : postingsSeekTable) {
string key = word.first;
if (key.size() > 30) {
key.resize(30);
}
string value = "";
for (auto entry : word.second) {
value += ("<" + to_string(entry.offset) + ", " + to_string(entry.realLocation) + "> ");
}
wordSeek.insert(key, value);
}
}
void Indexer::verbose_save ( )
{
map< string, vector< size_t > > maps( masterDictionary.begin( ), masterDictionary.end( ) );
......@@ -190,4 +207,5 @@ void Indexer::reset ( )
currentBlockNumberWords = 0;
currentBlockNumberDocs = 0;
}
currentFile++;
}
......@@ -46,8 +46,8 @@ public:
private:
void save ( );
void saveChunkDictionary ( );
void saveWordSeek();
void saveChunkDictionary ( );
void reset ( );
......@@ -63,6 +63,7 @@ private:
size_t currentBlockNumberWords;
size_t currentBlockNumberDocs;
};
};
#endif /*indexer_h*/
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment