mandlm/VectorSearch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

docs: describe BucketFinder

Browse source
This commit is contained in:
Michael Mandl 2024-03-23 11:24:46 +01:00
parent c1b4fc67e9
commit e4c643880a
Signed by: mandlm
GPG key ID: 4AA25D647AA54CC7
2 changed files with 35 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

32
lib_vector_search/src/bucket_finder.cpp
View file

@ -1,8 +1,11 @@
#include "bucket_finder.h"
#include <algorithm>
#include <iterator>
#include <mutex>
#include <strings.h>
#include <string>
#include <thread>
#include <vector>
void Bucket::insert(const WordList &word_list, size_t first_index,
size_t last_index) {
@ -28,34 +31,35 @@ WordRefList Bucket::find_prefix(std::string_view search_term) const {
return result;
}
BucketFinder::BucketFinder(const WordList &word_list) {
BucketFinder::BucketFinder(const WordList &word_list) { insert(word_list); }
void BucketFinder::insert(const WordList &word_list) {
if (word_list.empty()) {
return;
}
const size_t max_threads = std::thread::hardware_concurrency();
const size_t word_list_size = word_list.size();
const size_t bucket_count =
std::min<size_t>(std::thread::hardware_concurrency(), word_list_size);
const size_t bucket_count = std::min<size_t>(max_threads, word_list_size);
const size_t bucket_size = word_list_size / bucket_count;
buckets_.resize(bucket_count);
std::vector<std::thread> threads;
std::vector<std::thread> insert_threads;
for (auto bucket_index = 0; bucket_index < bucket_count; ++bucket_index) {
auto &bucket = buckets_[bucket_index];
bool is_last_bucket = bucket_index == bucket_count - 1;
const bool is_last_bucket = bucket_index == bucket_count - 1;
const size_t first_word_index = bucket_index * bucket_size;
const size_t last_word_index =
is_last_bucket ? word_list_size : first_word_index + bucket_size;
const size_t first_index = bucket_index * bucket_size;
const size_t last_index =
is_last_bucket ? word_list_size : first_index + bucket_size;
threads.emplace_back([&, first_index, last_index] {
bucket.insert(word_list, first_index, last_index);
insert_threads.emplace_back([&, first_word_index, last_word_index] {
bucket.insert(word_list, first_word_index, last_word_index);
});
}
for (auto &thread : threads) {
for (auto &thread : insert_threads) {
thread.join();
}
}
@ -69,7 +73,7 @@ WordRefList BucketFinder::find_prefix(std::string_view search_term) const {
threads.emplace_back([&] {
auto thread_search_results = bucket.find_prefix(search_term);
if (!thread_search_results.empty()) {
std::lock_guard result_lock(search_results_mutex);
const std::lock_guard result_lock(search_results_mutex);
std::move(thread_search_results.begin(), thread_search_results.end(),
std::back_inserter(search_results));
}