implement scalar quantization as data type for NGT-graph and QBG

yahoojapan · Oct 28, 2024 · 0416078 · 0416078
1 parent 1e44fff
commit 0416078
Show file tree

Hide file tree

Showing 37 changed files with 7,722 additions and 3,342 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.2.4
+2.3.0
diff --git a/bin/ngt/ngt.cpp b/bin/ngt/ngt.cpp
@@ -122,6 +122,8 @@ main(int argc, char **argv)
     }
   } catch(NGT::Exception &err) {
     cerr << "ngt: Error: " << err.what() << endl;
+    return 1;
   }
+  return 0;
 
 }
diff --git a/bin/qbg/qbg.cpp b/bin/qbg/qbg.cpp
@@ -23,7 +23,16 @@ main(int argc, char **argv)
 
   QBG::CLI ngt;
 
-  ngt.execute(args);
+  try {
+    ngt.execute(args);
+  } catch(NGT::Exception &err) {
+    cerr << "qbg: Error: " << err.what() << endl;
+    return 1;
+  } catch(...) {
+    cerr << "qbg: Error: " << endl;
+    return 1;
+  }
+  return 0;
 }
 
 

diff --git a/lib/NGT/Capi.cpp b/lib/NGT/Capi.cpp
@@ -331,6 +331,18 @@ bool ngt_set_property_object_type_integer(NGTProperty prop, NGTError error) {
   return true;
 }
 
+bool ngt_set_property_object_type_qsint8(NGTProperty prop, NGTError error) {
+  if(prop == NULL){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : parametor error: prop = " << prop;
+    operate_error_string_(ss, error);
+    return false;
+  }
+
+  (*static_cast<NGT::Property*>(prop)).objectType = NGT::ObjectSpace::ObjectType::Qsuint8;
+  return true;
+}
+
 bool ngt_set_property_distance_type(NGTProperty prop, NGT::Index::Property::DistanceType type, NGTError error) {
   if(prop == NULL){
     std::stringstream ss;
@@ -428,11 +440,7 @@ NGTPropertyInfo ngt_get_property_info(NGTIndex index, NGTError error) {
     prop.prefetchSize,
     prop.accuracyTable.c_str(),
     prop.searchType.c_str(),
-#ifdef NGT_INNER_PRODUCT
     prop.maxMagnitude,
-#else
-    -1,
-#endif
     prop.nOfNeighborsForInsertionOrder,
     prop.epsilonForInsertionOrder,
 #ifdef NGT_REFINEMENT
@@ -1055,6 +1063,47 @@ ObjectID ngt_append_index_as_float16(NGTIndex index, NGTFloat16 *obj, uint32_t o
   }
 }
 
+ObjectID ngt_insert_to_refinement_as_float(NGTIndex index, float *obj, uint32_t obj_dim, NGTError error) {
+  if(index == NULL || obj == NULL || obj_dim == 0){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " obj = " << obj << " obj_dim = " << obj_dim;
+    operate_error_string_(ss, error);
+    return 0;
+  }
+
+  try{
+    NGT::Index* pindex = static_cast<NGT::Index*>(index);
+    std::vector<float> vobj(&obj[0], &obj[obj_dim]);
+    return pindex->insertToRefinement(vobj);
+  }catch(std::exception &err) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return 0;
+  }
+}
+
+ObjectID ngt_append_to_refinement_as_float(NGTIndex index, float *obj, uint32_t obj_dim, NGTError error) {
+  if(index == NULL || obj == NULL || obj_dim == 0){
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : parametor error: index = " << index << " obj = " << obj << " obj_dim = " << obj_dim;
+    operate_error_string_(ss, error);
+    return 0;
+  }
+
+  try{
+    NGT::Index* pindex = static_cast<NGT::Index*>(index);
+    std::vector<float> vobj(&obj[0], &obj[obj_dim]);
+    return pindex->appendToRefinement(vobj);
+  }catch(std::exception &err) {
+    std::stringstream ss;
+    ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what();
+    operate_error_string_(ss, error);
+    return 0;
+  }
+}
+
+
 bool ngt_batch_append_index(NGTIndex index, float *obj, uint32_t data_count, NGTError error) {
   try{
     NGT::Index* pindex = static_cast<NGT::Index*>(index);

diff --git a/lib/NGT/Capi.h b/lib/NGT/Capi.h
@@ -196,6 +196,8 @@ bool ngt_set_property_object_type_float16(NGTProperty, NGTError);
 
 bool ngt_set_property_object_type_integer(NGTProperty, NGTError);
 
+bool ngt_set_property_object_type_qsint8(NGTProperty, NGTError);
+
 bool ngt_set_property_distance_type_l1(NGTProperty, NGTError);
 
 bool ngt_set_property_distance_type_l2(NGTProperty, NGTError);
@@ -272,6 +274,10 @@ ObjectID ngt_insert_index_as_float16(NGTIndex, NGTFloat16*, uint32_t, NGTError);
 
 ObjectID ngt_append_index_as_float16(NGTIndex, NGTFloat16*, uint32_t, NGTError);
 
+ObjectID ngt_append_to_refinement_as_float(NGTIndex, float*, uint32_t, NGTError);
+
+ObjectID ngt_insert_to_refinement_as_float(NGTIndex, float*, uint32_t, NGTError);
+
 bool ngt_batch_append_index(NGTIndex, float*, uint32_t, NGTError);
 
 bool ngt_batch_insert_index(NGTIndex, float*, uint32_t, uint32_t *, NGTError);

diff --git a/lib/NGT/Clustering.h b/lib/NGT/Clustering.h
@@ -125,7 +125,7 @@ namespace NGT {
     }
 
     static void
-      loadVectors(const std::string &file, std::vector<std::vector<float> > &vectors)
+      loadVectors(const std::string &file, std::vector<std::vector<float>> &vectors)
     {
       std::ifstream is(file);
       if (!is) {
@@ -152,7 +152,7 @@ namespace NGT {
     }
 
     static void
-      saveVectors(const std::string &file, std::vector<std::vector<float> > &vectors)
+      saveVectors(const std::string &file, std::vector<std::vector<float>> &vectors)
     {
       std::ofstream os(file);
       for (auto vit = vectors.begin(); vit != vectors.end(); ++vit) {
@@ -167,6 +167,22 @@ namespace NGT {
       }
     }
 
+    static void
+      saveVectors(const std::string &file, std::vector<std::vector<uint32_t>> &vectors)
+    {
+      std::ofstream os(file);
+      for (auto vit = vectors.begin(); vit != vectors.end(); ++vit) {
+	std::vector<uint32_t> &v = *vit;
+	for (auto it = v.begin(); it != v.end(); ++it) {
+	  os << (*it);
+	  if (it + 1 != v.end()) {
+	    os << "\t";
+	  }
+	}
+	os << std::endl;
+      }
+    }
+
     static void
       loadVector(const std::string &file, std::vector<size_t> &vectors)
     {
@@ -403,7 +419,7 @@ namespace NGT {
 	}
       }
 
-      std::vector<Entry> sortedObjects(vectors.size());
+      std::vector<Entry> sortedObjects(vectors.size());	
 #pragma omp parallel for
       for (size_t vi = 0; vi < vectors.size(); vi++) {
 	auto vit = vectors.begin() + vi;