update parameters for the scalar quantization

yahoojapan · Oct 29, 2024 · 9908df2 · 9908df2
1 parent 0416078
commit 9908df2
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 7 deletions.
diff --git a/bin/ngt/README-jp.md b/bin/ngt/README-jp.md
@@ -91,6 +91,7 @@ ANNGやBKNNGを指定した場合には登録データ（ノード）からエ
 - __c__: 1バイト整数
 - __f__: 4バイト浮動小数点（デフォルト）
 - __h__: 2バイト浮動小数点
+- __q__: 1バイトスカラー量子化
 
 **-D** *distance\_function*  
 距離関数を指定します。

diff --git a/bin/ngt/README.md b/bin/ngt/README.md
@@ -93,6 +93,7 @@ Specify the data object type.
 - __c__: 1 byte unsigned integer
 - __f__: 4 byte floating point number (default)
 - __h__: 2 byte floating point number
+- __q__: 1 byte scalar quantization
 
 **-D** *distance\_function*  
 Specify the distance function as follows.

diff --git a/bin/qbg/README.md b/bin/qbg/README.md
@@ -97,7 +97,7 @@ Specify the expansion ratio of the number of approximate inner search objects to
 ### CREATE
 Make and initialize a QBG directory for the quantized blob graph.
 
-      $ qbg create [-d number_of_dimension] [-P number_of_extended_dimensions] [-O object_type] [-D distance_function] [-C number_of_blobs] index
+      $ qbg create [-d number_of_dimension] [-P number_of_extended_dimensions] [-O object_type] [-D distance_function] [-C cluster_data_type] index
 
 *index*  
 Specify the name of the directory for QBG.
@@ -118,8 +118,10 @@ Specify the distance function as follows.
 - __2__: L2 distance (default)
 - __c__: Cosine similarity
 
-**-C** *number_of_blobs*  
-Specify the number of blobs that should be less than or equal to the number of quantization clusters.
+**-C** *cluster_data_type*  
+Specify the cluster data type.
+- __pq4__: 4 bit product quantization (default)
+- __sq8__: 1 byte scalar quantization
 
 **-N** *number_of_subvectors*  
 Specify the number of subvectors that should be a divisor of the number of the extended dimensions.

diff --git a/lib/NGT/Command.cpp b/lib/NGT/Command.cpp
@@ -129,7 +129,7 @@ using namespace std;
     } else if (objectType == "h") {
       property.objectType = NGT::Index::Property::ObjectType::Float16;
 #endif
-    } else if (objectType == "s8" || objectType == "sqsu8") {
+    } else if (objectType == "qsu8" || objectType == "qs8" || objectType == "q") {
       property.objectType = NGT::Index::Property::ObjectType::Qsuint8;
 #ifdef NGT_BFLOAT
     } else if (objectType == "H") {

diff --git a/lib/NGT/NGTQ/QbgCli.cpp b/lib/NGT/NGTQ/QbgCli.cpp
@@ -97,7 +97,7 @@ class QbgCliBuildParameters : public QBG::BuildParameters {
     transform(clusterDataType.begin(), clusterDataType.end(), clusterDataType.begin(), ::tolower);
     if (clusterDataType == "-" || clusterDataType == "pq4") {
       creation.localClusterDataType = NGTQ::ClusterDataTypePQ4;
-    } else if (clusterDataType == "sqsu8" || clusterDataType == "sq8") {
+    } else if (clusterDataType == "sqsu8" || clusterDataType == "sqs8" || clusterDataType == "sq8") {
       creation.localClusterDataType = NGTQ::ClusterDataTypeSQSU8;
     } else if (clusterDataType == "nq") {
       creation.localClusterDataType = NGTQ::ClusterDataTypeNQ;

diff --git a/lib/NGT/NGTQ/QuantizedBlobGraph.h b/lib/NGT/NGTQ/QuantizedBlobGraph.h
@@ -580,8 +580,9 @@ namespace QBG {
 	ifstream *ifs = new ifstream;
 	ifs->ifstream::open(data);
 	if (!(*ifs)) {
-	  cerr << "Cannot open the specified file. " << data << endl;
-	  return;
+	  std::stringstream msg;
+	  msg << "Cannot open the specified file. " << data;
+	  NGTThrowException(msg);
 	}
 	is = ifs;
       }