Support Spark query runner

facebookincubator · Apr 25, 2024 · f5dcf32 · f5dcf32
1 parent 28a1f18
commit f5dcf32
Show file tree

Hide file tree

Showing 26 changed files with 5,317 additions and 389 deletions.
diff --git a/CMake/Findgrpc.cmake b/CMake/Findgrpc.cmake
diff --git a/velox/exec/fuzzer/AggregationFuzzer.cpp b/velox/exec/fuzzer/AggregationFuzzer.cpp
@@ -47,6 +47,21 @@ namespace facebook::velox::exec::test {
 class AggregationFuzzerBase;
 
 namespace {
+namespace {
+bool isSupportedDwrfType(const TypePtr& type) {
+  if (type->isDate() || type->isIntervalDayTime() || type->isUnKnown()) {
+    return false;
+  }
+
+  for (auto i = 0; i < type->size(); ++i) {
+    if (!isSupportedDwrfType(type->childAt(i))) {
+      return false;
+    }
+  }
+
+  return true;
+}
+} // namespace
 
 class AggregationFuzzer : public AggregationFuzzerBase {
  public:
@@ -1067,8 +1082,8 @@ bool AggregationFuzzer::compareEquivalentPlanResults(
           LOG(INFO) << "Verified results against reference DB";
         }
       } else if (referenceQueryRunner_->supportsVeloxVectorResults()) {
-        if (isSupportedType(firstPlan->outputType()) &&
-            isSupportedType(input.front()->type())) {
+        if (isSupportedDwrfType(firstPlan->outputType()) &&
+            isSupportedDwrfType(input.front()->type())) {
           auto referenceResult =
               computeReferenceResultsAsVector(firstPlan, input);
           stats_.updateReferenceQueryStats(referenceResult.second);

diff --git a/velox/exec/fuzzer/AggregationFuzzerBase.cpp b/velox/exec/fuzzer/AggregationFuzzerBase.cpp
@@ -91,21 +91,6 @@ int32_t AggregationFuzzerBase::randInt(int32_t min, int32_t max) {
   return boost::random::uniform_int_distribution<int32_t>(min, max)(rng_);
 }
 
-bool AggregationFuzzerBase::isSupportedType(const TypePtr& type) const {
-  // Date / IntervalDayTime/ Unknown are not currently supported by DWRF.
-  if (type->isDate() || type->isIntervalDayTime() || type->isUnKnown()) {
-    return false;
-  }
-
-  for (auto i = 0; i < type->size(); ++i) {
-    if (!isSupportedType(type->childAt(i))) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
 bool AggregationFuzzerBase::addSignature(
     const std::string& name,
     const FunctionSignaturePtr& signature) {
@@ -467,7 +452,7 @@ AggregationFuzzerBase::computeReferenceResults(
               sql.value(), input, plan->outputType()),
           ReferenceQueryErrorCode::kSuccess);
     } catch (...) {
-      LOG(WARNING) << "Query failed in the reference DB";
+      LOG(WARNING) << "Query failed in the reference DB: " << sql.value();
       return std::make_pair(
           std::nullopt, ReferenceQueryErrorCode::kReferenceQueryFail);
     }
@@ -493,7 +478,7 @@ AggregationFuzzerBase::computeReferenceResultsAsVector(
               sql.value(), input, plan->outputType()),
           ReferenceQueryErrorCode::kSuccess);
     } catch (...) {
-      LOG(WARNING) << "Query failed in the reference DB";
+      LOG(WARNING) << "Query failed in the reference DB: " << sql.value();
       return std::make_pair(
           std::nullopt, ReferenceQueryErrorCode::kReferenceQueryFail);
     }

diff --git a/velox/exec/fuzzer/AggregationFuzzerBase.h b/velox/exec/fuzzer/AggregationFuzzerBase.h
@@ -241,12 +241,6 @@ class AggregationFuzzerBase {
       const std::vector<std::shared_ptr<ResultVerifier>>& customVerifiers,
       const velox::test::ResultOrError& expected);
 
-  /// Returns false if the type or its children are unsupported.
-  /// Currently returns false if type is Date,IntervalDayTime or Unknown.
-  /// @param type
-  /// @return bool
-  bool isSupportedType(const TypePtr& type) const;
-
   // @param customVerification If false, results are compared as is. Otherwise,
   // only row counts are compared.
   // @param customVerifiers Custom verifier for each aggregate function. These

diff --git a/velox/exec/fuzzer/CMakeLists.txt b/velox/exec/fuzzer/CMakeLists.txt
@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-add_library(velox_fuzzer_util DuckQueryRunner.cpp PrestoQueryRunner.cpp)
+add_library(velox_fuzzer_util DuckQueryRunner.cpp PrestoQueryRunner.cpp
+                              QueryTranslator.cpp)
 
 target_link_libraries(velox_fuzzer_util velox_core velox_exec_test_lib cpr::cpr
                       velox_type_parser Folly::folly)