-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add QuerySplitTracer to records and rebuild splits
- Loading branch information
Showing
9 changed files
with
233 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #include "velox/exec/QuerySplitTracer.h" | ||
| #include "QueryTraceUtil.h" | ||
| #include "velox/connectors/hive/HiveConnectorSplit.h" | ||
| #include "velox/exec/QueryTraceTraits.h" | ||
|
|
||
| using namespace facebook::velox::connector::hive; | ||
|
|
||
| namespace facebook::velox::exec::trace { | ||
| /// Used to record and load the input splits from a tracing 'TableScan' | ||
| /// operator, and for getting the traced splits when replaies 'TableScan'. | ||
| /// | ||
| /// Currently, it only works with 'HiveConnectorSplit'. In the future, it will | ||
| /// be extended to handle more types of splits, such as | ||
| /// 'IcebergHiveConnectorSplit'. | ||
| QuerySplitTracer::QuerySplitTracer(std::string traceDir) | ||
| : traceDir_(std::move(traceDir)), | ||
| fs_(filesystems::getFileSystem(traceDir_, nullptr)) { | ||
| VELOX_CHECK_NOT_NULL(fs_); | ||
| } | ||
|
|
||
| void QuerySplitTracer::write(const exec::Split& split) { | ||
| VELOX_CHECK(!split.hasGroup()); | ||
| VELOX_CHECK(split.hasConnectorSplit()); | ||
| const auto splitInfoFile = fs_->openFileForWrite(fmt::format( | ||
| "{}/trace{}.{}", | ||
| traceDir_, | ||
| QueryTraceTraits::kSplitInfoFileSuffix, | ||
| fileId_)); | ||
| const auto splitObj = split.connectorSplit->serialize(); | ||
| const auto splitJson = folly::toJson(splitObj); | ||
| splitInfoFile->append(splitJson); | ||
| splitInfoFile->flush(); | ||
| splitInfoFile->close(); | ||
| ++fileId_; | ||
| } | ||
|
|
||
| // static | ||
| int32_t QuerySplitTracer::extractFileIndex(const std::string& str) { | ||
| std::string capturedStr; | ||
| if (!RE2::FullMatch(str, kFileRegExp, &capturedStr)) { | ||
| return -1; | ||
| } | ||
| return std::stoul(capturedStr); | ||
| } | ||
|
|
||
| std::vector<exec::Split> QuerySplitTracer::read() const { | ||
| std::vector<exec::Split> splits; | ||
| std::map<int32_t, std::string> fileMap; | ||
| for (const auto& filePath : fs_->list(traceDir_)) { | ||
| const auto index = extractFileIndex(filePath); | ||
| if (index == -1) { | ||
| continue; | ||
| } | ||
| fileMap[index] = filePath; | ||
| } | ||
|
|
||
| for (const auto& [_, filePath] : fileMap) { | ||
| const auto splitInfoFilePath = fs_->openFileForRead(filePath); | ||
| folly::dynamic splitInfoObj = getMetadata(filePath, fs_); | ||
| const auto split = | ||
| ISerializable::deserialize<HiveConnectorSplit>(splitInfoObj); | ||
| splits.emplace_back( | ||
| std::make_shared<HiveConnectorSplit>( | ||
| split->connectorId, | ||
| split->filePath, | ||
| split->fileFormat, | ||
| split->start, | ||
| split->length, | ||
| split->partitionKeys, | ||
| split->tableBucketNumber, | ||
| split->customSplitInfo, | ||
| split->extraFileInfo, | ||
| split->serdeParameters, | ||
| split->splitWeight, | ||
| split->infoColumns, | ||
| split->properties), | ||
| -1); | ||
| } | ||
| return splits; | ||
| } | ||
| } // namespace facebook::velox::exec::trace |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include "velox/common/file/FileSystems.h" | ||
| #include "velox/exec/Split.h" | ||
|
|
||
| #include <re2/re2.h> | ||
|
|
||
| namespace facebook::velox::exec::trace { | ||
|
|
||
| class QuerySplitTracer { | ||
| public: | ||
| explicit QuerySplitTracer(std::string traceDir); | ||
|
|
||
| /// Serializes and writes out each split. Each serialized split is immediately | ||
| /// flushed to a separate file to ensure that we can still replay a traced | ||
| /// operator even if a crash occurs during execution. | ||
| void write(const exec::Split& split); | ||
|
|
||
| /// Lists the split info files and deserializes the splits. The splits are | ||
| /// sorted by the file index generated during the tracing process, allowing us | ||
| /// to replay the execution in the same order as the original split | ||
| /// processing. | ||
| std::vector<exec::Split> read() const; | ||
|
|
||
| private: | ||
| static int32_t extractFileIndex(const std::string& str); | ||
|
|
||
| const std::string traceDir_; | ||
| const std::shared_ptr<filesystems::FileSystem> fs_; | ||
| int32_t fileId_{0}; | ||
|
|
||
| static inline RE2 kFileRegExp{std::string(R"(.+\.split\.(\d+)$)")}; | ||
| }; | ||
| } // namespace facebook::velox::exec::trace |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters