refactor: remove datanode instance (#2342)

* pass nextest

Signed-off-by: Ruihang Xia <[email protected]>

* remove deadcode

Signed-off-by: Ruihang Xia <[email protected]>

* rename region_alive_keepers

Signed-off-by: Ruihang Xia <[email protected]>

---------

Signed-off-by: Ruihang Xia <[email protected]>

src/cmd/src/standalone.rs

@@ -20,8 +20,8 @@ use common_config::WalConfig;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
 use datanode::datanode::{Datanode, DatanodeOptions, ProcedureConfig, StorageConfig};
-use datanode::instance::InstanceRef;
 use datanode::region_server::RegionServer;
+use datanode::Instance as InstanceRef;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use frontend::service_config::{

src/common/greptimedb-telemetry/src/lib.rs

@@ -18,7 +18,7 @@ use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use common_runtime::error::{Error, Result};
-use common_runtime::{BoxedTaskFunction, RepeatedTask, Runtime, TaskFunction};
+use common_runtime::{BoxedTaskFunction, RepeatedTask, TaskFunction};
 use common_telemetry::{debug, info};
 use reqwest::{Client, Response};
 use serde::{Deserialize, Serialize};
@@ -49,11 +49,11 @@ impl GreptimeDBTelemetryTask {
         GreptimeDBTelemetryTask::Disable
     }
 
-    pub fn start(&self, runtime: Runtime) -> Result<()> {
+    pub fn start(&self) -> Result<()> {
         match self {
             GreptimeDBTelemetryTask::Enable(task) => {
                 print_anonymous_usage_data_disclaimer();
-                task.start(runtime)
+                task.start(common_runtime::bg_runtime())
             }
             GreptimeDBTelemetryTask::Disable => Ok(()),
         }

src/datanode/src/datanode.rs

@@ -23,6 +23,7 @@ use common_base::readable_size::ReadableSize;
 use common_base::Plugins;
 use common_config::WalConfig;
 use common_error::ext::BoxedError;
+use common_greptimedb_telemetry::GreptimeDBTelemetryTask;
 pub use common_procedure::options::ProcedureConfig;
 use common_runtime::Runtime;
 use common_telemetry::info;
@@ -52,6 +53,7 @@ use tokio::fs;
 use crate::error::{
     CreateDirSnafu, OpenLogStoreSnafu, Result, RuntimeResourceSnafu, ShutdownInstanceSnafu,
 };
+use crate::greptimedb_telemetry::get_greptimedb_telemetry_task;
 use crate::heartbeat::HeartbeatTask;
 use crate::region_server::RegionServer;
 use crate::server::Services;
@@ -399,6 +401,7 @@ pub struct Datanode {
     services: Option<Services>,
     heartbeat_task: Option<HeartbeatTask>,
     region_server: RegionServer,
+    greptimedb_telemetry_task: Arc<GreptimeDBTelemetryTask>,
 }
 
 impl Datanode {
@@ -420,7 +423,7 @@ impl Datanode {
                 .context(RuntimeResourceSnafu)?,
         );
 
-        let mut region_server = RegionServer::new(query_engine, runtime);
+        let mut region_server = RegionServer::new(query_engine, runtime.clone());
         let log_store = Self::build_log_store(&opts).await?;
         let object_store = store::new_object_store(&opts).await?;
         let engines = Self::build_store_engines(&opts, log_store, object_store).await?;
@@ -439,12 +442,19 @@ impl Datanode {
             }
             Mode::Standalone => None,
         };
+        let greptimedb_telemetry_task = get_greptimedb_telemetry_task(
+            Some(opts.storage.data_home.clone()),
+            &opts.mode,
+            opts.enable_telemetry,
+        )
+        .await;
 
         Ok(Self {
             opts,
             services,
             heartbeat_task,
             region_server,
+            greptimedb_telemetry_task,
         })
     }
 
@@ -453,6 +463,7 @@ impl Datanode {
         if let Some(task) = &self.heartbeat_task {
             task.start().await?;
         }
+        let _ = self.greptimedb_telemetry_task.start();
         self.start_services().await
     }
 
@@ -476,6 +487,7 @@ impl Datanode {
     pub async fn shutdown(&self) -> Result<()> {
         // We must shutdown services first
         self.shutdown_services().await?;
+        let _ = self.greptimedb_telemetry_task.stop().await;
         if let Some(heartbeat_task) = &self.heartbeat_task {
             heartbeat_task
                 .close()

src/datanode/src/heartbeat.rs

@@ -16,15 +16,17 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
-use api::v1::meta::{HeartbeatRequest, Peer, RegionStat};
+use api::v1::meta::{HeartbeatRequest, Peer, RegionStat, Role};
+use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
 use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
 use common_meta::heartbeat::handler::{
     HandlerGroupExecutor, HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutorRef,
 };
 use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MailboxRef};
 use common_meta::heartbeat::utils::outgoing_message_to_mailbox_message;
 use common_telemetry::{debug, error, info, trace, warn};
-use meta_client::client::{HeartbeatSender, MetaClient};
+use meta_client::client::{HeartbeatSender, MetaClient, MetaClientBuilder};
+use meta_client::MetaClientOptions;
 use snafu::{OptionExt, ResultExt};
 use tokio::sync::mpsc;
 use tokio::time::Instant;
@@ -35,7 +37,6 @@ use crate::datanode::DatanodeOptions;
 use crate::error::{
     self, MetaClientInitSnafu, MissingMetasrvOptsSnafu, MissingNodeIdSnafu, Result,
 };
-use crate::instance::new_metasrv_client;
 use crate::region_server::RegionServer;
 
 pub(crate) mod handler;
@@ -306,6 +307,39 @@ fn resolve_addr(bind_addr: &str, hostname_addr: &Option<String>) -> String {
     }
 }
 
+/// Create metasrv client instance and spawn heartbeat loop.
+pub async fn new_metasrv_client(
+    node_id: u64,
+    meta_config: &MetaClientOptions,
+) -> Result<MetaClient> {
+    let cluster_id = 0; // TODO(hl): read from config
+    let member_id = node_id;
+
+    let config = ChannelConfig::new()
+        .timeout(Duration::from_millis(meta_config.timeout_millis))
+        .connect_timeout(Duration::from_millis(meta_config.connect_timeout_millis))
+        .tcp_nodelay(meta_config.tcp_nodelay);
+    let channel_manager = ChannelManager::with_config(config);
+
+    let mut meta_client = MetaClientBuilder::new(cluster_id, member_id, Role::Datanode)
+        .enable_heartbeat()
+        .enable_router()
+        .enable_store()
+        .channel_manager(channel_manager)
+        .build();
+    meta_client
+        .start(&meta_config.metasrv_addrs)
+        .await
+        .context(MetaClientInitSnafu)?;
+
+    // required only when the heartbeat_client is enabled
+    meta_client
+        .ask_leader()
+        .await
+        .context(MetaClientInitSnafu)?;
+    Ok(meta_client)
+}
+
 #[cfg(test)]
 mod tests {
     #[test]