命令行

首先从服务启动的命令行开始说起：

./bin/yb-tserver --flagfile ./conf/local-tserver.conf

TServer服务启动分为单机版和集群版，配置文件集中在编译后包的conf目录：

conf
├── cluster-master-1.conf
├── cluster-master-2.conf
├── cluster-master-3.conf
├── cluster-tserver-1.conf
├── cluster-tserver-2.conf
├── cluster-tserver-3.conf
├── local-master.conf
└── local-tserver.conf

Yugabyte对命令行参数处理采用的是Google的GFlag库，具体可参看Google Flags使用说明。

打开单机版配置文件：

# local-tserver.conf
--tserver_master_addrs=127.0.0.1:7100
--rpc_bind_addresses=127.0.0.1:9100
--start_pgsql_proxy
--pgsql_proxy_bind_address=0.0.0.0:5433
--fs_data_dirs=/mnt/ssd
--ysql_enable_auth=true
--pg_yb_session_timeout_ms=120000
--ysql_num_shards_per_tserver=1

源码入口

首先找到main函数:

// yb/tserver/tablet_server_main.cc
int main(int argc, char** argv) {
  return yb::tserver::TabletServerMain(argc, argv);
}

继续：

// yb/tserver/tablet_server_main.cc
int TabletServerMain(int argc, char** argv) {
  // Reset some default values before parsing gflags.
  FLAGS_rpc_bind_addresses = strings::Substitute("0.0.0.0:$0",
                                                 TabletServer::kDefaultPort);
  FLAGS_webserver_port = TabletServer::kDefaultWebPort;
  FLAGS_redis_proxy_webserver_port = RedisServer::kDefaultWebPort;
  FLAGS_cql_proxy_webserver_port = CQLServer::kDefaultWebPort;

  ...
  ...
  ...

  ParseCommandLineFlags(&argc, &argv, true);
  if (argc != 1) {
    std::cerr << "usage: " << argv[0] << std::endl;
    return 1;
  }

  ...
  ...
  ...

  return 0;
}

服务启动

找到了程序入口，接下来看看TabletServerMain函数里主要做了什么。

参数初始化及命令行参数解析

参数初始化

// yb/tserver/tablet_server_main.cc
  // Reset some default values before parsing gflags.
  FLAGS_rpc_bind_addresses = strings::Substitute("0.0.0.0:$0",
                                                 TabletServer::kDefaultPort);
  FLAGS_webserver_port = TabletServer::kDefaultWebPort;
  FLAGS_redis_proxy_webserver_port = RedisServer::kDefaultWebPort;
  FLAGS_cql_proxy_webserver_port = CQLServer::kDefaultWebPort;

  string host_name;
  if (GetHostname(&host_name).ok()) {
    FLAGS_metric_node_name = strings::Substitute("$0:$1", host_name, TabletServer::kDefaultWebPort);
  } else {
    LOG(INFO) << "Failed to get tablet's host name, keeping default metric_node_name";
  }

解析命令行参数

// yb/tserver/tablet_server_main.cc
  ParseCommandLineFlags(&argc, &argv, true);
  if (argc != 1) {
    std::cerr << "usage: " << argv[0] << std::endl;
    return 1;
  }
  LOG_AND_RETURN_FROM_MAIN_NOT_OK(log::ModifyDurableWriteFlagIfNotODirect());
  LOG_AND_RETURN_FROM_MAIN_NOT_OK(InitYB(TabletServerOptions::kServerType, argv[0]));

时钟服务注册

// yb/tserver/tablet_server_main.cc
server::SkewedClock::Register();

Yugabyte采用混合逻辑时钟（HLC）的方式来解决事务控制中的时钟同步问题。

创建并启动tablet服务

// yb/tserver/tablet_server_main.cc
  auto tablet_server_options = TabletServerOptions::CreateTabletServerOptions();
  LOG_AND_RETURN_FROM_MAIN_NOT_OK(tablet_server_options);
  enterprise::Factory factory;

  auto server = factory.CreateTabletServer(*tablet_server_options);

  // ----------------------------------------------------------------------------------------------
  // Starting to instantiate servers
  // ----------------------------------------------------------------------------------------------

  LOG(INFO) << "Initializing tablet server...";
  LOG_AND_RETURN_FROM_MAIN_NOT_OK(server->Init());
  LOG(INFO) << "Starting tablet server...";
  UlimitUtil::InitUlimits();
  LOG(INFO) << "ulimit cur(max)..." << UlimitUtil::GetUlimitInfo();
  LOG_AND_RETURN_FROM_MAIN_NOT_OK(server->Start());
  LOG(INFO) << "Tablet server successfully started.";

  std::unique_ptr<CallHome> call_home;
  call_home = std::make_unique<CallHome>(server.get(), ServerType::TSERVER);
  call_home->ScheduleCallHome();

Yugabyte是基于Tablet的分布式存储，支持基于hash和range的分片策略。

创建后台postgres服务

// yb/tserver/tablet_server_main.cc
  std::unique_ptr<PgSupervisor> pg_supervisor;
  if (FLAGS_start_pgsql_proxy || FLAGS_enable_ysql) {
    auto pg_process_conf_result = PgProcessConf::CreateValidateAndRunInitDb(
        FLAGS_pgsql_proxy_bind_address,
        tablet_server_options->fs_opts.data_paths.front() + "/pg_data",
        server->GetSharedMemoryFd());
    LOG_AND_RETURN_FROM_MAIN_NOT_OK(pg_process_conf_result);
    auto& pg_process_conf = *pg_process_conf_result;
    pg_process_conf.master_addresses = tablet_server_options->master_addresses_flag;
    pg_process_conf.certs_dir = FLAGS_certs_dir.empty()
        ? server::DefaultCertsDir(*server->fs_manager())
        : FLAGS_certs_dir;
    pg_process_conf.certs_for_client_dir = FLAGS_certs_for_client_dir.empty()
        ? pg_process_conf.certs_dir
        : FLAGS_certs_for_client_dir;
    pg_process_conf.enable_tls = FLAGS_use_client_to_server_encryption;

    // Follow the same logic as elsewhere, check FLAGS_cert_node_filename then
    // server_broadcast_addresses then rpc_bind_addresses.
    if (!FLAGS_cert_node_filename.empty()) {
      pg_process_conf.cert_base_name = FLAGS_cert_node_filename;
    } else {
      const auto server_broadcast_addresses =
          HostPort::ParseStrings(server->options().server_broadcast_addresses, 0);
      LOG_AND_RETURN_FROM_MAIN_NOT_OK(server_broadcast_addresses);
      const auto rpc_bind_addresses =
          HostPort::ParseStrings(server->options().rpc_opts.rpc_bind_addresses, 0);
      LOG_AND_RETURN_FROM_MAIN_NOT_OK(rpc_bind_addresses);
      pg_process_conf.cert_base_name = !server_broadcast_addresses->empty()
                                     ? server_broadcast_addresses->front().host()
                                     : rpc_bind_addresses->front().host();
    }
    LOG(INFO) << "Starting PostgreSQL server listening on "
              << pg_process_conf.listen_addresses << ", port " << pg_process_conf.pg_port;

    pg_supervisor = std::make_unique<PgSupervisor>(pg_process_conf);
    LOG_AND_RETURN_FROM_MAIN_NOT_OK(pg_supervisor->Start());
  }

创建redis服务

// yb/tserver/tablet_server_main.cc
  std::unique_ptr<RedisServer> redis_server;
  if (FLAGS_start_redis_proxy) {
    RedisServerOptions redis_server_options;
    redis_server_options.rpc_opts.rpc_bind_addresses = FLAGS_redis_proxy_bind_address;
    redis_server_options.webserver_opts.port = FLAGS_redis_proxy_webserver_port;
    redis_server_options.master_addresses_flag = tablet_server_options->master_addresses_flag;
    redis_server_options.SetMasterAddresses(tablet_server_options->GetMasterAddresses());
    redis_server_options.dump_info_path =
        (tablet_server_options->dump_info_path.empty()
             ? ""
             : tablet_server_options->dump_info_path + "-redis");
    redis_server.reset(new RedisServer(redis_server_options, server.get()));
    LOG(INFO) << "Starting redis server...";
    LOG_AND_RETURN_FROM_MAIN_NOT_OK(redis_server->Start());
    LOG(INFO) << "Redis server successfully started.";
  }

创建cql服务

// yb/tserver/tablet_server_main.cc
  // TODO(neil): After CQL server is starting, it blocks this thread from moving on.
  // This should be fixed such that all processes or service by tablet server are treated equally
  // by using different threads for each process.
  std::unique_ptr<CQLServer> cql_server;
  if (FLAGS_start_cql_proxy) {
    CQLServerOptions cql_server_options;
    cql_server_options.rpc_opts.rpc_bind_addresses = FLAGS_cql_proxy_bind_address;
    cql_server_options.broadcast_rpc_address = FLAGS_cql_proxy_broadcast_rpc_address;
    cql_server_options.webserver_opts.port = FLAGS_cql_proxy_webserver_port;
    cql_server_options.master_addresses_flag = tablet_server_options->master_addresses_flag;
    cql_server_options.SetMasterAddresses(tablet_server_options->GetMasterAddresses());
    cql_server_options.dump_info_path =
        (tablet_server_options->dump_info_path.empty()
             ? ""
             : tablet_server_options->dump_info_path + "-cql");
    boost::asio::io_service io;
    cql_server = factory.CreateCQLServer(cql_server_options, &io, server.get());
    LOG(INFO) << "Starting CQL server...";
    LOG_AND_RETURN_FROM_MAIN_NOT_OK(cql_server->Start());
    LOG(INFO) << "CQL server successfully started.";

    // Should run forever unless there are some errors.
    boost::system::error_code ec;
    io.run(ec);
    if (ec) {
      LOG(WARNING) << "IO service run failure: " << ec;
    }

    LOG (WARNING) << "CQL Server shutting down";
    cql_server->Shutdown();
  }

Yugabyte支持SQLhe CQL协议。

后续会继续跟踪代码对上述服务进行阅读。