Support raw config of etcd (#9)

* [options]: Support raw config of etcd * [options]: check if peer url exists * [options]: Fix typo & make manual cleaner * [options]: Add newline
xxx7xxxx · Jun 1, 2021 · c57aaff · c57aaff
1 parent 723c9a6
commit c57aaff
Show file tree

Hide file tree

Showing 12 changed files with 160 additions and 84 deletions.
diff --git a/example/reader-004/conf/config.yaml b/example/reader-004/conf/config.yaml
@@ -1,9 +1,11 @@
 name: reader-004
 cluster-name: cluster-test
 cluster-role: reader
-cluster-client-url:
-cluster-peer-url:
-cluster-join-urls: http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380
+cluster-listen-client-urls:
+cluster-listen-peer-urls:
+cluster-advertise-client-urls:
+cluster-initial-advertise-peer-urls:
+cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380]
 api-addr: 127.0.0.1:42381
 data-dir: ./data
 wal-dir: ""

diff --git a/example/reader-005/conf/config.yaml b/example/reader-005/conf/config.yaml
@@ -1,9 +1,11 @@
 name: reader-005
-cluster-name: cluster-A
+cluster-name: cluster-test
 cluster-role: reader
-cluster-client-url:
-cluster-peer-url:
-cluster-join-urls: http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380
+cluster-listen-client-urls:
+cluster-listen-peer-urls:
+cluster-advertise-client-urls:
+cluster-initial-advertise-peer-urls:
+cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380]
 api-addr: 127.0.0.1:52381
 data-dir: ./data
 wal-dir: ""

diff --git a/example/writer-001/conf/config.yaml b/example/writer-001/conf/config.yaml
@@ -1,10 +1,11 @@
 name: writer-001
-labels: {mesh_role: worker}
 cluster-name: cluster-test
 cluster-role: writer
-cluster-client-url: http://127.0.0.1:12379
-cluster-peer-url: http://127.0.0.1:12380
-cluster-join-urls: http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380
+cluster-listen-client-urls: [http://127.0.0.1:12379]
+cluster-listen-peer-urls: [http://127.0.0.1:12380]
+cluster-advertise-client-urls: [http://127.0.0.1:12379]
+cluster-initial-advertise-peer-urls: [http://127.0.0.1:12380]
+cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380]
 api-addr: 127.0.0.1:12381
 data-dir: ./data
 wal-dir: ""

diff --git a/example/writer-002/conf/config.yaml b/example/writer-002/conf/config.yaml
@@ -1,9 +1,11 @@
 name: writer-002
 cluster-name: cluster-test
 cluster-role: writer
-cluster-client-url: http://127.0.0.1:22379
-cluster-peer-url: http://127.0.0.1:22380
-cluster-join-urls: http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380
+cluster-listen-client-urls: [http://127.0.0.1:22379]
+cluster-listen-peer-urls: [http://127.0.0.1:22380]
+cluster-advertise-client-urls: [http://127.0.0.1:22379]
+cluster-initial-advertise-peer-urls: [http://127.0.0.1:22380]
+cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380]
 api-addr: 127.0.0.1:22381
 data-dir: ./data
 wal-dir: ""

diff --git a/example/writer-003/conf/config.yaml b/example/writer-003/conf/config.yaml
@@ -1,12 +1,11 @@
 name: writer-003
-# The different cluster-name is about to test
-# joining a existed cluster with a different name
-# will panic and exit as expected.
 cluster-name: cluster-test
 cluster-role: writer
-cluster-client-url: http://127.0.0.1:32379
-cluster-peer-url: http://127.0.0.1:32380
-cluster-join-urls: http://127.0.0.1:12380,http://127.0.0.1:22380,http://127.0.0.1:32380
+cluster-listen-client-urls: [http://127.0.0.1:32379]
+cluster-listen-peer-urls: [http://127.0.0.1:32380]
+cluster-advertise-client-urls: [http://127.0.0.1:32379]
+cluster-initial-advertise-peer-urls: [http://127.0.0.1:32380]
+cluster-join-urls: [http://127.0.0.1:12380, http://127.0.0.1:22380, http://127.0.0.1:32380]
 api-addr: 127.0.0.1:32381
 data-dir: ./data
 wal-dir: ""

diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
@@ -316,7 +316,7 @@ func (c *cluster) addSelfToCluster() error {
  return err
  }
 
- respAdd, err := client.MemberAdd(c.requestContext(), []string{c.opt.ClusterPeerURL})
+ respAdd, err := client.MemberAdd(c.requestContext(), c.opt.ClusterInitialAdvertisePeerURLs)
  if err != nil {
  return fmt.Errorf("add member failed: %v", err)
  }
@@ -667,7 +667,7 @@ func (c *cluster) defrag() {
  }
 
  // NOTICE: It need longer time than normal ones.
- _, err = client.Defragment(c.longRequestContext(), c.opt.ClusterPeerURL)
+ _, err = client.Defragment(c.longRequestContext(), c.opt.ClusterAdvertiseClientURLs[0])
  if err != nil {
  defragInterval = defragFailedInterval
  logger.Errorf("defrag failed: %v", err)

diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go
@@ -36,7 +36,7 @@ func mockClusters(count int) []*cluster {
  time.Sleep(HeartbeatInterval)
 
  for i := 1; i < count; i++ {
- opts[i].ClusterJoinURLs = []string{opts[0].ClusterPeerURL}
+ opts[i].ClusterJoinURLs = opts[0].ClusterListenPeerURLs
  cls, err := New(opts[i])
  if err != nil {
  panic(fmt.Errorf("new cluster failed: %v", err))

diff --git a/pkg/cluster/config.go b/pkg/cluster/config.go
@@ -51,14 +51,39 @@ func (c *cluster) prepareEtcdConfig() (*embed.Config, error) {
  ec := embed.NewConfig()
  opt := c.opt
 
- peerURL, err := url.Parse(opt.ClusterPeerURL)
- if err != nil {
- return nil, err
+ var (
+ clientURLs []url.URL
+ peerURLs []url.URL
+ clientAdURLs []url.URL
+ peerAdURLs []url.URL
+ )
+ for _, u := range opt.ClusterListenClientURLs {
+ clientURL, err := url.Parse(u)
+ if err != nil {
+ return nil, err
+ }
+ clientURLs = append(clientURLs, *clientURL)
  }
-
- clientURL, err := url.Parse(opt.ClusterClientURL)
- if err != nil {
- return nil, err
+ for _, u := range opt.ClusterListenPeerURLs {
+ peerURL, err := url.Parse(u)
+ if err != nil {
+ return nil, err
+ }
+ peerURLs = append(peerURLs, *peerURL)
+ }
+ for _, u := range opt.ClusterAdvertiseClientURLs {
+ clientAdURL, err := url.Parse(u)
+ if err != nil {
+ return nil, err
+ }
+ clientAdURLs = append(clientAdURLs, *clientAdURL)
+ }
+ for _, u := range opt.ClusterInitialAdvertisePeerURLs {
+ peerAdURL, err := url.Parse(u)
+ if err != nil {
+ return nil, err
+ }
+ peerAdURLs = append(peerAdURLs, *peerAdURL)
  }
 
  ec.Name = opt.Name
@@ -67,10 +92,10 @@ func (c *cluster) prepareEtcdConfig() (*embed.Config, error) {
  ec.WalDir = opt.AbsWALDir
  ec.InitialClusterToken = opt.ClusterName
  ec.EnableV2 = false
- ec.LPUrls = []url.URL{*peerURL}
- ec.APUrls = []url.URL{*peerURL}
- ec.LCUrls = []url.URL{*clientURL}
- ec.ACUrls = []url.URL{*clientURL}
+ ec.LCUrls = clientURLs
+ ec.ACUrls = clientAdURLs
+ ec.LPUrls = peerURLs
+ ec.APUrls = peerAdURLs
  ec.AutoCompactionMode = autoCompactionMode
  ec.AutoCompactionRetention = autoCompactionRetention
  ec.QuotaBackendBytes = quotaBackendBytes

diff --git a/pkg/cluster/member.go b/pkg/cluster/member.go
@@ -75,10 +75,10 @@ func newMembers(opt *option.Options) (*members, error) {
  }
 
  initMS := make(membersSlice, 0)
- if opt.ClusterPeerURL != "" {
+ if opt.ClusterRole == "writer" && len(opt.ClusterInitialAdvertisePeerURLs) != 0 {
  initMS = append(initMS, &member{
  Name: opt.Name,
- PeerURL: opt.ClusterPeerURL,
+ PeerURL: opt.ClusterInitialAdvertisePeerURLs[0],
  })
  }
  m.ClusterMembers.update(initMS)
@@ -179,9 +179,15 @@ func (m *members) _self() *member {
  logger.Errorf("BUG: can't get self from cluster members: %s "+
  "knownMembers: %s", m.ClusterMembers, m.KnownMembers)
  }
+
+ peerURL := ""
+ if len(m.opt.ClusterInitialAdvertisePeerURLs) != 0 {
+ peerURL = m.opt.ClusterInitialAdvertisePeerURLs[0]
+ }
+
  return &member{
  Name: m.opt.Name,
- PeerURL: m.opt.ClusterPeerURL,
+ PeerURL: peerURL,
  }
 }
 

diff --git a/pkg/cluster/member_test.go b/pkg/cluster/member_test.go
@@ -67,8 +67,10 @@ func mockTestOpt() *option.Options {
  opt.ClusterName = "test-cluster"
  opt.ClusterRole = "writer"
  opt.ClusterRequestTimeout = "10s"
- opt.ClusterClientURL = fmt.Sprintf("http://localhost:%d", ports[0])
- opt.ClusterPeerURL = fmt.Sprintf("http://localhost:%d", ports[1])
+ opt.ClusterListenClientURLs = []string{fmt.Sprintf("http://localhost:%d", ports[0])}
+ opt.ClusterAdvertiseClientURLs = opt.ClusterListenClientURLs
+ opt.ClusterListenPeerURLs = []string{fmt.Sprintf("http://localhost:%d", ports[1])}
+ opt.ClusterInitialAdvertisePeerURLs = opt.ClusterListenPeerURLs
  opt.APIAddr = fmt.Sprintf("localhost:%d", ports[2])
  opt.HomeDir = filepath.Join(tempDir, name)
  opt.DataDir = "data"
@@ -97,13 +99,13 @@ func mockMembers(count int) ([]*option.Options, membersSlice, []*pb.Member) {
  members[i] = &member{
  ID: id,
  Name: opt.Name,
- PeerURL: opt.ClusterPeerURL,
+ PeerURL: opt.ClusterInitialAdvertisePeerURLs[0],
  }
  pbMembers[i] = &pb.Member{
  ID: id,
  Name: opt.Name,
- PeerURLs: []string{opt.ClusterPeerURL},
- ClientURLs: []string{opt.ClusterClientURL},
+ PeerURLs: []string{opt.ClusterInitialAdvertisePeerURLs[0]},
+ ClientURLs: []string{opt.ClusterAdvertiseClientURLs[0]},
  }
 
  env.InitServerDir(opt)

diff --git a/pkg/common/utils.go b/pkg/common/utils.go
@@ -246,7 +246,7 @@ func ExpandDir(dir string) string {
 
 // MkdirAll wraps os.MakeAll with fixed perm.
 func MkdirAll(path string) error {
- return os.MkdirAll(ExpandDir(path), 0750)
+ return os.MkdirAll(ExpandDir(path), 0700)
 }
 
 // RemoveAll wraps os.RemoveAll.