more logging & fix signature bug

integration-tests/deployment/clo/don_nodeset_test.go

@@ -20,34 +20,27 @@ import (
 // sufficient for testing
 var (
  writerFilter = func(n *models.Node) bool {
- return strings.Contains(n.Name, "Cap One") && !strings.Contains(n.Name, "Boot")
+ return strings.Contains(n.Name, "Prod Keystone Cap One") && !strings.Contains(n.Name, "Boot")
  }
 
  assetFilter = func(n *models.Node) bool {
- return strings.Contains(n.Name, "DF 2 stage testnet") && !strings.Contains(n.Name, "Bootstrap")
+ return strings.Contains(n.Name, "Prod Keystone Asset") && !strings.Contains(n.Name, "Bootstrap")
  }
 
  wfFilter = func(n *models.Node) bool {
- return strings.Contains(n.Name, "Keystone One") && !strings.Contains(n.Name, "Boot")
- }
-
- aptosFilter = func(n *models.Node) bool {
- return strings.Contains(n.Name, "Keystone Aptos")
+ return strings.Contains(n.Name, "Prod Keystone One") && !strings.Contains(n.Name, "Boot")
  }
 )
 
 func TestGenerateNopNodesData(t *testing.T) {
- //t.Skipf("this test is for generating test data only")
+ t.Skipf("this test is for generating test data only")
  // use for generating keystone deployment test data
  // `./bin/fmscli --config ~/.fmsclient/prod.yaml login`
  // `./bin/fmscli --config ~/.fmsclient/prod.yaml get nodeOperators > /tmp/all-clo-nops.json`
 
- regenerateFromCLO := true
- d := "/tmp"
- target := filepath.Join(d, "keystone_nops.json")
+ regenerateFromCLO := false
  if regenerateFromCLO {
- path := "/tmp/clo-staging-nops.json"
-
+ path := "/tmp/all-clo-nops.json"
  f, err := os.ReadFile(path)
  require.NoError(t, err)
  type cloData struct {
@@ -62,27 +55,25 @@ func TestGenerateNopNodesData(t *testing.T) {
  })
 
  ksFilter := func(n *models.Node) bool {
- return writerFilter(n) || assetFilter(n) || wfFilter(n) || aptosFilter(n)
+ return writerFilter(n) || assetFilter(n) || wfFilter(n)
  }
  ksNops := clo.FilterNopNodes(allNops, ksFilter)
  require.NotEmpty(t, ksNops)
  b, err := json.MarshalIndent(ksNops, "", " ")
  require.NoError(t, err)
- require.NoError(t, os.WriteFile(target, b, 0644)) // nolint: gosec
+ require.NoError(t, os.WriteFile("testdata/keystone_nops.json", b, 0644)) // nolint: gosec
  }
- keystoneNops := loadTestNops(t, target)
+ keystoneNops := loadTestNops(t, "testdata/keystone_nops.json")
 
  m := clo.CapabilityNodeSets(keystoneNops, map[string]clo.FilterFuncT[*models.Node]{
  "workflow": wfFilter,
  "chainWriter": writerFilter,
  "asset": assetFilter,
- "aptos": aptosFilter,
  })
- assert.Len(t, m, 4)
- assert.Len(t, m["workflow"], 7)
- assert.Len(t, m["chainWriter"], 4)
- assert.Len(t, m["asset"], 7)
- assert.Len(t, m["aptos"], 3)
+ assert.Len(t, m, 3)
+ assert.Len(t, m["workflow"], 10)
+ assert.Len(t, m["chainWriter"], 10)
+ assert.Len(t, m["asset"], 16)
 
  // can be used to derive the test data for the keystone deployment
  updateTestData := true
@@ -98,9 +89,6 @@ func TestGenerateNopNodesData(t *testing.T) {
  b, err = json.MarshalIndent(m["asset"], "", " ")
  require.NoError(t, err)
  require.NoError(t, os.WriteFile(filepath.Join(d, "asset_nodes.json"), b, 0600))
- b, err = json.MarshalIndent(m["aptos"], "", " ")
- require.NoError(t, err)
- require.NoError(t, os.WriteFile(filepath.Join(d, "aptos_nodes.json"), b, 0600))
  }
 }
 

integration-tests/deployment/keystone/deploy.go

@@ -267,7 +267,7 @@ func ConfigureForwardContracts(env *deployment.Environment, dons []RegisteredDon
  return fmt.Errorf("no forwarder contract found for chain %d", chain.Selector)
  }
 
- err := configureForwarder(chain, fwrd, dons)
+ err := configureForwarder(env.Logger, chain, fwrd, dons)
  if err != nil {
  return fmt.Errorf("failed to configure forwarder for chain selector %d: %w", chain.Selector, err)
  }
@@ -675,7 +675,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
  // track hash of sorted p2pids to don name because the registry return value does not include the don name
  // and we need to map it back to the don name to access the other mapping data such as the don's capabilities & nodes
  p2pIdsToDon := make(map[string]string)
- var registeredDonCnt = 0
+ var registeredDons = 0
 
  for don, ocr2nodes := range req.donToOcr2Nodes {
  var p2pIds [][32]byte
@@ -725,18 +725,20 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
  return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don, err)
  }
  lggr.Debugw("registered DON", "don", don, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", f)
- registeredDonCnt++
+ registeredDons++
  }
- lggr.Debugf("Registered all DONS %d, waiting for registry to update", registeredDonCnt)
- // todo real retry with backoff
+ lggr.Debugf("Registered all DONS %d, waiting for registry to update", registeredDons)
+
+ // occasionally the registry does not return the expected number of DONS, so we retry a few times
+ // while crude, it is effective
  var donInfos []capabilities_registry.CapabilitiesRegistryDONInfo
  var err error
  for i := 0; i < 10; i++ {
  lggr.Debug("attempting to get DONS from registry", i)
  donInfos, err = req.registry.GetDONs(&bind.CallOpts{})
- if len(donInfos) != registeredDonCnt {
- lggr.Debugw("expected dons not registered", "expected", registeredDonCnt, "got", len(donInfos))
- time.Sleep(4 * time.Second)
+ if len(donInfos) != registeredDons {
+ lggr.Debugw("expected dons not registered", "expected", registeredDons, "got", len(donInfos))
+ time.Sleep(2 * time.Second)
  } else {
  break
  }
@@ -745,9 +747,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
  err = DecodeErr(kcr.CapabilitiesRegistryABI, err)
  return nil, fmt.Errorf("failed to call GetDONs: %w", err)
  }
- if len(donInfos) != registeredDonCnt {
- return nil, fmt.Errorf("expected %d dons, got %d", registeredDonCnt, len(donInfos))
- }
+
  for i, donInfo := range donInfos {
  donName, ok := p2pIdsToDon[sortedHash(donInfo.NodeP2PIds)]
  if !ok {
@@ -756,19 +756,16 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
  lggr.Debugw("adding don info", "don", donName, "cnt", i)
  resp.donInfos[donName] = donInfos[i]
  }
- lggr.Debugw("registered DONS", "count", len(resp.donInfos))
- if len(resp.donInfos) != len(donInfos) {
- return nil, fmt.Errorf("mismatch len after loop got %d want %d", len(resp.donInfos), len(donInfos))
- }
- if len(resp.donInfos) != registeredDonCnt {
- return nil, fmt.Errorf("expected %d dons, got %d", registeredDonCnt, len(resp.donInfos))
+ lggr.Debugw("found registered DONs", "count", len(resp.donInfos))
+ if len(resp.donInfos) != registeredDons {
+ return nil, fmt.Errorf("expected %d dons, got %d", registeredDons, len(resp.donInfos))
  }
  return &resp, nil
 }
 
 // configureForwarder sets the config for the forwarder contract on the chain for all Dons that accept workflows
 // dons that don't accept workflows are not registered with the forwarder
-func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error {
+func configureForwarder(lggr logger.Logger, chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error {
  if fwdr == nil {
  return errors.New("nil forwarder contract")
  }
@@ -787,6 +784,7 @@ func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons
  err = DecodeErr(kf.KeystoneForwarderABI, err)
  return fmt.Errorf("failed to confirm SetConfig for forwarder %s: %w", fwdr.Address().String(), err)
  }
+ lggr.Debugw("configured forwarder", "forwarder", fwdr.Address().String(), "donId", dn.Info.Id, "version", ver, "f", dn.Info.F, "signers", dn.signers())
  }
  return nil
 }

integration-tests/deployment/keystone/types.go

@@ -70,7 +70,8 @@ type ocr2Node struct {
 }
 
 func (o *ocr2Node) signerAddress() common.Address {
- return common.BytesToAddress(o.Signer[:])
+ // eth address is the first 20 bytes of the Signer
+ return common.BytesToAddress(o.Signer[:20])
 }
 
 func (o *ocr2Node) toNodeKeys() NodeKeys {
@@ -128,6 +129,10 @@ func newOcr2Node(id string, ccfgs map[chaintype.ChainType]*v1.ChainConfig, csaPu
  }
 
  var sigb [32]byte
+ // initialize sigb to 0s
+ for i := range sigb {
+ sigb[i] = 0
+ }
  copy(sigb[:], signerB)
 
  n := &ocr2Node{