From d770c08dc83a228748ca2dc355657b25a85a2920 Mon Sep 17 00:00:00 2001 From: krehermann <16602512+krehermann@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:53:00 -0600 Subject: [PATCH] more logging & fix signature bug --- .../deployment/clo/don_nodeset_test.go | 38 +++++++------------ .../deployment/keystone/deploy.go | 34 ++++++++--------- .../deployment/keystone/types.go | 7 +++- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/integration-tests/deployment/clo/don_nodeset_test.go b/integration-tests/deployment/clo/don_nodeset_test.go index e33cf41a5b5..6e8c64060df 100644 --- a/integration-tests/deployment/clo/don_nodeset_test.go +++ b/integration-tests/deployment/clo/don_nodeset_test.go @@ -20,34 +20,27 @@ import ( // sufficient for testing var ( writerFilter = func(n *models.Node) bool { - return strings.Contains(n.Name, "Cap One") && !strings.Contains(n.Name, "Boot") + return strings.Contains(n.Name, "Prod Keystone Cap One") && !strings.Contains(n.Name, "Boot") } assetFilter = func(n *models.Node) bool { - return strings.Contains(n.Name, "DF 2 stage testnet") && !strings.Contains(n.Name, "Bootstrap") + return strings.Contains(n.Name, "Prod Keystone Asset") && !strings.Contains(n.Name, "Bootstrap") } wfFilter = func(n *models.Node) bool { - return strings.Contains(n.Name, "Keystone One") && !strings.Contains(n.Name, "Boot") - } - - aptosFilter = func(n *models.Node) bool { - return strings.Contains(n.Name, "Keystone Aptos") + return strings.Contains(n.Name, "Prod Keystone One") && !strings.Contains(n.Name, "Boot") } ) func TestGenerateNopNodesData(t *testing.T) { - //t.Skipf("this test is for generating test data only") + t.Skipf("this test is for generating test data only") // use for generating keystone deployment test data // `./bin/fmscli --config ~/.fmsclient/prod.yaml login` // `./bin/fmscli --config ~/.fmsclient/prod.yaml get nodeOperators > /tmp/all-clo-nops.json` - regenerateFromCLO := true - d := "/tmp" - target := filepath.Join(d, "keystone_nops.json") + regenerateFromCLO := false if regenerateFromCLO { - path := "/tmp/clo-staging-nops.json" - + path := "/tmp/all-clo-nops.json" f, err := os.ReadFile(path) require.NoError(t, err) type cloData struct { @@ -62,27 +55,25 @@ func TestGenerateNopNodesData(t *testing.T) { }) ksFilter := func(n *models.Node) bool { - return writerFilter(n) || assetFilter(n) || wfFilter(n) || aptosFilter(n) + return writerFilter(n) || assetFilter(n) || wfFilter(n) } ksNops := clo.FilterNopNodes(allNops, ksFilter) require.NotEmpty(t, ksNops) b, err := json.MarshalIndent(ksNops, "", " ") require.NoError(t, err) - require.NoError(t, os.WriteFile(target, b, 0644)) // nolint: gosec + require.NoError(t, os.WriteFile("testdata/keystone_nops.json", b, 0644)) // nolint: gosec } - keystoneNops := loadTestNops(t, target) + keystoneNops := loadTestNops(t, "testdata/keystone_nops.json") m := clo.CapabilityNodeSets(keystoneNops, map[string]clo.FilterFuncT[*models.Node]{ "workflow": wfFilter, "chainWriter": writerFilter, "asset": assetFilter, - "aptos": aptosFilter, }) - assert.Len(t, m, 4) - assert.Len(t, m["workflow"], 7) - assert.Len(t, m["chainWriter"], 4) - assert.Len(t, m["asset"], 7) - assert.Len(t, m["aptos"], 3) + assert.Len(t, m, 3) + assert.Len(t, m["workflow"], 10) + assert.Len(t, m["chainWriter"], 10) + assert.Len(t, m["asset"], 16) // can be used to derive the test data for the keystone deployment updateTestData := true @@ -98,9 +89,6 @@ func TestGenerateNopNodesData(t *testing.T) { b, err = json.MarshalIndent(m["asset"], "", " ") require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(d, "asset_nodes.json"), b, 0600)) - b, err = json.MarshalIndent(m["aptos"], "", " ") - require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(d, "aptos_nodes.json"), b, 0600)) } } diff --git a/integration-tests/deployment/keystone/deploy.go b/integration-tests/deployment/keystone/deploy.go index 5d2655cb2e4..101f6ea58e0 100644 --- a/integration-tests/deployment/keystone/deploy.go +++ b/integration-tests/deployment/keystone/deploy.go @@ -267,7 +267,7 @@ func ConfigureForwardContracts(env *deployment.Environment, dons []RegisteredDon return fmt.Errorf("no forwarder contract found for chain %d", chain.Selector) } - err := configureForwarder(chain, fwrd, dons) + err := configureForwarder(env.Logger, chain, fwrd, dons) if err != nil { return fmt.Errorf("failed to configure forwarder for chain selector %d: %w", chain.Selector, err) } @@ -675,7 +675,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes // track hash of sorted p2pids to don name because the registry return value does not include the don name // and we need to map it back to the don name to access the other mapping data such as the don's capabilities & nodes p2pIdsToDon := make(map[string]string) - var registeredDonCnt = 0 + var registeredDons = 0 for don, ocr2nodes := range req.donToOcr2Nodes { var p2pIds [][32]byte @@ -725,18 +725,20 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don, err) } lggr.Debugw("registered DON", "don", don, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", f) - registeredDonCnt++ + registeredDons++ } - lggr.Debugf("Registered all DONS %d, waiting for registry to update", registeredDonCnt) - // todo real retry with backoff + lggr.Debugf("Registered all DONS %d, waiting for registry to update", registeredDons) + + // occasionally the registry does not return the expected number of DONS, so we retry a few times + // while crude, it is effective var donInfos []capabilities_registry.CapabilitiesRegistryDONInfo var err error for i := 0; i < 10; i++ { lggr.Debug("attempting to get DONS from registry", i) donInfos, err = req.registry.GetDONs(&bind.CallOpts{}) - if len(donInfos) != registeredDonCnt { - lggr.Debugw("expected dons not registered", "expected", registeredDonCnt, "got", len(donInfos)) - time.Sleep(4 * time.Second) + if len(donInfos) != registeredDons { + lggr.Debugw("expected dons not registered", "expected", registeredDons, "got", len(donInfos)) + time.Sleep(2 * time.Second) } else { break } @@ -745,9 +747,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes err = DecodeErr(kcr.CapabilitiesRegistryABI, err) return nil, fmt.Errorf("failed to call GetDONs: %w", err) } - if len(donInfos) != registeredDonCnt { - return nil, fmt.Errorf("expected %d dons, got %d", registeredDonCnt, len(donInfos)) - } + for i, donInfo := range donInfos { donName, ok := p2pIdsToDon[sortedHash(donInfo.NodeP2PIds)] if !ok { @@ -756,19 +756,16 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes lggr.Debugw("adding don info", "don", donName, "cnt", i) resp.donInfos[donName] = donInfos[i] } - lggr.Debugw("registered DONS", "count", len(resp.donInfos)) - if len(resp.donInfos) != len(donInfos) { - return nil, fmt.Errorf("mismatch len after loop got %d want %d", len(resp.donInfos), len(donInfos)) - } - if len(resp.donInfos) != registeredDonCnt { - return nil, fmt.Errorf("expected %d dons, got %d", registeredDonCnt, len(resp.donInfos)) + lggr.Debugw("found registered DONs", "count", len(resp.donInfos)) + if len(resp.donInfos) != registeredDons { + return nil, fmt.Errorf("expected %d dons, got %d", registeredDons, len(resp.donInfos)) } return &resp, nil } // configureForwarder sets the config for the forwarder contract on the chain for all Dons that accept workflows // dons that don't accept workflows are not registered with the forwarder -func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error { +func configureForwarder(lggr logger.Logger, chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error { if fwdr == nil { return errors.New("nil forwarder contract") } @@ -787,6 +784,7 @@ func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons err = DecodeErr(kf.KeystoneForwarderABI, err) return fmt.Errorf("failed to confirm SetConfig for forwarder %s: %w", fwdr.Address().String(), err) } + lggr.Debugw("configured forwarder", "forwarder", fwdr.Address().String(), "donId", dn.Info.Id, "version", ver, "f", dn.Info.F, "signers", dn.signers()) } return nil } diff --git a/integration-tests/deployment/keystone/types.go b/integration-tests/deployment/keystone/types.go index 562a87de3b4..65eff0b9e21 100644 --- a/integration-tests/deployment/keystone/types.go +++ b/integration-tests/deployment/keystone/types.go @@ -70,7 +70,8 @@ type ocr2Node struct { } func (o *ocr2Node) signerAddress() common.Address { - return common.BytesToAddress(o.Signer[:]) + // eth address is the first 20 bytes of the Signer + return common.BytesToAddress(o.Signer[:20]) } func (o *ocr2Node) toNodeKeys() NodeKeys { @@ -128,6 +129,10 @@ func newOcr2Node(id string, ccfgs map[chaintype.ChainType]*v1.ChainConfig, csaPu } var sigb [32]byte + // initialize sigb to 0s + for i := range sigb { + sigb[i] = 0 + } copy(sigb[:], signerB) n := &ocr2Node{