Skip to content

Commit

Permalink
gazctl shards prune: add test case for new prune behavior
Browse files Browse the repository at this point in the history
  • Loading branch information
@jgraettinger @psFried
jgraettinger authored and psFried committed Jan 31, 2024
1 parent 605c81e commit 4caca87
Showing 1 changed file with 282 additions and 0 deletions.
282 changes: 282 additions & 0 deletions cmd/gazctl/gazctlcmd/shards_prune_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,295 @@
package gazctlcmd

import (
"encoding/json"
"testing"

"github.com/stretchr/testify/require"
pb "go.gazette.dev/core/broker/protocol"
pc "go.gazette.dev/core/consumer/protocol"
"go.gazette.dev/core/consumer/recoverylog"
)

func TestFragmentsAreRetainedWhenAnyHintsRequireThem(t *testing.T) {
// The goal is to allow a shard to recover using any of the persisted
// hints. In order for that to work, all fragments that come after the final
// hinted segment of _any_ hints must be retained. In this case, the primary
// hints are quite old. So we test a fragment that would otherwise be elidgable
// for pruning, and assert that it is retained due to coming after the segment
// in the primary hints.
var fixture = `{
"header": {
"process_id": {
"zone": "us-central1-c",
"suffix": "flow-reactor-74fcd7f9db-n5c5j"
},
"route": {
"members": null,
"primary": -1
},
"etcd": {
"cluster_id": 9261053468663690000,
"member_id": 14358579373743376000,
"revision": 57121751,
"raft_term": 140
}
},
"primary_hints": {
"hints": {
"log": "example/log",
"live_nodes": [
{
"fnode": 1018,
"segments": [
{
"author": 3418428788,
"first_seq_no": 1018,
"first_offset": 129100,
"first_checksum": 2628272215,
"last_seq_no": 1024
}
]
}
],
"properties": null
}
},
"backup_hints": [
{
"hints": {
"log": "example/log",
"live_nodes": [
{
"fnode": 1109,
"segments": [
{
"author": 3418428788,
"first_seq_no": 1109,
"first_offset": 140706,
"first_checksum": 1095500179,
"last_seq_no": 1115,
"last_offset": 141535
}
]
},
{
"fnode": 1289,
"segments": [
{
"author": 2080950356,
"first_seq_no": 1289,
"first_offset": 152255,
"first_checksum": 777889612,
"last_seq_no": 1290,
"last_offset": 153297
}
]
},
{
"fnode": 1642,
"segments": [
{
"author": 245017003,
"first_seq_no": 1642,
"first_offset": 245137,
"first_checksum": 3605342194,
"last_seq_no": 1645,
"last_offset": 251592
}
]
},
{
"fnode": 1657,
"segments": [
{
"author": 3711033845,
"first_seq_no": 1657,
"first_offset": 252114,
"first_checksum": 794580610,
"last_seq_no": 1660,
"last_offset": 252379
}
]
},
{
"fnode": 1661,
"segments": [
{
"author": 3711033845,
"first_seq_no": 1661,
"first_offset": 252379,
"first_checksum": 157414297,
"last_seq_no": 1665,
"last_offset": 252574
}
]
},
{
"fnode": 1666,
"segments": [
{
"author": 3711033845,
"first_seq_no": 1666,
"first_offset": 252574,
"first_checksum": 157986885,
"last_seq_no": 1666,
"last_offset": 252610
}
]
},
{
"fnode": 1670,
"segments": [
{
"author": 3711033845,
"first_seq_no": 1670,
"first_offset": 252737,
"first_checksum": 1747582972,
"last_seq_no": 1673,
"last_offset": 259192
}
]
}
],
"properties": [
{
"path": "/IDENTITY",
"content": "0a4d4620-a93f-468a-9c12-26de543261f0\n"
}
]
}
},
{
"hints": {
"log": "example/log",
"live_nodes": [
{
"fnode": 1109,
"segments": [
{
"author": 3418428788,
"first_seq_no": 1109,
"first_offset": 140706,
"first_checksum": 1095500179,
"last_seq_no": 1115,
"last_offset": 141535
}
]
},
{
"fnode": 1289,
"segments": [
{
"author": 2080950356,
"first_seq_no": 1289,
"first_offset": 152255,
"first_checksum": 777889612,
"last_seq_no": 1290,
"last_offset": 153297
}
]
},
{
"fnode": 1614,
"segments": [
{
"author": 1701696287,
"first_seq_no": 1614,
"first_offset": 237537,
"first_checksum": 3210869242,
"last_seq_no": 1617,
"last_offset": 243992
}
]
},
{
"fnode": 1629,
"segments": [
{
"author": 245017003,
"first_seq_no": 1629,
"first_offset": 244514,
"first_checksum": 3153286517,
"last_seq_no": 1632,
"last_offset": 244779
}
]
},
{
"fnode": 1633,
"segments": [
{
"author": 245017003,
"first_seq_no": 1633,
"first_offset": 244779,
"first_checksum": 3701750198,
"last_seq_no": 1637,
"last_offset": 244974
}
]
},
{
"fnode": 1638,
"segments": [
{
"author": 245017003,
"first_seq_no": 1638,
"first_offset": 244974,
"first_checksum": 2949463732,
"last_seq_no": 1638,
"last_offset": 245010
}
]
},
{
"fnode": 1642,
"segments": [
{
"author": 245017003,
"first_seq_no": 1642,
"first_offset": 245137,
"first_checksum": 3605342194,
"last_seq_no": 1645,
"last_offset": 251592
}
]
}
],
"properties": [
{
"path": "/IDENTITY",
"content": "0a4d4620-a93f-468a-9c12-26de543261f0\n"
}
]
}
}
]
}`

var resp pc.GetHintsResponse
require.NoError(t, json.Unmarshal([]byte(fixture), &resp))

var set = make(map[pb.Journal][]recoverylog.Segment)

for _, hints := range resp.BackupHints {
foldHintsIntoSegments(*hints.Hints, set)
}
// Assert that the fragment would be pruned if we only accounted for the backup hints
require.False(t, overlapsAnySegment(set["example/log"], pb.Fragment{
Journal: "example/log",
Begin: 243993,
End: 244000,
}))

// Now add the primary hints and assert that the fragment would be retained
foldHintsIntoSegments(*resp.PrimaryHints.Hints, set)
require.True(t, overlapsAnySegment(set["example/log"], pb.Fragment{
Journal: "example/log",
Begin: 243993,
End: 244000,
}))
}

func TestSegmentFoldingWithSingleLog(t *testing.T) {
var m = make(map[pb.Journal][]recoverylog.Segment)

Expand Down

0 comments on commit 4caca87

Please sign in to comment.