diff --git a/cmd/gazctl/gazctlcmd/shards_prune.go b/cmd/gazctl/gazctlcmd/shards_prune.go index e30ba10a..fc7f7a96 100644 --- a/cmd/gazctl/gazctlcmd/shards_prune.go +++ b/cmd/gazctl/gazctlcmd/shards_prune.go @@ -152,11 +152,8 @@ func foldHintsIntoSegments(hints recoverylog.FSMHints, sets map[pb.Journal]recov for _, segment := range segments { var set = sets[segment.Log] - // set.Add() will return an error if we attempt to add a segment having a - // greater SeqNo and LastOffset != 0, to a set already having a lesser - // SeqNo and LastOffset == 0. Or, if FirstSeqNo is equal, it will replace - // a zero LastOffset with a non-zero one (which is not what we want - // in this case). + // If FirstSeqNo is equal, set.Add will replace a zero LastOffset with a + // non-zero one (which is not what we want in this case). // // So, zero LastOffset here if |segment| isn't strictly less than // and non-overlapping with a pre-existing last LastOffset==0 element. diff --git a/consumer/recoverylog/segment.go b/consumer/recoverylog/segment.go index 56f76c64..9d0d85f4 100644 --- a/consumer/recoverylog/segment.go +++ b/consumer/recoverylog/segment.go @@ -141,8 +141,6 @@ func reduceSegment(a, b Segment) (Segment, error) { // Offset ordering constraint checks. case a.FirstSeqNo < b.FirstSeqNo && a.Log == b.Log && a.FirstOffset > b.FirstOffset: return a, fmt.Errorf("expected monotonic FirstOffset: %#v vs %#v", a, b) - case a.FirstSeqNo < b.FirstSeqNo && a.LastOffset == 0 && b.LastOffset != 0: - return a, fmt.Errorf("expected preceding Segment to also include LastOffset: %#v vs %#v", a, b) case a.LastOffset == 0 && a.Log != b.Log: // This cases isn't strictly required, but it's true of the intended diff --git a/consumer/recoverylog/segment_test.go b/consumer/recoverylog/segment_test.go index 09f5c00e..a4771d87 100644 --- a/consumer/recoverylog/segment_test.go +++ b/consumer/recoverylog/segment_test.go @@ -76,9 +76,9 @@ func (s *SegmentSuite) TestSegmentReductionCases(c *gc.C) { }, // Overlap, and preceding Segment is missing LastOffset. { - a: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 7, FirstOffset: 200, LastOffset: 000, FirstChecksum: 0x22, Log: A}, - b: Segment{Author: 0x1, FirstSeqNo: 4, LastSeqNo: 9, FirstOffset: 400, LastOffset: 901, FirstChecksum: 0x44, Log: A}, - err: "expected preceding Segment to also include LastOffset: .*", + a: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 7, FirstOffset: 200, LastOffset: 000, FirstChecksum: 0x22, Log: A}, + b: Segment{Author: 0x1, FirstSeqNo: 4, LastSeqNo: 9, FirstOffset: 400, LastOffset: 901, FirstChecksum: 0x44, Log: A}, + e: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 9, FirstOffset: 200, LastOffset: 901, FirstChecksum: 0x22, Log: A}, }, // Overlap, mismatched authors. { @@ -98,6 +98,12 @@ func (s *SegmentSuite) TestSegmentReductionCases(c *gc.C) { b: Segment{Author: 0x1, FirstSeqNo: 3, LastSeqNo: 6, FirstOffset: 300, LastOffset: 000, FirstChecksum: 0x33, Log: A}, e: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 7, FirstOffset: 200, LastOffset: 701, FirstChecksum: 0x22, Log: A}, }, + // Covered, with 0 LastOffset + { + a: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 7, FirstOffset: 200, LastOffset: 701, FirstChecksum: 0x22, Log: A}, + b: Segment{Author: 0x1, FirstSeqNo: 1, LastSeqNo: 8, FirstOffset: 2, LastOffset: 0, FirstChecksum: 0x33, Log: A}, + e: Segment{Author: 0x1, FirstSeqNo: 1, LastSeqNo: 8, FirstOffset: 2, LastOffset: 0, FirstChecksum: 0x33, Log: A}, + }, // Disjoint. { a: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 4, FirstOffset: 200, LastOffset: 401, FirstChecksum: 0x22, Log: A}, @@ -122,6 +128,12 @@ func (s *SegmentSuite) TestSegmentReductionCases(c *gc.C) { b: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 9, FirstOffset: 210, LastOffset: 901, FirstChecksum: 0x22, Log: A}, e: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 9, FirstOffset: 210, LastOffset: 901, FirstChecksum: 0x22, Log: A}, }, + // Equal-left, with LastOffset: 0 + { + a: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 7, FirstOffset: 200, LastOffset: 701, FirstChecksum: 0x22, Log: A}, + b: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 9, FirstOffset: 200, LastOffset: 000, FirstChecksum: 0x22, Log: A}, + e: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 9, FirstOffset: 200, LastOffset: 000, FirstChecksum: 0x22, Log: A}, + }, // Equal-left, but checksum mismatch. { a: Segment{Author: 0x1, FirstSeqNo: 2, LastSeqNo: 7, FirstOffset: 200, LastOffset: 701, FirstChecksum: 0x22, Log: A}, @@ -254,9 +266,6 @@ func (s *SegmentSuite) TestSetConsistencyChecks(c *gc.C) { // Overlapping, incorrect log. {Segment{Author: 0xa, FirstSeqNo: 4, LastSeqNo: 5, FirstOffset: 400, LastOffset: 501, Log: "wrong"}, "expected Segment Log equality: .*"}, - // Missing LastOffset. - {Segment{Author: 0xb, FirstSeqNo: 12, LastSeqNo: 12, FirstOffset: 1200, LastOffset: 000, Log: A}, - "expected preceding Segment to also include LastOffset: .*"}, // Non-monotonic offsets. {Segment{Author: 0xb, FirstSeqNo: 12, LastSeqNo: 12, FirstOffset: 1050, LastOffset: 1200, Log: A}, "expected monotonic FirstOffset: .*"},