fixed incorrect handling of subscription id in event envelopes

cmd/publicatr/go.mod

@@ -1,15 +1,16 @@
 module github.com/Hubmakerlabs/replicatr/cmd/publicatr
 
-go 1.21
+go 1.21.4
 
-toolchain go1.21.1
+toolchain go1.21.5
 
 require (
  github.com/fatih/color v1.15.0
  github.com/mdp/qrterminal/v3 v3.2.0
  github.com/nbd-wtf/go-nostr v0.25.0
  github.com/nbd-wtf/nostr-sdk v0.0.1
  github.com/urfave/cli/v2 v2.25.7
+ mleku.online/git/log v1.0.7
 )
 
 require (
@@ -18,6 +19,7 @@ require (
  github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
  github.com/cespare/xxhash/v2 v2.2.0 // indirect
  github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
  github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
  github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
  github.com/dgraph-io/ristretto v0.1.1 // indirect

cmd/publicatr/go.sum

@@ -177,5 +177,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+mleku.online/git/log v1.0.7 h1:jNDph/CW/GerRluqcWV6F+NMng1gmm5Qi/TFAJRAfpo=
+mleku.online/git/log v1.0.7/go.mod h1:OdomTvlDYHzX1daD1LaqU+TtX484EbECPKOiMrlGkOY=
 rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY=
 rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs=

cmd/publicatr/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
  "context"
+ "encoding/hex"
  "encoding/json"
  "errors"
  "fmt"
@@ -15,13 +16,20 @@ import (
  "time"
 
  "github.com/urfave/cli/v2"
+ log2 "mleku.online/git/log"
 
  "github.com/fatih/color"
  "github.com/nbd-wtf/go-nostr"
  "github.com/nbd-wtf/go-nostr/nip04"
  "github.com/nbd-wtf/go-nostr/nip19"
 )
 
+var (
+ log = log2.GetLogger()
+ fails = log.D.Chk
+ hexDecode, encodeToHex = hex.DecodeString, hex.EncodeToString
+)
+
 const name = "publicatr"
 
 const version = "0.0.53"
@@ -144,38 +152,44 @@ func (cfg *Config) GetFollows(profile string) (map[string]Profile, error) {
  mu.Unlock()
  m := map[string]struct{}{}
 
- cfg.Do(Relay{Read: true}, func(ctx context.Context, relay *nostr.Relay) bool {
- evs, err := relay.QuerySync(ctx, nostr.Filter{Kinds: []int{nostr.KindContactList}, Authors: []string{pub}, Limit: 1})
- if err != nil {
- return true
- }
- for _, ev := range evs {
- var rm map[string]Relay
- if cfg.tempRelay == false {
- if err := json.Unmarshal([]byte(ev.Content), &rm); err == nil {
- for k, v1 := range cfg.Relays {
- if v2, ok := rm[k]; ok {
- v2.Search = v1.Search
+ cfg.Do(Relay{Read: true},
+ func(ctx context.Context, relay *nostr.Relay) bool {
+ evs, err := relay.QuerySync(ctx,
+ nostr.Filter{
+ Kinds: []int{nostr.KindContactList},
+ Authors: []string{pub},
+ Limit: 1,
+ })
+ if err != nil {
+ return true
+ }
+ for _, ev := range evs {
+ var rm map[string]Relay
+ if cfg.tempRelay == false {
+ if err := json.Unmarshal([]byte(ev.Content), &rm); err == nil {
+ for k, v1 := range cfg.Relays {
+ if v2, ok := rm[k]; ok {
+ v2.Search = v1.Search
+ }
  }
+ cfg.Relays = rm
  }
- cfg.Relays = rm
  }
- }
- for _, tag := range ev.Tags {
- if len(tag) >= 2 && tag[0] == "p" {
- mu.Lock()
- m[tag[1]] = struct{}{}
- mu.Unlock()
+  for _, tag := range ev.Tags {
+  if len(tag) >= 2 && tag[0] == "p" {
+  mu.Lock()
+  m[tag[1]] = struct{}{}
+  mu.Unlock()
+ }
  }
  }
- }
- return true
- })
+ return true
+ })
  if cfg.verbose {
  fmt.Printf("found %d followers\n", len(m))
  }
  if len(m) > 0 {
- follows := []string{}
+ var follows []string
  for k := range m {
  follows = append(follows, k)
  }
@@ -338,7 +352,9 @@ func (cfg *Config) Decode(ev *nostr.Event) error {
 }
 
 // PrintEvents is
-func (cfg *Config) PrintEvents(evs []*nostr.Event, followsMap map[string]Profile, j, extra bool) {
+func (cfg *Config) PrintEvents(evs []*nostr.Event,
+ followsMap map[string]Profile, j, extra bool) {
+
  if j {
  if extra {
  var events []Event
@@ -384,36 +400,37 @@ func (cfg *Config) Events(filter nostr.Filter) []*nostr.Event {
  var mu sync.Mutex
  found := false
  var m sync.Map
- cfg.Do(Relay{Read: true}, func(ctx context.Context, relay *nostr.Relay) bool {
- mu.Lock()
- if found {
+ cfg.Do(Relay{Read: true},
+ func(ctx context.Context, relay *nostr.Relay) bool {
+ mu.Lock()
+ if found {
+ mu.Unlock()
+ return false
+ }
  mu.Unlock()
- return false
- }
- mu.Unlock()
- evs, err := relay.QuerySync(ctx, filter)
- if err != nil {
- return true
- }
- for _, ev := range evs {
- if _, ok := m.Load(ev.ID); !ok {
- if ev.Kind == nostr.KindEncryptedDirectMessage {
- if err := cfg.Decode(ev); err != nil {
- continue
+ evs, err := relay.QuerySync(ctx, filter)
+ if err != nil {
+ return true
+ }
+ for _, ev := range evs {
+ if _, ok := m.Load(ev.ID); !ok {
+ if ev.Kind == nostr.KindEncryptedDirectMessage {
+ if err := cfg.Decode(ev); err != nil {
+ continue
+ }
+ }
+ m.LoadOrStore(ev.ID, ev)
+ if len(filter.IDs) == 1 {
+ mu.Lock()
+ found = true
+ ctx.Done()
+ mu.Unlock()
+ break
  }
- }
- m.LoadOrStore(ev.ID, ev)
- if len(filter.IDs) == 1 {
- mu.Lock()
- found = true
- ctx.Done()
- mu.Unlock()
- break
  }
  }
- }
- return true
- })
+ return true
+ })
 
  keys := []string{}
  m.Range(func(k, v any) bool {

cmd/replicatrd/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+ "encoding/hex"
  "fmt"
  "github.com/Hubmakerlabs/replicatr/pkg/relay"
  log2 "mleku.online/git/log"
@@ -9,6 +10,12 @@ import (
  "github.com/Hubmakerlabs/replicatr/pkg/relay/eventstore/badger"
 )
 
+var (
+ log = log2.GetLogger()
+ fails = log.D.Chk
+ hexDecode, encodeToHex = hex.DecodeString, hex.EncodeToString
+)
+
 func main() {
  log2.SetLogLevel(log2.Trace)
  rl := relay.New()

pkg/nostr/nip1/enveloper.go

@@ -87,7 +87,7 @@ func ProcessEnvelope(b []byte) (env Enveloper, label []byte, buf *text.Buffer,
  // log.D.F("processing envelope:\n%s", string(b))
  // The bytes must be valid JSON but we can't assume they are free of
  // whitespace... So we will use some tools.
- buf = text.New(b)
+ buf = text.NewBuffer(b)
  // First there must be an opening bracket.
  if e = buf.ScanThrough('['); e != nil {
  return

pkg/nostr/nip1/event.go

@@ -61,7 +61,7 @@ func (ev *Event) MarshalJSON() (bytes []byte, e error) {
 // ToCanonical returns a structure that provides a byte stringer that generates
 // the canonical form used to generate the ID hash that can be signed.
 func (ev *Event) ToCanonical() (o array.T) {
- log.D.S(ev)
+ // log.D.S(ev)
  return array.T{0, ev.PubKey, ev.CreatedAt, ev.Kind, ev.Tags, ev.Content}
 }
 

pkg/nostr/nip1/eventenvelope.go

@@ -73,29 +73,34 @@ func (E *EventEnvelope) Unmarshal(buf *text.Buffer) (e error) {
  // Next, find the comma after the label (note we aren't checking that only
  // whitespace intervenes because laziness, usually this is the very next
  // character).
- if e = buf.ScanUntil(','); e != nil {
+ if e = buf.ScanUntil(','); log.D.Chk(e) {
  return
  }
- // Next character we find will be open quotes for the subscription ID.
- if e = buf.ScanThrough('"'); e != nil {
+ // Next character we find will be open quotes for the subscription ID, or
+ // the open brace of the embedded event.
+ var matched byte
+ if matched, e = buf.ScanForOneOf(false, '"', '{'); log.D.Chk(e) {
  return
  }
- var sid []byte
- // read the string
- if sid, e = buf.ReadUntil('"'); fails(e) {
- return fmt.Errorf("unterminated quotes in JSON, probably truncated read")
- }
- E.SubscriptionID = SubscriptionID(sid[:])
- // Next, find the comma after the subscription ID (note we aren't checking
- // that only whitespace intervenes because laziness, usually this is the
- // very next
- // character)
- if e = buf.ScanUntil(','); e != nil {
- return fmt.Errorf("event not found in event envelope")
+ if matched == '"' {
+ // Advance the cursor to consume the quote character.
+ buf.Pos++
+ var sid []byte
+ // Read the string.
+ if sid, e = buf.ReadUntil('"'); log.D.Chk(e) {
+ return fmt.Errorf("unterminated quotes in JSON, probably truncated read")
+ }
+ E.SubscriptionID = SubscriptionID(sid[:])
+ // Next, find the comma after the subscription ID (note we aren't checking
+ // that only whitespace intervenes because laziness, usually this is the
+ // very next character).
+ if e = buf.ScanUntil(','); log.D.Chk(e) {
+ return fmt.Errorf("event not found in event envelope")
+ }
  }
  // find the opening brace of the event object, usually this is the very next
  // character, we aren't checking for valid whitespace because laziness.
- if e = buf.ScanUntil('{'); e != nil {
+ if e = buf.ScanUntil('{'); log.D.Chk(e) {
  return fmt.Errorf("event not found in event envelope")
  }
  // now we should have an event object next. It has no embedded object so it
@@ -113,7 +118,7 @@ func (E *EventEnvelope) Unmarshal(buf *text.Buffer) (e error) {
  }
  // technically we maybe should read ahead further to make sure the JSON
  // closes correctly. Not going to abort because of this.
- if e = buf.ScanUntil(']'); e != nil {
+ if e = buf.ScanUntil(']'); log.D.Chk(e) {
  return fmt.Errorf("malformed JSON, no closing bracket on array")
  }
  // whatever remains doesn't matter as the envelope has fully unmarshaled.

pkg/nostr/nip1/helpers.go

@@ -43,7 +43,7 @@ func NewSubscriptionID(s string) (SubscriptionID, error) {
 
 // IsValid returns true if the subscription id is between 1 and 64 characters.
 // Invalid means too long or not present.
-func (si SubscriptionID) IsValid() bool { return len(si) <= 64 }
+func (si SubscriptionID) IsValid() bool { return len(si) <= 64 && len(si) > 0 }
 
 // EventID is the SHA256 hash in hexadecimal of the canonical form of an event
 // as produced by the output of Event.ToCanonical().Bytes().

pkg/wire/text/mangle.go

@@ -15,9 +15,9 @@ type Buffer struct {
  Buf []byte
 }
 
-// New returns a new buffer containing the provided slice. This slice
+// NewBuffer returns a new buffer containing the provided slice. This slice
 // can/will be mutated.
-func New(b []byte) (buf *Buffer) {
+func NewBuffer(b []byte) (buf *Buffer) {
  return &Buffer{Buf: b}
 }
 
@@ -95,6 +95,7 @@ func (b *Buffer) Scan(c byte, through, slice bool) (subSlice []byte, e error) {
  var inQuotes bool
  quotes := c == '"'
  for i := b.Pos; i < bLen; i++ {
+ // log.D.F("'%s' searching '%s' found: %v, inquotes: %v, through %v quotes %v", string(b.Buf[i]), string(c), c == b.Buf[i], inQuotes, through, quotes)
  // log.D.F("first: quotes: %v %d/%d '%s'", quotes, i, bLen,
  // string(b.Buf[i]))
  if !quotes {
@@ -200,6 +201,35 @@ func (b *Buffer) ReadEnclosed() (bb []byte, e error) {
  return
 }
 
+// ScanForOneOf provides the ability to scan for two or more different bytes.
+//
+// For simplicity it does not skip quotes, it was actually written to find
+// quotes or braces but just to make it clear this is very bare.
+//
+// if through is set to true, the cursor is advanced to the next after the match
+func (b *Buffer) ScanForOneOf(through bool, c ...byte) (which byte, e error) {
+ if len(c) < 2 {
+ e = fmt.Errorf("at least two bytes required for ScanUntilOneOf, " +
+ "otherwise just use ScanUntil")
+ return
+ }
+ bLen := len(b.Buf)
+ for i := b.Pos; i < bLen; i++ {
+ for _, d := range c {
+ if b.Buf[i] == d {
+ which = d
+ if through {
+ i++
+ }
+ b.Pos = i
+ return
+ }
+ }
+ }
+ e = io.EOF
+ return
+}
+
 // Tail returns the buffer starting from the current Pos position.
 func (b *Buffer) Tail() []byte { return b.Buf[b.Pos:] }
 

pkg/wire/text/unescape.go

@@ -125,8 +125,8 @@ func SecondHexCharToValue(in byte) (out byte) {
 // characters that must be escaped for JSON/HTML encoding. This means octal
 // `\xxx` unicode backslash escapes \uXXXX and \UXXXX
 func UnescapeByteString(bs []byte) (o []byte) {
- in := New(bs) // read side
- out := New(bs) // write side
+ in := NewBuffer(bs) // read side
+ out := NewBuffer(bs) // write side
  var e error
  var segment []byte
  var c byte