From d3949aa91d2d4319459d32bc41e261389934bb19 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 11 Nov 2019 11:17:20 -0500 Subject: [PATCH] ipns(pubsub): utilize persistent pubsub value store --- core/node/groups.go | 5 ++++- core/node/libp2p/pubsub.go | 11 ++++++----- core/node/libp2p/routing.go | 13 +++++++++---- core/node/libp2p/topicdiscovery.go | 31 ++++++++++++++++++++++++++++++ go.mod | 5 +++-- go.sum | 16 ++++----------- 6 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 core/node/libp2p/topicdiscovery.go diff --git a/core/node/groups.go b/core/node/groups.go index 5eb222ba5ac..b567f34fac8 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -67,8 +67,10 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option { // parse PubSub config - ps := fx.Options() + ps, disc := fx.Options(), fx.Options() if bcfg.getOpt("pubsub") || bcfg.getOpt("ipnsps") { + disc = fx.Provide(libp2p.TopicDiscovery()) + var pubsubOptions []pubsub.Option pubsubOptions = append( pubsubOptions, @@ -113,6 +115,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option { maybeInvoke(libp2p.AutoNATService(cfg.Experimental.QUIC), cfg.Swarm.EnableAutoNATService), connmgr, ps, + disc, ) return opts diff --git a/core/node/libp2p/pubsub.go b/core/node/libp2p/pubsub.go index c065cd7ee40..001a787da71 100644 --- a/core/node/libp2p/pubsub.go +++ b/core/node/libp2p/pubsub.go @@ -1,7 +1,8 @@ package libp2p import ( - host "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/discovery" + "github.com/libp2p/go-libp2p-core/host" pubsub "github.com/libp2p/go-libp2p-pubsub" "go.uber.org/fx" @@ -9,13 +10,13 @@ import ( ) func FloodSub(pubsubOptions ...pubsub.Option) interface{} { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host) (service *pubsub.PubSub, err error) { - return pubsub.NewFloodSub(helpers.LifecycleCtx(mctx, lc), host, pubsubOptions...) + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, disc discovery.Discovery) (service *pubsub.PubSub, err error) { + return pubsub.NewFloodSub(helpers.LifecycleCtx(mctx, lc), host, append(pubsubOptions, pubsub.WithDiscovery(disc))...) } } func GossipSub(pubsubOptions ...pubsub.Option) interface{} { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host) (service *pubsub.PubSub, err error) { - return pubsub.NewGossipSub(helpers.LifecycleCtx(mctx, lc), host, pubsubOptions...) + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, disc discovery.Discovery) (service *pubsub.PubSub, err error) { + return pubsub.NewGossipSub(helpers.LifecycleCtx(mctx, lc), host, append(pubsubOptions, pubsub.WithDiscovery(disc))...) } } diff --git a/core/node/libp2p/routing.go b/core/node/libp2p/routing.go index e5ba1de30bb..cc32ce4b3ca 100644 --- a/core/node/libp2p/routing.go +++ b/core/node/libp2p/routing.go @@ -3,6 +3,7 @@ package libp2p import ( "context" "sort" + "time" host "github.com/libp2p/go-libp2p-core/host" routing "github.com/libp2p/go-libp2p-core/routing" @@ -83,15 +84,19 @@ type p2pPSRoutingIn struct { PubSub *pubsub.PubSub `optional:"true"` } -func PubsubRouter(mctx helpers.MetricsCtx, lc fx.Lifecycle, in p2pPSRoutingIn) (p2pRouterOut, *namesys.PubsubValueStore) { - psRouter := namesys.NewPubsubValueStore( +func PubsubRouter(mctx helpers.MetricsCtx, lc fx.Lifecycle, in p2pPSRoutingIn) (p2pRouterOut, *namesys.PubsubValueStore, error) { + psRouter, err := namesys.NewPubsubValueStore( helpers.LifecycleCtx(mctx, lc), in.Host, - in.BaseIpfsRouting, in.PubSub, in.Validator, + namesys.WithRebroadcastInterval(time.Minute), ) + if err != nil { + return p2pRouterOut{}, nil, err + } + return p2pRouterOut{ Router: Router{ Routing: &routinghelpers.Compose{ @@ -102,5 +107,5 @@ func PubsubRouter(mctx helpers.MetricsCtx, lc fx.Lifecycle, in p2pPSRoutingIn) ( }, Priority: 100, }, - }, psRouter + }, psRouter, nil } diff --git a/core/node/libp2p/topicdiscovery.go b/core/node/libp2p/topicdiscovery.go new file mode 100644 index 00000000000..fd2cbe00637 --- /dev/null +++ b/core/node/libp2p/topicdiscovery.go @@ -0,0 +1,31 @@ +package libp2p + +import ( + "math/rand" + "time" + + "github.com/libp2p/go-libp2p-core/discovery" + "github.com/libp2p/go-libp2p-core/host" + disc "github.com/libp2p/go-libp2p-discovery" + + "github.com/ipfs/go-ipfs/core/node/helpers" + "go.uber.org/fx" +) + +func TopicDiscovery() interface{} { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, cr BaseIpfsRouting) (service discovery.Discovery, err error) { + baseDisc := disc.NewRoutingDiscovery(cr) + minBackoff, maxBackoff := time.Second*60, time.Hour + rng := rand.New(rand.NewSource(rand.Int63())) + d, err := disc.NewBackoffDiscovery( + baseDisc, + disc.NewExponentialBackoff(minBackoff, maxBackoff, disc.FullJitter, time.Second, 5.0, 0, rng), + ) + + if err != nil { + return nil, err + } + + return d, nil + } +} diff --git a/go.mod b/go.mod index b3c7aa7341e..2554d765c41 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/ipfs/go-blockservice v0.1.2 github.com/ipfs/go-cid v0.0.3 github.com/ipfs/go-cidutil v0.0.2 - github.com/ipfs/go-datastore v0.1.1 + github.com/ipfs/go-datastore v0.3.1 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.0.7 github.com/ipfs/go-ds-flatfs v0.1.0 @@ -62,6 +62,7 @@ require ( github.com/libp2p/go-libp2p-circuit v0.1.4 github.com/libp2p/go-libp2p-connmgr v0.1.1 github.com/libp2p/go-libp2p-core v0.2.5 + github.com/libp2p/go-libp2p-discovery v0.2.0 github.com/libp2p/go-libp2p-http v0.1.4 github.com/libp2p/go-libp2p-kad-dht v0.3.1 github.com/libp2p/go-libp2p-kbucket v0.2.1 @@ -70,7 +71,7 @@ require ( github.com/libp2p/go-libp2p-peerstore v0.1.4 github.com/libp2p/go-libp2p-pnet v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.2.4 - github.com/libp2p/go-libp2p-pubsub-router v0.1.0 + github.com/libp2p/go-libp2p-pubsub-router v0.1.1-0.20191209162204-0edbae38a844 github.com/libp2p/go-libp2p-quic-transport v0.2.2 github.com/libp2p/go-libp2p-record v0.1.2 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 diff --git a/go.sum b/go.sum index 81c68259b10..dad5ce04744 100644 --- a/go.sum +++ b/go.sum @@ -8,10 +8,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190823232136-616930265c33/go.mod h1:bOv github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/MichaelMure/go-ipfs-pinner v0.0.0-20191120152055-f93a308647da h1:VujEWo1PUA+EIMvYU26fNvqykLMBuyXkILKbue86bzg= -github.com/MichaelMure/go-ipfs-pinner v0.0.0-20191120152055-f93a308647da/go.mod h1:0G0RKJr9eZwNvcwdLNVymjCcFhWtX9aXVGOxCZglwMg= -github.com/MichaelMure/go-merkledag v0.2.1-0.20191119160700-c20b9a52f504 h1:2qPGrw2YNfWNaOTPHY/WiSQZUKgiGQWh34my2zSppzo= -github.com/MichaelMure/go-merkledag v0.2.1-0.20191119160700-c20b9a52f504/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= @@ -172,6 +168,8 @@ github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9 github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -248,7 +246,6 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= -github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0= github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -272,10 +269,6 @@ github.com/ipfs/go-unixfs v0.2.1 h1:g51t9ODICFZ3F51FPivm8dE7NzYcdAQNUL9wGP5AYa0= github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipfs/interface-go-ipfs-core v0.2.3 h1:E6uQ+1fJjkxJWlL9lAE72a5FWeyeeNL3GitLy8+jq3Y= -github.com/ipfs/interface-go-ipfs-core v0.2.3 h1:E6uQ+1fJjkxJWlL9lAE72a5FWeyeeNL3GitLy8+jq3Y= -github.com/ipfs/interface-go-ipfs-core v0.2.3/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= -github.com/ipfs/interface-go-ipfs-core v0.2.3/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= github.com/ipfs/interface-go-ipfs-core v0.2.5 h1:/rspOe8RbIxwtssEXHB+X9JXhOBDCQt8x50d2kFPXL8= github.com/ipfs/interface-go-ipfs-core v0.2.5/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -434,11 +427,10 @@ github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0 github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= -github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-pubsub v0.2.4 h1:O4BcaKpPQ9p82yTBtzIzgDFoOXkqhrQpfcVac3FAywU= github.com/libp2p/go-libp2p-pubsub v0.2.4/go.mod h1:1tJwAfySvZQ49R9uTVlkwtSTMVLeQQdrnLTJrr91gVc= -github.com/libp2p/go-libp2p-pubsub-router v0.1.0 h1:xA5B8Sdx64tNlSRIcay2QUngtlu8LpUJClaUk/dYYrg= -github.com/libp2p/go-libp2p-pubsub-router v0.1.0/go.mod h1:PnHOshBr/2I2ZxVfEsqfgCQPsVg09zo+DhSlWkOhPFM= +github.com/libp2p/go-libp2p-pubsub-router v0.1.1-0.20191209162204-0edbae38a844 h1:o5nKi3DP0mJ4NzespTkuv1KzBf4fgOPLf8fhY8TrvL4= +github.com/libp2p/go-libp2p-pubsub-router v0.1.1-0.20191209162204-0edbae38a844/go.mod h1:ULJTrV0Y0pIs8CXKgB/tZ934pIETO894ldEB8x6LPcw= github.com/libp2p/go-libp2p-quic-transport v0.2.2 h1:XyGRqFHD1oHdI2k98P1tWWRb9s27fl1SfmCcaX8plso= github.com/libp2p/go-libp2p-quic-transport v0.2.2/go.mod h1:rVzcsiuOFBomAqvNOxeBUcP4vM4wE+NqqRZWvxjkbe0= github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q=