wallet: bitcoind support with RPC polling (ark-network#254)

* support bitcoind connection

* fix: wallet should be the varibale, not s.wallet

* block cache: up to 2M blocks

* WithBitcoind wait for sync

* WithBitcoind rename

* switch based on the ENV VARs

* Add make targets

server/.gitignore

@@ -25,4 +25,4 @@ go.work.sum
 
 .vscode/
 
-tmp/
+data/

server/Makefile

@@ -1,4 +1,4 @@
-.PHONY: build clean cov help intergrationtest lint run run-signet test vet proto proto-lint
+.PHONY: build clean cov help intergrationtest lint run run-neutrino run-neutrino-signet test vet proto proto-lint
 
 ## build: build for all platforms
 build:
@@ -30,26 +30,43 @@ lint:
 	@echo "Linting code..."
 	@golangci-lint run --fix
 
-## run: run in regtest mode
+## run: run in regtest mode with bitcoind
 run: clean
-	@echo "Running arkd in dev mode..."
-	@export ARK_NEUTRINO_PEER=localhost:18444; \
-	export ARK_ROUND_INTERVAL=10; \
+	@echo "Running arkd with Bitcoin Core in regtest mode ..."
+	@export ARK_ROUND_INTERVAL=10; \
 	export ARK_LOG_LEVEL=5; \
 	export ARK_NETWORK=regtest; \
 	export ARK_PORT=7070; \
 	export ARK_NO_TLS=true; \
 	export ARK_NO_MACAROONS=true; \
+	export ARK_MIN_RELAY_FEE=200; \
 	export ARK_TX_BUILDER_TYPE=covenantless; \
 	export ARK_ESPLORA_URL=http://localhost:3000; \
-	export ARK_MIN_RELAY_FEE=200; \
+	export ARK_BITCOIND_RPC_USER=admin1; \
+	export ARK_BITCOIND_RPC_PASS=123; \
+	export ARK_BITCOIND_RPC_HOST=localhost:18443; \
+	export ARK_DATADIR=./data/regtest; \
 	go run ./cmd/arkd
 
-## 	export ARK_NEUTRINO_PEER=44.240.54.180:38333 ; \
+## run-neutrino: run in regtest mode with neutrino
+run-neutrino: clean
+	@echo "Running arkd with Neutrino in regtest mode ..."
+	@export ARK_ROUND_INTERVAL=10; \
+	export ARK_LOG_LEVEL=5; \
+	export ARK_NETWORK=regtest; \
+	export ARK_PORT=7070; \
+	export ARK_NO_TLS=true; \
+	export ARK_NO_MACAROONS=true; \
+	export ARK_MIN_RELAY_FEE=200; \
+	export ARK_TX_BUILDER_TYPE=covenantless; \
+	export ARK_ESPLORA_URL=http://localhost:3000; \
+	export ARK_NEUTRINO_PEER=localhost:18444; \
+	export ARK_DATADIR=./data/regtest; \
+	go run ./cmd/arkd
 
-## run: run in signet mode
-run-signet: clean
-	@echo "Running arkd in signet mode..."
+## run-neutrino-signet: run in signet mode
+run-neutrino-signet: clean
+	@echo "Running arkd with Neutrino in signet mode ..."
 	@export ARK_ROUND_INTERVAL=10; \
 	export ARK_LOG_LEVEL=5; \
 	export ARK_NETWORK=signet; \
@@ -60,7 +77,7 @@ run-signet: clean
 	export ARK_ESPLORA_URL=https://mutinynet.com/api; \
 	export ARK_NEUTRINO_PEER=45.79.52.207:38333; \
 	export ARK_MIN_RELAY_FEE=200; \
-	export ARK_DATADIR=./tmp/signet/arkd-data; \
+	export ARK_DATADIR=./data/signet; \
 	go run ./cmd/arkd
 
 ## test: runs unit and component tests

server/cmd/arkd/main.go

@@ -76,6 +76,9 @@ func mainAction(_ *cli.Context) error {
 		UnilateralExitDelay:   cfg.UnilateralExitDelay,
 		EsploraURL:            cfg.EsploraURL,
 		NeutrinoPeer:          cfg.NeutrinoPeer,
+		BitcoindRpcUser:       cfg.BitcoindRpcUser,
+		BitcoindRpcPass:       cfg.BitcoindRpcPass,
+		BitcoindRpcHost:       cfg.BitcoindRpcHost,
 	}
 	svc, err := grpcservice.NewService(svcConfig, appConfig)
 	if err != nil {

server/internal/app-config/config.go

@@ -64,8 +64,11 @@ type Config struct {
 	RoundLifetime         int64
 	UnilateralExitDelay   int64
 
-	EsploraURL   string
-	NeutrinoPeer string
+	EsploraURL      string
+	NeutrinoPeer    string
+	BitcoindRpcUser string
+	BitcoindRpcPass string
+	BitcoindRpcHost string
 
 	repo      ports.RepoManager
 	svc       application.Service
@@ -230,13 +233,34 @@ func (c *Config) walletService() error {
 		return fmt.Errorf("missing esplora url, covenant-less ark requires ARK_ESPLORA_URL to be set")
 	}
 
-	svc, err := btcwallet.NewService(btcwallet.WalletConfig{
-		Datadir:    c.DbDir,
-		Network:    c.Network,
-		EsploraURL: c.EsploraURL,
-	},
-		btcwallet.WithNeutrino(c.NeutrinoPeer),
-	)
+	// Check if both Neutrino peer and Bitcoind RPC credentials are provided
+	if c.NeutrinoPeer != "" && (c.BitcoindRpcUser != "" || c.BitcoindRpcPass != "") {
+		return fmt.Errorf("cannot use both Neutrino peer and Bitcoind RPC credentials")
+	}
+
+	var svc ports.WalletService
+	var err error
+
+	switch {
+	case c.NeutrinoPeer != "":
+		svc, err = btcwallet.NewService(btcwallet.WalletConfig{
+			Datadir:    c.DbDir,
+			Network:    c.Network,
+			EsploraURL: c.EsploraURL,
+		}, btcwallet.WithNeutrino(c.NeutrinoPeer))
+
+	case c.BitcoindRpcUser != "" && c.BitcoindRpcPass != "":
+		svc, err = btcwallet.NewService(btcwallet.WalletConfig{
+			Datadir:    c.DbDir,
+			Network:    c.Network,
+			EsploraURL: c.EsploraURL,
+		}, btcwallet.WithPollingBitcoind(c.BitcoindRpcHost, c.BitcoindRpcUser, c.BitcoindRpcPass))
+
+	// Placeholder for future initializers like WithBitcoindZMQ
+	default:
+		return fmt.Errorf("either Neutrino peer or Bitcoind RPC credentials must be provided")
+	}
+
 	if err != nil {
 		return err
 	}

server/internal/config/config.go

@@ -31,6 +31,9 @@ type Config struct {
 	UnilateralExitDelay   int64
 	EsploraURL            string
 	NeutrinoPeer          string
+	BitcoindRpcUser       string
+	BitcoindRpcPass       string
+	BitcoindRpcHost       string
 	TLSExtraIPs           []string
 	TLSExtraDomains       []string
 }
@@ -53,6 +56,9 @@ var (
 	UnilateralExitDelay   = "UNILATERAL_EXIT_DELAY"
 	EsploraURL            = "ESPLORA_URL"
 	NeutrinoPeer          = "NEUTRINO_PEER"
+	BitcoindRpcUser       = "BITCOIND_RPC_USER"
+	BitcoindRpcPass       = "BITCOIND_RPC_PASS"
+	BitcoindRpcHost       = "BITCOIND_RPC_HOST"
 	NoMacaroons           = "NO_MACAROONS"
 	NoTLS                 = "NO_TLS"
 	TLSExtraIP            = "TLS_EXTRA_IP"
@@ -128,6 +134,9 @@ func LoadConfig() (*Config, error) {
 		UnilateralExitDelay:   viper.GetInt64(UnilateralExitDelay),
 		EsploraURL:            viper.GetString(EsploraURL),
 		NeutrinoPeer:          viper.GetString(NeutrinoPeer),
+		BitcoindRpcUser:       viper.GetString(BitcoindRpcUser),
+		BitcoindRpcPass:       viper.GetString(BitcoindRpcPass),
+		BitcoindRpcHost:       viper.GetString(BitcoindRpcHost),
 		NoMacaroons:           viper.GetBool(NoMacaroons),
 		TLSExtraIPs:           viper.GetStringSlice(TLSExtraIP),
 		TLSExtraDomains:       viper.GetStringSlice(TLSExtraDomain),

server/internal/infrastructure/wallet/btc-embedded/wallet.go

@@ -138,6 +138,68 @@ func WithNeutrino(initialPeer string) WalletOption {
 	}
 }
 
+func WithPollingBitcoind(host, user, pass string) WalletOption {
+	return func(s *service) error {
+		netParams := s.cfg.chainParams()
+		// Create a new bitcoind configuration
+		bitcoindConfig := &chain.BitcoindConfig{
+			ChainParams: netParams,
+			Host:        host,
+			User:        user,
+			Pass:        pass,
+			PollingConfig: &chain.PollingConfig{
+				BlockPollingInterval:    10 * time.Second,
+				TxPollingInterval:       5 * time.Second,
+				TxPollingIntervalJitter: 0.1,
+				RPCBatchSize:            20,
+				RPCBatchInterval:        1 * time.Second,
+			},
+		}
+
+		chain.UseLogger(logger("chain"))
+
+		// Create the BitcoindConn first
+		bitcoindConn, err := chain.NewBitcoindConn(bitcoindConfig)
+		if err != nil {
+			return fmt.Errorf("failed to create bitcoind connection: %w", err)
+		}
+
+		// Start the bitcoind connection
+		if err := bitcoindConn.Start(); err != nil {
+			return fmt.Errorf("failed to start bitcoind connection: %w", err)
+		}
+
+		// Now create the BitcoindClient using the connection
+		chainClient := bitcoindConn.NewBitcoindClient()
+
+		// Start the chain client
+		if err := chainClient.Start(); err != nil {
+			bitcoindConn.Stop()
+			return fmt.Errorf("failed to start bitcoind client: %w", err)
+		}
+
+		// wait for bitcoind to sync
+		for !chainClient.IsCurrent() {
+			time.Sleep(1 * time.Second)
+		}
+
+		// Set up the wallet as chain source and scanner
+		if err := withChainSource(chainClient)(s); err != nil {
+			chainClient.Stop()
+			bitcoindConn.Stop()
+			return fmt.Errorf("failed to set chain source: %w", err)
+		}
+
+		if err := withScanner(chainClient)(s); err != nil {
+			chainClient.Stop()
+			bitcoindConn.Stop()
+			return fmt.Errorf("failed to set scanner: %w", err)
+		}
+
+		return nil
+	}
+}
+
 // NewService creates the wallet service, an option must be set to configure the chain source.
 func NewService(cfg WalletConfig, options ...WalletOption) (ports.WalletService, error) {
 	wallet.UseLogger(logger("wallet"))