(WIP) spin up audiusd container without audius-docker-compose

dev-tools/templates/devnet.yaml

@@ -13,6 +13,7 @@ nodes:
     httpPort: 4000
     httpsPort: 4001
     version: prerelease
+    isLocalhost: true
     privateKey: 21118f9a6de181061a2abd549511105adb4877cf9026f271092e6813b7cf58ab
     wallet: 0x0D38e653eC28bdea5A2296fD5940aaB2D0B8875c
     rewardsWallet: 0xb3c66e682Bf9a85F6800c769AC5A05c18C3F331d

go.mod

@@ -50,7 +50,6 @@ require (
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/go-openapi/errors v0.22.0
 	github.com/go-openapi/runtime v0.28.0
 	github.com/go-openapi/strfmt v0.23.0
@@ -137,6 +136,7 @@ require (
 	github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
 	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/dgraph-io/badger/v4 v4.2.0 // indirect

pkg/orchestration/docker.go

@@ -45,6 +45,15 @@ var (
 		},
 	}
 
+	devnetHosts = []string{
+		"creator-1.devnet.audius-d:172.100.0.1",
+		"discovery-1.devnet.audius-d:172.100.0.1",
+		"identity.devnet.audius-d:172.100.0.1",
+		"eth-ganache.devnet.audius-d:172.100.0.1",
+		"acdc-ganache.devnet.audius-d:172.100.0.1",
+		"solana-test-validator.devnet.audius-d:172.100.0.1",
+	}
+
 	semverRegex = regexp.MustCompile(`\d+\.\d+\.\d+`)
 )
 
@@ -169,14 +178,7 @@ func runNode(
 
 	if nconf.DeployOn == conf.Devnet {
 		hostConfig.NetworkMode = "deployments_devnet"
-		hostConfig.ExtraHosts = []string{
-			"creator-1.devnet.audius-d:172.100.0.1",
-			"discovery-1.devnet.audius-d:172.100.0.1",
-			"identity.devnet.audius-d:172.100.0.1",
-			"eth-ganache.devnet.audius-d:172.100.0.1",
-			"acdc-ganache.devnet.audius-d:172.100.0.1",
-			"solana-test-validator.devnet.audius-d:172.100.0.1",
-		}
+		hostConfig.ExtraHosts = devnetHosts
 		containerConfig.Env = []string{"HOST_DOCKER_INTERNAL=172.100.0.1"}
 	}
 
@@ -365,6 +367,218 @@ func runNode(
 	return nil
 }
 
+func runNodeWrapperless(
+	host string,
+	config conf.NodeConfig,
+	nconf conf.NetworkConfig,
+) error {
+	if config.Type != conf.Content {
+		return logger.Errorf("Unsupported node type %s", config.Type)
+	}
+
+	execHost := host
+	if config.IsLocalhost {
+		execHost = "localhost"
+	}
+	containerName := host
+	if host == "localhost" {
+		containerName = "audiusd"
+	}
+
+	dockerClient, err := getDockerClient(execHost)
+	if err != nil {
+		return logger.Error(err)
+	}
+	defer dockerClient.Close()
+
+	if isContainerRunning(dockerClient, containerName) {
+		logger.Infof("audiusd container already running for %s", host)
+		logger.Infof("Use 'audius-ctl restart %s' for a clean restart", host)
+		return nil
+	} else if isContainerNameInUse(dockerClient, containerName) {
+		logger.Infof("stopped audiusd container '%s' exists on %s, removing and starting with current config", containerName, host)
+		if err := removeContainerByName(dockerClient, containerName); err != nil {
+			return logger.Error(err)
+		}
+	}
+
+	logger.Infof("\nStarting %s\n", host)
+
+	// Configure tag (will replace branch configuration)
+	var tag string
+	switch config.Version {
+	case "prerelease", "edge", "current":
+		tag = config.Version
+	case "":
+		tag = "current"
+	default:
+		if semverRegex.MatchString(config.Version) {
+			tag = config.Version
+		}
+	}
+
+	containerConfig := &container.Config{
+		Image: fmt.Sprintf("audius/audiusd:%s", tag),
+	}
+	hostConfig := &container.HostConfig{
+		Mounts: []mount.Mount{
+			mount.Mount{
+				Type:   mount.TypeBind,
+				Source: "/var/k8s/mediorum",
+				Target: "/tmp/mediorum",
+			},
+			mount.Mount{
+				Type:   mount.TypeBind,
+				Source: "/var/k8s/creator-node-backend",
+				Target: "/file_storage",
+			},
+			mount.Mount{
+				Type:   mount.TypeBind,
+				Source: "/var/k8s/creator-node-db-15",
+				Target: "/var/lib/postgresql/data",
+			},
+			mount.Mount{
+				Type:   mount.TypeBind,
+				Source: "/var/k8s/bolt",
+				Target: "/bolt",
+			},
+			mount.Mount{
+				Type:   mount.TypeBind,
+				Source: "/var/k8s/bolt",
+				Target: "/audius-core",
+			},
+			mount.Mount{
+				Type:   mount.TypeBind,
+				Source: "/var/k8s/env",
+				Target: "/env",
+			},
+		},
+	}
+
+	var port uint = 80
+	var tlsPort uint = 443
+	if config.HttpPort != 0 {
+		port = config.HttpPort
+	}
+	if config.HttpsPort != 0 {
+		tlsPort = config.HttpsPort
+	}
+	httpPorts := fmt.Sprintf("%d:%d", port, port)
+	httpsPorts := fmt.Sprintf("%d:%d", tlsPort, tlsPort)
+	allPorts := []string{httpPorts, httpsPorts}
+	if config.HostPorts != "" {
+		allPorts = append(allPorts, strings.Split(config.HostPorts, ",")...)
+	}
+
+	if config.CorePortP2P == 0 {
+		config.CorePortP2P = 26656
+	}
+
+	if config.CorePortRPC == 0 {
+		config.CorePortRPC = 26657
+	}
+
+	allPorts = append(allPorts, fmt.Sprintf("%d:26656", config.CorePortP2P), fmt.Sprintf("%d:26657", config.CorePortRPC))
+
+	portSet, portBindings, err := nat.ParsePortSpecs(allPorts)
+	if err != nil {
+		return logger.Error(err)
+	}
+	containerConfig.ExposedPorts = portSet
+	hostConfig.PortBindings = portBindings
+
+	if nconf.DeployOn == conf.Devnet {
+		hostConfig.NetworkMode = "deployments_devnet"
+		hostConfig.ExtraHosts = devnetHosts
+		containerConfig.Env = []string{"HOST_DOCKER_INTERNAL=172.100.0.1"}
+	}
+
+	logger.Info("Generating configuration...")
+
+	privateKey, err := NormalizedPrivateKey(execHost, config.PrivateKey)
+	if err != nil {
+		return logger.Error(err)
+	}
+	config.PrivateKey = privateKey
+
+	override := config.ToOverrideEnv(host, nconf)
+
+	// generate the override.env file locally
+	localOverrideFile, err := os.CreateTemp(".", fmt.Sprintf("%s*.env", host))
+	if err != nil {
+		return logger.Error("Could not create local temp file:", err)
+	}
+	localOverridePath := localOverrideFile.Name()
+	localOverrideFile.Close()
+
+	if err := appendRemoteConfig(execHost, override, config.RemoteConfigFile); err != nil {
+		return logger.Error(err)
+	}
+	if err := godotenv.Write(override, localOverridePath); err != nil {
+		return logger.Error(err)
+	}
+	if err := copyFileToHost(execHost, localOverridePath, "/var/k8s/env/override.env"); err != nil {
+		return logger.Errorf("Could not copy override config to host: %v", err)
+	}
+	if err := os.Remove(localOverridePath); err != nil {
+		return logger.Error(err)
+	}
+
+	logger.Info("Pulling audiusd image...")
+	pullResp, err := dockerClient.ImagePull(context.Background(), containerConfig.Image, types.ImagePullOptions{})
+	if err != nil {
+		return logger.Error("Failed to pull image:", err)
+	}
+	defer pullResp.Close()
+	if err := readAndLogCommandOutput(bufio.NewReader(pullResp)); err != nil {
+		return logger.Error("Error reading ImagePull output:", err)
+	}
+
+	// create audiusd container
+	createResponse, err := dockerClient.ContainerCreate(
+		context.Background(),
+		containerConfig,
+		hostConfig,
+		nil,
+		nil,
+		containerName,
+	)
+	if err != nil {
+		return logger.Error("Failed to create container:", err)
+	}
+	if err := dockerClient.ContainerStart(
+		context.Background(),
+		createResponse.ID,
+		container.StartOptions{},
+	); err != nil {
+		return logger.Error("Failed to start container:", err)
+	}
+
+	// Wait for audius-d wrapper to be ready
+	ready := false
+	timeout := time.After(30 * time.Second)
+	for !ready {
+		select {
+		case <-timeout:
+			return logger.Error("Timeout waiting for audiusd container to start")
+		default:
+			inspect, err := dockerClient.ContainerInspect(context.Background(), createResponse.ID)
+			if err != nil {
+				return logger.Error("Could not get status of audiusd container:", err)
+			}
+			time.Sleep(3 * time.Second)
+			ready = inspect.State.Running
+			logger.Debugf("audiusd container status: %s", inspect.State.Status)
+		}
+	}
+
+	if err := setAutoUpdateCron(host, config.Version, nconf.DeployOn); err != nil {
+		return logger.Error(err)
+	}
+
+	return nil
+}
+
 func isContainerRunning(dockerClient *client.Client, containerName string) bool {
 	containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{})
 	if err != nil {

pkg/orchestration/run.go

@@ -4,12 +4,15 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"math/rand"
 	"net"
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/AudiusProject/audius-protocol/pkg/conf"
+	"github.com/AudiusProject/audius-protocol/pkg/core/config"
 	"github.com/AudiusProject/audius-protocol/pkg/logger"
 	"github.com/AudiusProject/audius-protocol/pkg/register"
 	"github.com/joho/godotenv"
@@ -51,12 +54,22 @@ func RunAudiusNodes(nodes map[string]conf.NodeConfig, network conf.NetworkConfig
 	}
 
 	for host, nodeConfig := range nodes {
-		if err := runNode(
-			host,
-			nodeConfig,
-			network,
-			audiusdTagOverride,
-		); err != nil {
+		var err error
+		if nodeConfig.Type == conf.Content {
+			err = runNodeWrapperless(
+				host,
+				nodeConfig,
+				network,
+			)
+		} else {
+			err = runNode(
+				host,
+				nodeConfig,
+				network,
+				audiusdTagOverride,
+			)
+		}
+		if err != nil {
 			logger.Warnf("Error encountered starting node %s: %s", host, err.Error())
 			logger.Warnf("View full debug log at %s", logger.GetLogFilepath())
 		} else {
@@ -171,3 +184,45 @@ func resolvesToLocalhost(host string) (bool, error) {
 	}
 	return false, nil
 }
+
+func setAutoUpdateCron(host, version string, network conf.NetworkType) error {
+	var updateInterval string
+	rand.Seed(time.Now().UnixNano())
+	if version == "prerelease" && network == conf.Testnet {
+		// Stage nodes should update continuously, slightly staggered
+		updateInterval = fmt.Sprintf("%d-59/5", rand.Intn(5))
+	} else if config.Version == "edge" {
+		// Frequent, staggerd release of foundation and other canary nodes
+		updateInterval = fmt.Sprintf("%d-59/25", rand.Intn(25))
+	} else {
+		// Hourly release for everything else
+		// starting 55 minutes from now (for randomness + prevent updates during CI)
+		fiveMinutesAgo := time.Now().Add(-5 * time.Minute).Minute()
+		updateInterval = fmt.Sprint(fiveMinutesAgo)
+	}
+	script := fmt.Sprintf(
+		`(crontab -l | grep -v '%s'; echo '%s * * * * audius-ctl restart %s -f # audius auto-upgrade for %s' ) | crontab - `,
+		host,
+		updateInterval,
+		host,
+		host,
+	)
+	if err := execLocal("bash", "-c", script); err != nil {
+		return logger.Error("Failed to add auto-upgrade cron job:", err)
+	}
+
+	logger.Infof("auto-upgrade cron job added successfully for %s", host)
+	return nil
+}
+
+func copyFileToHost(host, src, dest string) error {
+	var cmd *exec.Cmd
+	if host == "localhost" {
+		cmd = exec.Command("cp", src, dest)
+	} else {
+		cmd = exec.Command("scp", src, fmt.Sprintf("%s:%s", host, dest))
+	}
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}