From 7de48451a3b43de49d2b4d7a672835fa24ef04f8 Mon Sep 17 00:00:00 2001 From: Daniel Phelps Date: Thu, 14 Nov 2024 02:16:12 -0800 Subject: [PATCH] (WIP) spin up audiusd container without audius-docker-compose --- dev-tools/templates/devnet.yaml | 1 + go.mod | 2 +- pkg/orchestration/docker.go | 230 ++++++++++++++++++++++++++++++-- pkg/orchestration/run.go | 67 +++++++++- 4 files changed, 285 insertions(+), 15 deletions(-) diff --git a/dev-tools/templates/devnet.yaml b/dev-tools/templates/devnet.yaml index cb7c000ca92..461989c9626 100644 --- a/dev-tools/templates/devnet.yaml +++ b/dev-tools/templates/devnet.yaml @@ -13,6 +13,7 @@ nodes: httpPort: 4000 httpsPort: 4001 version: prerelease + isLocalhost: true privateKey: 21118f9a6de181061a2abd549511105adb4877cf9026f271092e6813b7cf58ab wallet: 0x0D38e653eC28bdea5A2296fD5940aaB2D0B8875c rewardsWallet: 0xb3c66e682Bf9a85F6800c769AC5A05c18C3F331d diff --git a/go.mod b/go.mod index fdb270732cf..15d33085a59 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,6 @@ require ( ) require ( - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/go-openapi/errors v0.22.0 github.com/go-openapi/runtime v0.28.0 github.com/go-openapi/strfmt v0.23.0 @@ -137,6 +136,7 @@ require ( github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/dgraph-io/badger/v4 v4.2.0 // indirect diff --git a/pkg/orchestration/docker.go b/pkg/orchestration/docker.go index 5f5a38ef3f8..728696cd617 100644 --- a/pkg/orchestration/docker.go +++ b/pkg/orchestration/docker.go @@ -45,6 +45,15 @@ var ( }, } + devnetHosts = []string{ + "creator-1.devnet.audius-d:172.100.0.1", + "discovery-1.devnet.audius-d:172.100.0.1", + "identity.devnet.audius-d:172.100.0.1", + "eth-ganache.devnet.audius-d:172.100.0.1", + "acdc-ganache.devnet.audius-d:172.100.0.1", + "solana-test-validator.devnet.audius-d:172.100.0.1", + } + semverRegex = regexp.MustCompile(`\d+\.\d+\.\d+`) ) @@ -169,14 +178,7 @@ func runNode( if nconf.DeployOn == conf.Devnet { hostConfig.NetworkMode = "deployments_devnet" - hostConfig.ExtraHosts = []string{ - "creator-1.devnet.audius-d:172.100.0.1", - "discovery-1.devnet.audius-d:172.100.0.1", - "identity.devnet.audius-d:172.100.0.1", - "eth-ganache.devnet.audius-d:172.100.0.1", - "acdc-ganache.devnet.audius-d:172.100.0.1", - "solana-test-validator.devnet.audius-d:172.100.0.1", - } + hostConfig.ExtraHosts = devnetHosts containerConfig.Env = []string{"HOST_DOCKER_INTERNAL=172.100.0.1"} } @@ -365,6 +367,218 @@ func runNode( return nil } +func runNodeWrapperless( + host string, + config conf.NodeConfig, + nconf conf.NetworkConfig, +) error { + if config.Type != conf.Content { + return logger.Errorf("Unsupported node type %s", config.Type) + } + + execHost := host + if config.IsLocalhost { + execHost = "localhost" + } + containerName := host + if host == "localhost" { + containerName = "audiusd" + } + + dockerClient, err := getDockerClient(execHost) + if err != nil { + return logger.Error(err) + } + defer dockerClient.Close() + + if isContainerRunning(dockerClient, containerName) { + logger.Infof("audiusd container already running for %s", host) + logger.Infof("Use 'audius-ctl restart %s' for a clean restart", host) + return nil + } else if isContainerNameInUse(dockerClient, containerName) { + logger.Infof("stopped audiusd container '%s' exists on %s, removing and starting with current config", containerName, host) + if err := removeContainerByName(dockerClient, containerName); err != nil { + return logger.Error(err) + } + } + + logger.Infof("\nStarting %s\n", host) + + // Configure tag (will replace branch configuration) + var tag string + switch config.Version { + case "prerelease", "edge", "current": + tag = config.Version + case "": + tag = "current" + default: + if semverRegex.MatchString(config.Version) { + tag = config.Version + } + } + + containerConfig := &container.Config{ + Image: fmt.Sprintf("audius/audiusd:%s", tag), + } + hostConfig := &container.HostConfig{ + Mounts: []mount.Mount{ + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/mediorum", + Target: "/tmp/mediorum", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/creator-node-backend", + Target: "/file_storage", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/creator-node-db-15", + Target: "/var/lib/postgresql/data", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/bolt", + Target: "/bolt", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/bolt", + Target: "/audius-core", + }, + mount.Mount{ + Type: mount.TypeBind, + Source: "/var/k8s/env", + Target: "/env", + }, + }, + } + + var port uint = 80 + var tlsPort uint = 443 + if config.HttpPort != 0 { + port = config.HttpPort + } + if config.HttpsPort != 0 { + tlsPort = config.HttpsPort + } + httpPorts := fmt.Sprintf("%d:%d", port, port) + httpsPorts := fmt.Sprintf("%d:%d", tlsPort, tlsPort) + allPorts := []string{httpPorts, httpsPorts} + if config.HostPorts != "" { + allPorts = append(allPorts, strings.Split(config.HostPorts, ",")...) + } + + if config.CorePortP2P == 0 { + config.CorePortP2P = 26656 + } + + if config.CorePortRPC == 0 { + config.CorePortRPC = 26657 + } + + allPorts = append(allPorts, fmt.Sprintf("%d:26656", config.CorePortP2P), fmt.Sprintf("%d:26657", config.CorePortRPC)) + + portSet, portBindings, err := nat.ParsePortSpecs(allPorts) + if err != nil { + return logger.Error(err) + } + containerConfig.ExposedPorts = portSet + hostConfig.PortBindings = portBindings + + if nconf.DeployOn == conf.Devnet { + hostConfig.NetworkMode = "deployments_devnet" + hostConfig.ExtraHosts = devnetHosts + containerConfig.Env = []string{"HOST_DOCKER_INTERNAL=172.100.0.1"} + } + + logger.Info("Generating configuration...") + + privateKey, err := NormalizedPrivateKey(execHost, config.PrivateKey) + if err != nil { + return logger.Error(err) + } + config.PrivateKey = privateKey + + override := config.ToOverrideEnv(host, nconf) + + // generate the override.env file locally + localOverrideFile, err := os.CreateTemp(".", fmt.Sprintf("%s*.env", host)) + if err != nil { + return logger.Error("Could not create local temp file:", err) + } + localOverridePath := localOverrideFile.Name() + localOverrideFile.Close() + + if err := appendRemoteConfig(execHost, override, config.RemoteConfigFile); err != nil { + return logger.Error(err) + } + if err := godotenv.Write(override, localOverridePath); err != nil { + return logger.Error(err) + } + if err := copyFileToHost(execHost, localOverridePath, "/var/k8s/env/override.env"); err != nil { + return logger.Errorf("Could not copy override config to host: %v", err) + } + if err := os.Remove(localOverridePath); err != nil { + return logger.Error(err) + } + + logger.Info("Pulling audiusd image...") + pullResp, err := dockerClient.ImagePull(context.Background(), containerConfig.Image, types.ImagePullOptions{}) + if err != nil { + return logger.Error("Failed to pull image:", err) + } + defer pullResp.Close() + if err := readAndLogCommandOutput(bufio.NewReader(pullResp)); err != nil { + return logger.Error("Error reading ImagePull output:", err) + } + + // create audiusd container + createResponse, err := dockerClient.ContainerCreate( + context.Background(), + containerConfig, + hostConfig, + nil, + nil, + containerName, + ) + if err != nil { + return logger.Error("Failed to create container:", err) + } + if err := dockerClient.ContainerStart( + context.Background(), + createResponse.ID, + container.StartOptions{}, + ); err != nil { + return logger.Error("Failed to start container:", err) + } + + // Wait for audius-d wrapper to be ready + ready := false + timeout := time.After(30 * time.Second) + for !ready { + select { + case <-timeout: + return logger.Error("Timeout waiting for audiusd container to start") + default: + inspect, err := dockerClient.ContainerInspect(context.Background(), createResponse.ID) + if err != nil { + return logger.Error("Could not get status of audiusd container:", err) + } + time.Sleep(3 * time.Second) + ready = inspect.State.Running + logger.Debugf("audiusd container status: %s", inspect.State.Status) + } + } + + if err := setAutoUpdateCron(host, config.Version, nconf.DeployOn); err != nil { + return logger.Error(err) + } + + return nil +} + func isContainerRunning(dockerClient *client.Client, containerName string) bool { containers, err := dockerClient.ContainerList(context.Background(), container.ListOptions{}) if err != nil { diff --git a/pkg/orchestration/run.go b/pkg/orchestration/run.go index 9dfb32565b8..c6b733e04c0 100644 --- a/pkg/orchestration/run.go +++ b/pkg/orchestration/run.go @@ -4,12 +4,15 @@ import ( "bytes" "fmt" "io" + "math/rand" "net" "os" "os/exec" "strings" + "time" "github.com/AudiusProject/audius-protocol/pkg/conf" + "github.com/AudiusProject/audius-protocol/pkg/core/config" "github.com/AudiusProject/audius-protocol/pkg/logger" "github.com/AudiusProject/audius-protocol/pkg/register" "github.com/joho/godotenv" @@ -51,12 +54,22 @@ func RunAudiusNodes(nodes map[string]conf.NodeConfig, network conf.NetworkConfig } for host, nodeConfig := range nodes { - if err := runNode( - host, - nodeConfig, - network, - audiusdTagOverride, - ); err != nil { + var err error + if nodeConfig.Type == conf.Content { + err = runNodeWrapperless( + host, + nodeConfig, + network, + ) + } else { + err = runNode( + host, + nodeConfig, + network, + audiusdTagOverride, + ) + } + if err != nil { logger.Warnf("Error encountered starting node %s: %s", host, err.Error()) logger.Warnf("View full debug log at %s", logger.GetLogFilepath()) } else { @@ -171,3 +184,45 @@ func resolvesToLocalhost(host string) (bool, error) { } return false, nil } + +func setAutoUpdateCron(host, version string, network conf.NetworkType) error { + var updateInterval string + rand.Seed(time.Now().UnixNano()) + if version == "prerelease" && network == conf.Testnet { + // Stage nodes should update continuously, slightly staggered + updateInterval = fmt.Sprintf("%d-59/5", rand.Intn(5)) + } else if config.Version == "edge" { + // Frequent, staggerd release of foundation and other canary nodes + updateInterval = fmt.Sprintf("%d-59/25", rand.Intn(25)) + } else { + // Hourly release for everything else + // starting 55 minutes from now (for randomness + prevent updates during CI) + fiveMinutesAgo := time.Now().Add(-5 * time.Minute).Minute() + updateInterval = fmt.Sprint(fiveMinutesAgo) + } + script := fmt.Sprintf( + `(crontab -l | grep -v '%s'; echo '%s * * * * audius-ctl restart %s -f # audius auto-upgrade for %s' ) | crontab - `, + host, + updateInterval, + host, + host, + ) + if err := execLocal("bash", "-c", script); err != nil { + return logger.Error("Failed to add auto-upgrade cron job:", err) + } + + logger.Infof("auto-upgrade cron job added successfully for %s", host) + return nil +} + +func copyFileToHost(host, src, dest string) error { + var cmd *exec.Cmd + if host == "localhost" { + cmd = exec.Command("cp", src, dest) + } else { + cmd = exec.Command("scp", src, fmt.Sprintf("%s:%s", host, dest)) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +}