dynamic load all config from informer cache

cmd/open-hydra-server/app/config/config.go

@@ -81,7 +81,6 @@ type (
  // default = "/root/public-dataset"
  PublicDatasetStudentMountPath string `json:"dataset_student_mount_path" yaml:"datasetStudentMountPath"`
  PublicCourseStudentMountPath string `json:"course_student_mount_path" yaml:"courseStudentMountPath"`
- OpenHydraNamespace string `json:"open-hydra_namespace" yaml:"open-hydraNamespace"`
  // should be no default value but fill it in installation script, because it is a runtime value
  // if not set we won't be able to start gpu pod at all
  DefaultGpuDriver string `json:"default_gpu_driver" yaml:"defaultGpuDriver"`
@@ -129,7 +128,6 @@ func DefaultConfig() *OpenHydraServerConfig {
  PublicCourseStudentMountPath: "/root/notebook/course-public",
  MySqlConfig: DefaultMySqlConfig(),
  EtcdConfig: DefaultEtcdConfig(),
- OpenHydraNamespace: defaultNamespace,
  LeaderElection: DefaultLeaderElection(),
  DefaultGpuDriver: "nvidia.com/gpu",
  GpuResourceKeys: []string{"nvidia.com/gpu", "amd.com/gpu"},

go.mod

@@ -4,15 +4,16 @@ go 1.21.4
 
 require (
  github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
+ github.com/emicklei/go-restful v2.16.0+incompatible
  github.com/emicklei/go-restful/v3 v3.11.0
  github.com/go-sql-driver/mysql v1.7.1
  github.com/google/uuid v1.3.0
- github.com/jedib0t/go-pretty/v6 v6.4.9
  github.com/onsi/ginkgo/v2 v2.13.2
  github.com/onsi/gomega v1.30.0
  github.com/spf13/cobra v1.7.0
  github.com/spf13/pflag v1.0.5
  golang.org/x/sync v0.5.0
+ golang.org/x/text v0.13.0
  gopkg.in/yaml.v2 v2.4.0
  k8s.io/api v0.29.0
  k8s.io/apimachinery v0.29.0
@@ -33,7 +34,6 @@ require (
  github.com/coreos/go-semver v0.3.1 // indirect
  github.com/coreos/go-systemd/v22 v22.5.0 // indirect
  github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/emicklei/go-restful v2.16.0+incompatible // indirect
  github.com/evanphx/json-patch v5.6.0+incompatible // indirect
  github.com/felixge/httpsnoop v1.0.3 // indirect
  github.com/fsnotify/fsnotify v1.7.0 // indirect
@@ -58,7 +58,6 @@ require (
  github.com/josharian/intern v1.0.0 // indirect
  github.com/json-iterator/go v1.1.12 // indirect
  github.com/mailru/easyjson v0.7.7 // indirect
- github.com/mattn/go-runewidth v0.0.13 // indirect
  github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
  github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
  github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -68,7 +67,6 @@ require (
  github.com/prometheus/client_model v0.4.0 // indirect
  github.com/prometheus/common v0.44.0 // indirect
  github.com/prometheus/procfs v0.10.1 // indirect
- github.com/rivo/uniseg v0.2.0 // indirect
  github.com/stoewer/go-strcase v1.2.0 // indirect
  go.etcd.io/etcd/api/v3 v3.5.10 // indirect
  go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
@@ -90,7 +88,6 @@ require (
  golang.org/x/oauth2 v0.10.0 // indirect
  golang.org/x/sys v0.14.0 // indirect
  golang.org/x/term v0.13.0 // indirect
- golang.org/x/text v0.13.0 // indirect
  golang.org/x/time v0.3.0 // indirect
  golang.org/x/tools v0.14.0 // indirect
  google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -112,8 +112,6 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/jedib0t/go-pretty/v6 v6.4.9 h1:vZ6bjGg2eBSrJn365qlxGcaWu09Id+LHtrfDWlB2Usc=
-github.com/jedib0t/go-pretty/v6 v6.4.9/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
 github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -131,8 +129,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
-github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -148,7 +144,6 @@ github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
@@ -159,8 +154,6 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
-github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -181,7 +174,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
@@ -262,7 +254,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=

pkg/apiserver/apiserver.go

@@ -64,7 +64,7 @@ func RunApiServer(recommendOption *option.Options, config *config.OpenHydraServe
 
  gApiServer, err := completedConfig.New("open-hydra api-server", genericApiServer.NewEmptyDelegate())
  if err != nil {
- slog.Error(fmt.Sprintf("Failed to create genericApiServer: %v", err))
+ slog.Error("Failed to create genericApiServer", "error", err)
  return err
 
  }
@@ -80,9 +80,9 @@ func RunApiServer(recommendOption *option.Options, config *config.OpenHydraServe
  // register the discovery service
  registerDiscoveryService(gApiServer)
  // register the api resource
- err = registerApiResource(gApiServer, config)
+ err = registerApiResource(gApiServer, config, stopChan)
  if err != nil {
- slog.Error(fmt.Sprintf("Failed to register api resource: %v", err))
+ slog.Error("Failed to register api resource", "error", err)
  return err
  }
 

pkg/apiserver/service-register.go

@@ -46,7 +46,7 @@ func registerDiscoveryService(apiServer *genericApiServer.GenericAPIServer) {
  apiServer.Handler.GoRestfulContainer.Add(ws)
 }
 
-func registerApiResource(apiServer *genericApiServer.GenericAPIServer, config *config.OpenHydraServerConfig) error {
+func registerApiResource(apiServer *genericApiServer.GenericAPIServer, config *config.OpenHydraServerConfig, stopChan <-chan struct{}) error {
  ws := getWebService()
  resourceIndexPathTemplate := "/apis/%s/%s"
  resourceIndexPath := fmt.Sprintf(resourceIndexPathTemplate, option.GroupVersion.Group, option.GroupVersion.Version)
@@ -73,17 +73,17 @@ func registerApiResource(apiServer *genericApiServer.GenericAPIServer, config *c
 
  err := db.InitDb()
  if err != nil {
- slog.Error("Failed to init db", err)
+ slog.Error("Failed to init db", "error", err)
  return err
  }
 
  kubeClient, err := kubernetes.NewForConfig(config.KubeConfig)
  if err != nil {
- slog.Error("Failed to create kube client", err)
+ slog.Error("Failed to create kube client", "error", err)
  return err
  }
 
- RBuilder := openHydraHandler.NewOpenHydraRouteBuilder(db, config, ws, kubeClient, openHydraK8s.NewDefaultK8sHelper())
+ RBuilder := openHydraHandler.NewOpenHydraRouteBuilder(db, ws, kubeClient, openHydraK8s.NewDefaultK8sHelper(kubeClient, stopChan))
  RBuilder.AddXUserListRoute()
  RBuilder.AddXUserCreateRoute()
  RBuilder.AddXUserGetRoute()

pkg/open-hydra/comment.go

@@ -21,6 +21,8 @@ import (
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const OpenhydraNamespace = "open-hydra"
+
 type HttpErrMsg struct {
  Error string `json:"errMsg"`
 }

pkg/open-hydra/course-handler.go

@@ -71,7 +71,15 @@ func (builder *OpenHydraRouteBuilder) AddCourseCreateRoute() {
 }
 
 func (builder *OpenHydraRouteBuilder) CourseCreateRouteHandler(request *restful.Request, response *restful.Response) {
- err := request.Request.ParseMultipartForm(builder.Config.PublicCourseMaxSize)
+
+ serverConfig, err := builder.GetServerConfigFromConfigMap()
+ if err != nil {
+ writeHttpResponseAndLogError(response, http.StatusInternalServerError,
+ fmt.Sprintf("Failed to get server config: %v", err))
+ return
+ }
+
+ err = request.Request.ParseMultipartForm(serverConfig.PublicCourseMaxSize)
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to parse multipart form: %v", err))
@@ -89,9 +97,9 @@ func (builder *OpenHydraRouteBuilder) CourseCreateRouteHandler(request *restful.
 
  // we need to get config map openhydra-plugin first
  // TODO: we should use informer to cache config map instead of query api-server directly for performance
- pluginConfigMap, err := builder.k8sHelper.GetMap("openhydra-plugin", builder.Config.OpenHydraNamespace, builder.kubeClient)
+ pluginConfigMap, err := builder.k8sHelper.GetConfigMap("openhydra-plugin", OpenhydraNamespace)
  if err != nil {
- writeHttpResponseAndLogError(response, http.StatusInternalServerError, fmt.Sprintf("Failed to get configmap: %v", err))
+ writeHttpResponseAndLogError(response, http.StatusInternalServerError, fmt.Sprintf("Failed to get config map: %v", err))
  return
  }
 
@@ -145,7 +153,7 @@ func (builder *OpenHydraRouteBuilder) CourseCreateRouteHandler(request *restful.
  return
  }
 
- coursePath, err := filepath.Abs(filepath.Join(builder.Config.PublicCourseBasePath, name))
+ coursePath, err := filepath.Abs(filepath.Join(serverConfig.PublicCourseBasePath, name))
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to get course path: %v", err.Error()))
@@ -190,8 +198,16 @@ func (builder *OpenHydraRouteBuilder) AddCourseUpdateRoute() {
 }
 
 func (builder *OpenHydraRouteBuilder) CourseUpdateRouteHandler(request *restful.Request, response *restful.Response) {
+
+ serverConfig, err := builder.GetServerConfigFromConfigMap()
+ if err != nil {
+ writeHttpResponseAndLogError(response, http.StatusInternalServerError,
+ fmt.Sprintf("Failed to get server config: %v", err))
+ return
+ }
+
  courseName := request.PathParameter("course-name")
- err := request.Request.ParseMultipartForm(builder.Config.PublicCourseMaxSize)
+ err = request.Request.ParseMultipartForm(serverConfig.PublicCourseMaxSize)
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to parse multipart form: %v", err))
@@ -229,7 +245,7 @@ func (builder *OpenHydraRouteBuilder) CourseUpdateRouteHandler(request *restful.
 
  // update course file
 
- coursePath, err := filepath.Abs(filepath.Join(builder.Config.PublicCourseBasePath, course.Name))
+ coursePath, err := filepath.Abs(filepath.Join(serverConfig.PublicCourseBasePath, course.Name))
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to get course path: %v", err.Error()))
@@ -254,13 +270,21 @@ func (builder *OpenHydraRouteBuilder) AddCourseDeleteRoute() {
 }
 
 func (builder *OpenHydraRouteBuilder) CourseDeleteRouteHandler(request *restful.Request, response *restful.Response) {
+
+ serverConfig, err := builder.GetServerConfigFromConfigMap()
+ if err != nil {
+ writeHttpResponseAndLogError(response, http.StatusInternalServerError,
+ fmt.Sprintf("Failed to get server config: %v", err))
+ return
+ }
+
  courseName := request.PathParameter("course-name")
- err := builder.Database.DeleteCourse(courseName)
+ err = builder.Database.DeleteCourse(courseName)
  if err != nil {
  writeAPIStatusError(response, err)
  return
  }
- coursePath, err := filepath.Abs(filepath.Join(builder.Config.PublicCourseBasePath, courseName))
+ coursePath, err := filepath.Abs(filepath.Join(serverConfig.PublicCourseBasePath, courseName))
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusInternalServerError,
  fmt.Sprintf("Failed to get course path: %v", err.Error()))

pkg/open-hydra/dataset-handler.go

@@ -84,7 +84,15 @@ func (builder *OpenHydraRouteBuilder) AddDatasetCreateRoute() {
 }
 
 func (builder *OpenHydraRouteBuilder) DatasetCreateRouteHandler(request *restful.Request, response *restful.Response) {
- err := request.Request.ParseMultipartForm(builder.Config.PublicDatasetMaxSize)
+
+ serverConfig, err := builder.GetServerConfigFromConfigMap()
+ if err != nil {
+ writeHttpResponseAndLogError(response, http.StatusInternalServerError,
+ fmt.Sprintf("Failed to get server config: %v", err))
+ return
+ }
+
+ err = request.Request.ParseMultipartForm(serverConfig.PublicDatasetMaxSize)
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to parse multipart form: %v", err))
@@ -120,7 +128,7 @@ func (builder *OpenHydraRouteBuilder) DatasetCreateRouteHandler(request *restful
  return
  }
 
- datasetPath, err := filepath.Abs(filepath.Join(builder.Config.PublicDatasetBasePath, name))
+ datasetPath, err := filepath.Abs(filepath.Join(serverConfig.PublicDatasetBasePath, name))
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to get dataset path: %v", err.Error()))
@@ -162,8 +170,16 @@ func (builder *OpenHydraRouteBuilder) AddDatasetUpdateRoute() {
 }
 
 func (builder *OpenHydraRouteBuilder) DatasetUpdateRouteHandler(request *restful.Request, response *restful.Response) {
+
+ serverConfig, err := builder.GetServerConfigFromConfigMap()
+ if err != nil {
+ writeHttpResponseAndLogError(response, http.StatusInternalServerError,
+ fmt.Sprintf("Failed to get server config: %v", err))
+ return
+ }
+
  datasetName := request.PathParameter("dataset-name")
- err := request.Request.ParseMultipartForm(builder.Config.PublicDatasetMaxSize)
+ err = request.Request.ParseMultipartForm(serverConfig.PublicDatasetMaxSize)
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to parse multipart form: %v", err))
@@ -201,7 +217,7 @@ func (builder *OpenHydraRouteBuilder) DatasetUpdateRouteHandler(request *restful
 
  // update dataset file
 
- datasetPath, err := filepath.Abs(filepath.Join(builder.Config.PublicDatasetBasePath, dataset.Name))
+ datasetPath, err := filepath.Abs(filepath.Join(serverConfig.PublicDatasetBasePath, dataset.Name))
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusBadRequest,
  fmt.Sprintf("Failed to get dataset path: %v", err.Error()))
@@ -226,13 +242,21 @@ func (builder *OpenHydraRouteBuilder) AddDatasetDeleteRoute() {
 }
 
 func (builder *OpenHydraRouteBuilder) DatasetDeleteRouteHandler(request *restful.Request, response *restful.Response) {
+
+ serverConfig, err := builder.GetServerConfigFromConfigMap()
+ if err != nil {
+ writeHttpResponseAndLogError(response, http.StatusInternalServerError,
+ fmt.Sprintf("Failed to get server config: %v", err))
+ return
+ }
+
  datasetName := request.PathParameter("dataset-name")
- err := builder.Database.DeleteDataset(datasetName)
+ err = builder.Database.DeleteDataset(datasetName)
  if err != nil {
  writeAPIStatusError(response, err)
  return
  }
- datasetPath, err := filepath.Abs(filepath.Join(builder.Config.PublicDatasetBasePath, datasetName))
+ datasetPath, err := filepath.Abs(filepath.Join(serverConfig.PublicDatasetBasePath, datasetName))
  if err != nil {
  writeHttpResponseAndLogError(response, http.StatusInternalServerError,
  fmt.Sprintf("Failed to get dataset path: %v", err.Error()))