feat: add basic auth to allow target

tibuild/api/api.go

@@ -102,7 +102,7 @@ func routeRestAPI(router *gin.Engine, cfg *configs.ConfigYaml) {
  panic(err)
  }
  devBuildGroup := apiGroup.Group("/devbuilds")
- devBuildHandler := controllers.NewDevBuildHandler(context.Background(), jenkins, database.DBConn.DB)
+ devBuildHandler := controllers.NewDevBuildHandler(context.Background(), jenkins, database.DBConn.DB, cfg.RestApiSecret)
  {
  devBuildGroup.POST("", devBuildHandler.Create)
  devBuildGroup.GET("", devBuildHandler.List)

tibuild/commons/configs/config.go

@@ -4,6 +4,7 @@ package configs
 
 import (
  "fmt"
+
  "github.com/jinzhu/configor"
 )
 
@@ -29,12 +30,22 @@ type ConfigYaml struct {
  Github struct {
  Token string
  }
+
+ RestApiSecret RestApiSecret
+}
+
+type RestApiSecret struct {
+ AdminToken string
+ TiBuildToken string
 }
 
 var Config = &ConfigYaml{}
 
 // Load config from file into 'Config' variable
 func LoadConfig(file string) {
  fmt.Printf("file:%s\n", file)
- configor.Load(Config, file)
+ err := configor.Load(Config, file)
+ if err != nil {
+ panic(err)
+ }
 }

tibuild/commons/configs/config_test.go

@@ -2,9 +2,12 @@ package configs
 
 import (
  "testing"
+
+ "github.com/stretchr/testify/assert"
 )
 
 func TestLoadConfig(t *testing.T) {
- t.Skip()
- LoadConfig("../../config.yaml")
+ LoadConfig("../../configs/config_example.yaml")
+ cfg := Config.RestApiSecret
+ assert.NotEmpty(t, cfg.TiBuildToken)
 }

tibuild/configs/config_example.yaml

@@ -3,16 +3,20 @@
 # All configuration information keys is in /tirelease/commons/configs/config.go in struct of ConfigYaml
 # Config like this:
 mysql:
- username: ""
- password: ""
- host: ""
- port: ""
- database: ""
+ username: "your user name"
+ password: "your password"
+ host: "127.0.0.1"
+ port: "127.0.0.1"
+ database: "you_db_name"
  charset: ""
  timezone: ""
 
 jenkins:
- username: ""
- password: ""
+ username: "your jenkins account"
+ password: "your password"
 github:
  token: ""
+
+restapisecret:
+ admintoken: "1"
+ tibuildtoken: "2"

tibuild/docs/docs.go

@@ -485,6 +485,9 @@ const docTemplate = `{
  "productDockerfile": {
  "type": "string"
  },
+ "targetImg": {
+ "type": "string"
+ },
  "version": {
  "type": "string"
  }

tibuild/docs/swagger.json

@@ -473,6 +473,9 @@
  "productDockerfile": {
  "type": "string"
  },
+ "targetImg": {
+ "type": "string"
+ },
  "version": {
  "type": "string"
  }

tibuild/docs/swagger.yaml

@@ -110,6 +110,8 @@ definitions:
  type: string
  productDockerfile:
  type: string
+ targetImg:
+ type: string
  version:
  type: string
  type: object

tibuild/pkg/rest/controller/common.go

@@ -33,6 +33,8 @@ func errorToCode(err error) int {
  return 400
  } else if errors.Is(err, service.ErrServerRefuse) {
  return 422
+ } else if errors.Is(err, service.ErrAuth) {
+ return 401
  } else {
  return 500
  }

tibuild/pkg/rest/controller/dev_build_handler.go

@@ -10,23 +10,50 @@ import (
  "github.com/gin-gonic/gin"
  "gorm.io/gorm"
 
+ "github.com/PingCAP-QE/ee-apps/tibuild/commons/configs"
  "github.com/PingCAP-QE/ee-apps/tibuild/pkg/rest/repo"
  "github.com/PingCAP-QE/ee-apps/tibuild/pkg/rest/service"
 )
 
 type DevBuildHandler struct {
- svc service.DevBuildService
+ svc service.DevBuildService
+ auth configs.RestApiSecret
 }
 
-func NewDevBuildHandler(ctx context.Context, jenkins service.Jenkins, db *gorm.DB) *DevBuildHandler {
+func NewDevBuildHandler(ctx context.Context, jenkins service.Jenkins, db *gorm.DB, auth configs.RestApiSecret) *DevBuildHandler {
  db.AutoMigrate(&service.DevBuild{})
  return &DevBuildHandler{svc: service.DevbuildServer{
  Repo: repo.DevBuildRepo{Db: db},
  Jenkins: jenkins,
  Now: time.Now},
+ auth: auth,
  }
 }
 
+func (h DevBuildHandler) authenticate(c *gin.Context) (context.Context, error) {
+ user, passwd, ok := c.Request.BasicAuth()
+ if !ok {
+ return c.Request.Context(), nil
+ }
+ if user == service.AdminApiAccount {
+ if passwd == h.auth.AdminToken {
+ ctx := context.WithValue(c.Request.Context(), service.KeyOfApiAccount, user)
+ return ctx, nil
+ } else {
+ return nil, fmt.Errorf("authenticate error%w", service.ErrAuth)
+ }
+ }
+ if user == service.TibuildApiAccount {
+ if passwd == h.auth.TiBuildToken {
+ ctx := context.WithValue(c.Request.Context(), service.KeyOfApiAccount, user)
+ return ctx, nil
+ } else {
+ return nil, fmt.Errorf("authenticate error%w", service.ErrAuth)
+ }
+ }
+ return c.Request.Context(), nil
+}
+
 // CreateDevbuild godoc
 // @Summary create and trigger devbuild
 // @Description create and trigger devbuild
@@ -40,8 +67,13 @@ func NewDevBuildHandler(ctx context.Context, jenkins service.Jenkins, db *gorm.D
 // @Failure 500 {object} HTTPError
 // @Router /api/devbuilds [post]
 func (h DevBuildHandler) Create(c *gin.Context) {
+ ctx, err := h.authenticate(c)
+ if err != nil {
+ respondError(c, err)
+ return
+ }
  params := service.DevBuild{}
- err := bindParam(&params, c)
+ err = bindParam(&params, c)
  if err != nil {
  return
  }
@@ -51,7 +83,7 @@ func (h DevBuildHandler) Create(c *gin.Context) {
  respondError(c, fmt.Errorf("%s%w", err.Error(), service.ErrBadRequest))
  return
  }
- entity, err := h.svc.Create(c.Request.Context(), params, query)
+ entity, err := h.svc.Create(ctx, params, query)
  if err != nil {
  respondError(c, err)
  return
@@ -132,6 +164,11 @@ func (h DevBuildHandler) Get(c *gin.Context) {
 // @Failure 500 {object} HTTPError
 // @Router /api/devbuilds/{id}/rerun [post]
 func (h DevBuildHandler) Rerun(c *gin.Context) {
+ ctx, err := h.authenticate(c)
+ if err != nil {
+ respondError(c, err)
+ return
+ }
  idStr := c.Param("id")
  id, err := strconv.Atoi(idStr)
  if err != nil {
@@ -144,7 +181,7 @@ func (h DevBuildHandler) Rerun(c *gin.Context) {
  respondError(c, fmt.Errorf("%s%w", err.Error(), service.ErrBadRequest))
  return
  }
- entity, err := h.svc.Rerun(c.Request.Context(), id, params)
+ entity, err := h.svc.Rerun(ctx, id, params)
  if err != nil {
  respondError(c, err)
  return

tibuild/pkg/rest/repo/dev_build_repo_test.go

@@ -24,7 +24,7 @@ func TestDevBuildCreate(t *testing.T) {
  mock.ExpectBegin()
  now := time.Unix(1, 0)
  mock.ExpectExec("INSERT INTO `dev_builds`").WithArgs(now, "", now, ProductBr, "", "v6.7.0", CommunityEdition, "",
- "AA=BB", "https://raw.example.com/Dockerfile", "", "pingcap/builder", "", false, "", false, "PENDING", 0, "", nil, nil, json.RawMessage("null")).WillReturnResult(sqlmock.NewResult(1, 1))
+ "AA=BB", "https://raw.example.com/Dockerfile", "", "pingcap/builder", "", false, "", false, "", "PENDING", 0, "", nil, nil, json.RawMessage("null")).WillReturnResult(sqlmock.NewResult(1, 1))
  mock.ExpectCommit()
  entity, err := repo.Create(context.TODO(), DevBuild{
  Meta: DevBuildMeta{CreatedAt: now, UpdatedAt: now},
@@ -61,7 +61,7 @@ func TestDevBuildUpdate(t *testing.T) {
  report_text, err := json.Marshal(report)
  require.NoError(t, err)
  mock.ExpectBegin()
- mock.ExpectExec("UPDATE `dev_builds` SET").WithArgs(now, "", sqlmock.AnyArg(), ProductBr, "", "", "", "", "", "", "", "", "", false, "", false, "SUCCESS", 0, "", nil, nil, report_text, 1).WillReturnResult(sqlmock.NewResult(1, 1))
+ mock.ExpectExec("UPDATE `dev_builds` SET").WithArgs(now, "", sqlmock.AnyArg(), ProductBr, "", "", "", "", "", "", "", "", "", false, "", false, "", "SUCCESS", 0, "", nil, nil, report_text, 1).WillReturnResult(sqlmock.NewResult(1, 1))
  mock.ExpectCommit()
  entity, err := repo.Update(context.TODO(),
  1,

tibuild/pkg/rest/service/dev_build_service.go

@@ -23,6 +23,11 @@ func (s DevbuildServer) Create(ctx context.Context, req DevBuild, option DevBuil
  req.Status = DevBuildStatus{}
  req.Meta.CreatedAt = s.Now()
  req.Status.Status = BuildStatusPending
+
+ if err := validatePermission(ctx, &req); err != nil {
+ return nil, fmt.Errorf("%s%w", err.Error(), ErrAuth)
+ }
+
  fillWithDefaults(&req)
  if err := validateReq(req); err != nil {
  return nil, fmt.Errorf("%s%w", err.Error(), ErrBadRequest)
@@ -52,6 +57,7 @@ func (s DevbuildServer) Create(ctx context.Context, req DevBuild, option DevBuil
  "BuilderImg": entity.Spec.BuilderImg,
  "ProductDockerfile": entity.Spec.ProductDockerfile,
  "ProductBaseImg": entity.Spec.ProductBaseImg,
+ "TargetImg": entity.Spec.TargetImg,
  }
  qid, err := s.Jenkins.BuildJob(ctx, jobname, params)
  if err != nil {
@@ -75,6 +81,13 @@ func (s DevbuildServer) Create(ctx context.Context, req DevBuild, option DevBuil
  return &entity, nil
 }
 
+func validatePermission(ctx context.Context, req *DevBuild) error {
+ if req.Spec.TargetImg != "" && ctx.Value(KeyOfApiAccount) != AdminApiAccount {
+ return fmt.Errorf("targetImage deny because of permission")
+ }
+ return nil
+}
+
 func fillWithDefaults(req *DevBuild) {
  spec := &req.Spec
  guessEnterprisePluginRef(spec)
@@ -173,6 +186,9 @@ func validateReq(req DevBuild) error {
  if !hotfixVersionValidator.MatchString((spec.Version)) {
  return fmt.Errorf("verion must be like v7.0.0-20230102... for hotfix")
  }
+ if spec.TargetImg != "" {
+ return fmt.Errorf("target image shall be empty for hotfix")
+ }
  }
  return nil
 }
@@ -279,14 +295,7 @@ type DevBuildRepository interface {
 
 var _ DevBuildService = DevbuildServer{}
 
-var versionValidator *regexp.Regexp
-var hotfixVersionValidator *regexp.Regexp
-var gitRefValidator *regexp.Regexp
-var githubRepoValidator *regexp.Regexp
-
-func init() {
- versionValidator = regexp.MustCompile(`^v(\d+\.\d+)\.\d+.*$`)
- hotfixVersionValidator = regexp.MustCompile(`^v(\d+\.\d+)\.\d+-\d{8,}.*$`)
- gitRefValidator = regexp.MustCompile(`^((v\d.*)|(pull/\d+)|([0-9a-fA-F]{40})|(release-.*)|master|main|(tag/[\w-_]+)|(branch/[\w-_\.]+))$`)
- githubRepoValidator = regexp.MustCompile(`^([\w_-]+/[\w_-]+)$`)
-}
+var versionValidator *regexp.Regexp = regexp.MustCompile(`^v(\d+\.\d+)\.\d+.*$`)
+var hotfixVersionValidator *regexp.Regexp = regexp.MustCompile(`^v(\d+\.\d+)\.\d+-\d{8,}.*$`)
+var gitRefValidator *regexp.Regexp = regexp.MustCompile(`^((v\d.*)|(pull/\d+)|([0-9a-fA-F]{40})|(release-.*)|master|main|(tag/.+)|(branch/.+))$`)
+var githubRepoValidator *regexp.Regexp = regexp.MustCompile(`^([\w_-]+/[\w_-]+)$`)

tibuild/pkg/rest/service/dev_build_service_test.go

@@ -75,7 +75,8 @@ func TestDevBuildCreate(t *testing.T) {
  require.Equal(t, int64(0), entity.Status.PipelineBuildID)
  require.Equal(t, map[string]string{"Edition": "enterprise", "GitRef": "pull/23", "Product": "tidb",
  "Version": "v6.1.2", "PluginGitRef": "master", "IsPushGCR": "false", "IsHotfix": "false", "Features": "",
- "GithubRepo": "pingcap/tidb", "TiBuildID": "1", "BuildEnv": "", "ProductDockerfile": "", "BuilderImg": "", "ProductBaseImg": ""}, mockedJenkins.params)
+ "GithubRepo": "pingcap/tidb", "TiBuildID": "1", "BuildEnv": "", "ProductDockerfile": "", "BuilderImg": "",
+ "ProductBaseImg": "", "TargetImg": ""}, mockedJenkins.params)
  mockedJenkins.resume <- struct{}{}
  time.Sleep(time.Millisecond)
  require.Equal(t, int64(2), mockedRepo.saved.Status.PipelineBuildID)
@@ -144,6 +145,23 @@ func TestDevBuildCreate(t *testing.T) {
  require.NoError(t, err)
  })
 
+ t.Run("validate target image", func(t *testing.T) {
+ obj := DevBuild{Spec: DevBuildSpec{Product: ProductTidb, Version: "v6.1.2", Edition: CommunityEdition, GitRef: "branch/feature/somefeat"}}
+ _, err := server.Create(context.TODO(), obj, DevBuildSaveOption{})
+ require.NoError(t, err)
+
+ obj.Spec.TargetImg = "hub.pingcap.net/temp/tidb:somefeat"
+ _, err = server.Create(context.TODO(), obj, DevBuildSaveOption{})
+ require.ErrorIs(t, err, ErrAuth)
+
+ _, err = server.Create(context.WithValue(context.TODO(), KeyOfApiAccount, "admi"), obj, DevBuildSaveOption{})
+ require.ErrorIs(t, err, ErrAuth)
+
+ _, err = server.Create(context.WithValue(context.TODO(), KeyOfApiAccount, AdminApiAccount), obj, DevBuildSaveOption{})
+ require.NoError(t, err)
+
+ })
+
  t.Run("bad githubRepo", func(t *testing.T) {
  _, err := server.Create(context.TODO(), DevBuild{Spec: DevBuildSpec{Product: ProductTidb, GitRef: "pull/23",
  Version: "v6.1.2", Edition: CommunityEdition, GithubRepo: "aa/bb/cc"}}, DevBuildSaveOption{})

tibuild/pkg/rest/service/error.go

@@ -9,3 +9,4 @@ var ErrInternalError = errors.New("")
 var ErrServerRefuse = errors.New("")
 
 var ErrNotFound = errors.New("")
+var ErrAuth = errors.New("")

tibuild/pkg/rest/service/model.go

@@ -169,6 +169,7 @@ type DevBuildSpec struct {
  IsPushGCR bool `json:"isPushGCR,omitempty"`
  Features string `json:"features,omitempty" gorm:"type:varchar(128)"`
  IsHotfix bool `json:"isHotfix,omitempty"`
+ TargetImg string `json:"targetImg,omitempty" gorm:"type:varchar(128)"`
 }
 
 type GitRef string
@@ -263,3 +264,10 @@ type ImageSyncRequest struct {
  Source string `json:"source"`
  Target string `json:"target"`
 }
+
+type TibuildCtxKey string
+
+var KeyOfApiAccount TibuildCtxKey = "apiAccount"
+
+const AdminApiAccount = "admin"
+const TibuildApiAccount = "tibuild"

tibuild/tbctl/tbctl.py

@@ -12,6 +12,7 @@
 
 NOBLOCK = False
 BUILD_CREATED_BY = ''
+BASIC_AUTH_CREDENTIAL=''
 
 
 def dev_build_url(build_id: int):
@@ -35,11 +36,14 @@ def trigger(args):
  "buildEnv": ' '.join(args.buildEnv) if args.buildEnv else '',
  "productDockerfile": args.productDockerfile,
  "productBaseImg": args.productBaseImg,
- "builderImg":args.builderImg}}
+ "builderImg":args.builderImg,
+ "targetImg": args.targetImg}}
  headers = {
  "Content-Type": "application/json",
  "Accept": "application/json",
  }
+ if BASIC_AUTH_CREDENTIAL:
+ headers['Authorization']="BASIC " + BASIC_AUTH_CREDENTIAL
  body = json.dumps(data).encode()
  req = urllib.request.Request(f"{devbuild_url}?dryrun={args.dryrun}", body, headers, method="POST")
  build_id = 0
@@ -107,6 +111,7 @@ def get_artifact(build: dict) -> str:
 if __name__ == "__main__":
  NOBLOCK = bool(os.environ.get('NOBLOCK'))
  BUILD_CREATED_BY = os.environ.get('BUILD_CREATED_BY') or ''
+ BASIC_AUTH_CREDENTIAL = os.environ.get('BASIC_AUTH_CREDENTIAL') or ''
  top_parser = argparse.ArgumentParser(
  prog='tbctl',
  description='tibuild commandline client'
@@ -133,6 +138,7 @@ def get_artifact(build: dict) -> str:
  parser_trigger.add_argument('--productDockerfile', help='dockerfile url for product')
  parser_trigger.add_argument('--productBaseImg', help='product base image')
  parser_trigger.add_argument('--builderImg', help='specify docker image for builder')
+ parser_trigger.add_argument('--targetImg', help=argparse.SUPPRESS)
  parser_trigger.set_defaults(handler=trigger)
  parser_poll = devbuild.add_parser('poll')
  parser_poll.add_argument('build_id', type=int, help="the triggered build id")