Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: metrics and build failed #1670

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

fix: metrics and build failed #1670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d36c3b2
fix: collect log nil points
yangkaa Jun 12, 2023
0cdc906
fix: metrics conflict
yangkaa Jun 14, 2023
123f2de
fix: metrics conflict2
yangkaa Jun 14, 2023
57f1775
fix: build task not scheduled to build node
yangkaa Jun 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 10 additions & 12 deletions builder/build/code_build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,20 +365,18 @@ func (s *slugBuild) runBuildJob(re *Request) error {
}
podSpec := corev1.PodSpec{RestartPolicy: corev1.RestartPolicyOnFailure} // only support never and onfailure
// schedule builder
if re.CacheMode == "hostpath" {
logrus.Debugf("builder cache mode using hostpath, schedule job into current node")
hostIP := os.Getenv("HOST_IP")
if hostIP != "" {
podSpec.NodeSelector = map[string]string{
"kubernetes.io/hostname": hostIP,
}
podSpec.Tolerations = []corev1.Toleration{
{
Operator: "Exists",
},
}
hostIP := os.Getenv("HOST_IP")
if hostIP != "" {
podSpec.NodeSelector = map[string]string{
"kubernetes.io/hostname": hostIP,
}
podSpec.Tolerations = []corev1.Toleration{
{
Operator: "Exists",
},
}
}

logrus.Debugf("request is: %+v", re)

volumes, mounts := s.createVolumeAndMount(re, sourceTarFileName)
Expand Down
45 changes: 28 additions & 17 deletions builder/sources/image.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,16 @@ import (
"time"
)

//ErrorNoAuth error no auth
// ErrorNoAuth error no auth
var ErrorNoAuth = fmt.Errorf("pull image require docker login")

//ErrorNoImage error no image
// ErrorNoImage error no image
var ErrorNoImage = fmt.Errorf("image not exist")

//Namespace containerd image namespace
// Namespace containerd image namespace
var Namespace = "k8s.io"

//ImagePull pull docker image
// ImagePull pull docker image
// Deprecated: use sources.ImageClient.ImagePull instead
func ImagePull(client *containerd.Client, ref string, username, password string, logger event.Logger, timeout int) (*containerd.Image, error) {
printLog(logger, "info", fmt.Sprintf("start get image:%s", ref), map[string]string{"step": "pullimage"})
Expand Down Expand Up @@ -169,7 +169,7 @@ func ImageTag(containerdClient *containerd.Client, source, target string, logger
return nil
}

//ImageNameHandle 解析imagename
// ImageNameHandle 解析imagename
func ImageNameHandle(imageName string) *model.ImageName {
var i model.ImageName
if strings.Contains(imageName, "/") {
Expand Down Expand Up @@ -197,7 +197,7 @@ func ImageNameHandle(imageName string) *model.ImageName {
return &i
}

//ImageNameWithNamespaceHandle if have namespace,will parse namespace
// ImageNameWithNamespaceHandle if have namespace,will parse namespace
func ImageNameWithNamespaceHandle(imageName string) *model.ImageName {
var i model.ImageName
if strings.Contains(imageName, "/") {
Expand Down Expand Up @@ -229,8 +229,8 @@ func ImageNameWithNamespaceHandle(imageName string) *model.ImageName {
return &i
}

//ImagePush push image to registry
//timeout minutes of the unit
// ImagePush push image to registry
// timeout minutes of the unit
// Deprecated: use sources.ImageClient.ImagePush instead
func ImagePush(client *containerd.Client, rawRef, user, pass string, logger event.Logger, timeout int) error {
printLog(logger, "info", fmt.Sprintf("start push image：%s", rawRef), map[string]string{"step": "pushimage"})
Expand Down Expand Up @@ -323,15 +323,15 @@ func ImagePush(client *containerd.Client, rawRef, user, pass string, logger even
return nil
}

//TrustedImagePush push image to trusted registry
// TrustedImagePush push image to trusted registry
func TrustedImagePush(containerdClient *containerd.Client, image, user, pass string, logger event.Logger, timeout int) error {
if err := CheckTrustedRepositories(image, user, pass); err != nil {
return err
}
return ImagePush(containerdClient, image, user, pass, logger, timeout)
}

//CheckTrustedRepositories check Repositories is exist ,if not create it.
// CheckTrustedRepositories check Repositories is exist ,if not create it.
func CheckTrustedRepositories(image, user, pass string) error {
ref, err := reference.ParseNormalizedNamed(image)
if err != nil {
Expand Down Expand Up @@ -389,7 +389,7 @@ func EncodeAuthToBase64(authConfig types.AuthConfig) (string, error) {
return base64.URLEncoding.EncodeToString(buf), nil
}

//ImageBuild use kaniko build image
// ImageBuild use kaniko build image
func ImageBuild(contextDir, cachePVCName, cacheMode, RbdNamespace, ServiceID, DeployVersion string, logger event.Logger, buildType, plugImageName, KanikoImage string, KanikoArgs []string) error {
// create image name
var buildImageName string
Expand Down Expand Up @@ -421,6 +421,17 @@ func ImageBuild(contextDir, cachePVCName, cacheMode, RbdNamespace, ServiceID, De
},
}
podSpec := corev1.PodSpec{RestartPolicy: corev1.RestartPolicyOnFailure} // only support never and onfailure
hostIP := os.Getenv("HOST_IP")
if hostIP != "" {
podSpec.NodeSelector = map[string]string{
"kubernetes.io/hostname": hostIP,
}
podSpec.Tolerations = []corev1.Toleration{
{
Operator: "Exists",
},
}
}
volumes, volumeMounts := CreateVolumesAndMounts(contextDir, buildType, cacheMode, cachePVCName)
podSpec.Volumes = volumes
// container config
Expand Down Expand Up @@ -460,7 +471,7 @@ func ImageBuild(contextDir, cachePVCName, cacheMode, RbdNamespace, ServiceID, De
return nil
}

//ImageInspectWithRaw get image inspect
// ImageInspectWithRaw get image inspect
func ImageInspectWithRaw(dockerCli *client.Client, image string) (*types.ImageInspect, error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -471,7 +482,7 @@ func ImageInspectWithRaw(dockerCli *client.Client, image string) (*types.ImageIn
return &ins, nil
}

//ImageSave save image to tar file
// ImageSave save image to tar file
// destination destination file name eg. /tmp/xxx.tar
func ImageSave(dockerCli *client.Client, image, destination string, logger event.Logger) error {
ctx, cancel := context.WithCancel(context.Background())
Expand All @@ -484,7 +495,7 @@ func ImageSave(dockerCli *client.Client, image, destination string, logger event
return CopyToFile(destination, rc)
}

//MultiImageSave save multi image to tar file
// MultiImageSave save multi image to tar file
// destination destination file name eg. /tmp/xxx.tar
func MultiImageSave(ctx context.Context, dockerCli *client.Client, destination string, logger event.Logger, images ...string) error {
rc, err := dockerCli.ImageSave(ctx, images)
Expand All @@ -495,7 +506,7 @@ func MultiImageSave(ctx context.Context, dockerCli *client.Client, destination s
return CopyToFile(destination, rc)
}

//ImageLoad load image from tar file
// ImageLoad load image from tar file
// destination destination file name eg. /tmp/xxx.tar
func ImageLoad(dockerCli *client.Client, tarFile string, logger event.Logger) error {
ctx, cancel := context.WithCancel(context.Background())
Expand Down Expand Up @@ -537,7 +548,7 @@ func ImageLoad(dockerCli *client.Client, tarFile string, logger event.Logger) er
return nil
}

//ImageImport save image to tar file
// ImageImport save image to tar file
// source source file name eg. /tmp/xxx.tar
func ImageImport(dockerCli *client.Client, image, source string, logger event.Logger) error {
ctx, cancel := context.WithCancel(context.Background())
Expand Down Expand Up @@ -608,7 +619,7 @@ func CopyToFile(outfile string, r io.Reader) error {
return nil
}

//ImageRemove remove image
// ImageRemove remove image
func ImageRemove(containerdClient *containerd.Client, image string) error {
ctx := namespaces.WithNamespace(context.Background(), Namespace)
imageStore := containerdClient.ImageService()
Expand Down
30 changes: 15 additions & 15 deletions node/nodem/logger/manage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ import (
_ "github.com/containerd/containerd/api/events"
)

//RFC3339NanoFixed time format
// RFC3339NanoFixed time format
var RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"

//ContainerLogManage container log manage
// ContainerLogManage container log manage
type ContainerLogManage struct {
ctx context.Context
cancel context.CancelFunc
Expand All @@ -47,7 +47,7 @@ type ContainerLogManage struct {
containerLogs sync.Map
}

//CreatContainerLogManage create a container log manage
// CreatContainerLogManage create a container log manage
func CreatContainerLogManage(conf *option.Conf) *ContainerLogManage {
ctx, cancel := context.WithCancel(context.Background())
return &ContainerLogManage{
Expand All @@ -58,7 +58,7 @@ func CreatContainerLogManage(conf *option.Conf) *ContainerLogManage {
}
}

//Start start
// Start start
func (c *ContainerLogManage) Start() error {
errchan := make(chan error)
go func() {
Expand All @@ -73,7 +73,7 @@ func (c *ContainerLogManage) Start() error {
return nil
}

//Stop stop all logger
// Stop stop all logger
func (c *ContainerLogManage) Stop() {
c.containerLogs.Range(func(k, v interface{}) bool {
cl := v.(*ContainerLog)
Expand Down Expand Up @@ -185,8 +185,8 @@ func (c *ContainerLogManage) loollist() {
return
case <-ticker.C:
for _, container := range c.listContainer() {
cj, _ := c.getContainer(container.GetId())
if cj.GetLogPath() == "" {
cj, err := c.getContainer(container.GetId())
if err != nil || cj.GetLogPath() == "" {
continue
}
if cj.ContainerRuntime == sources.ContainerRuntimeDocker {
Expand Down Expand Up @@ -227,7 +227,7 @@ func (c *ContainerLogManage) listAndWatchContainer(errchan chan error) {
}
}

//Out -
// Out -
type Out struct {
Timestamp time.Time
Namespace string
Expand All @@ -254,7 +254,7 @@ func createContainerLog(ctx context.Context, container *sources.ContainerDesc, r
}
}

//ContainerLog container log struct
// ContainerLog container log struct
type ContainerLog struct {
ctx context.Context
cancel context.CancelFunc
Expand All @@ -267,7 +267,7 @@ type ContainerLog struct {
stoped *bool
}

//StartLogging start copy log
// StartLogging start copy log
func (container *ContainerLog) StartLogging() error {
loggers, err := container.startLogger()
if err != nil {
Expand All @@ -285,7 +285,7 @@ func (container *ContainerLog) StartLogging() error {
return nil
}

//ContainerLoggerConfig logger config
// ContainerLoggerConfig logger config
type ContainerLoggerConfig struct {
Name string
Options map[string]string
Expand Down Expand Up @@ -321,7 +321,7 @@ func getLoggerConfig(envs []string) []*ContainerLoggerConfig {
return re
}

//ErrNeglectedContainer not define logger name
// ErrNeglectedContainer not define logger name
var ErrNeglectedContainer = fmt.Errorf("Neglected container")

// containerInfo is extra info returned by containerd grpc api
Expand Down Expand Up @@ -475,7 +475,7 @@ func (container *ContainerLog) startLogger() ([]Logger, error) {
return loggers, nil
}

//Restart restart
// Restart restart
func (container *ContainerLog) Restart() {
if container == nil {
return
Expand All @@ -488,7 +488,7 @@ func (container *ContainerLog) Restart() {
}
}

//Stop stop copy container log
// Stop stop copy container log
func (container *ContainerLog) Stop() {
if container.LogCopier != nil {
container.LogCopier.Close()
Expand All @@ -499,7 +499,7 @@ func (container *ContainerLog) Stop() {
logrus.Debugf("rainbond logger stop for container %s", container.ContainerStatus.GetMetadata().GetName())
}

//Close close
// Close close
func (container *ContainerLog) Close() {
if container.LogCopier != nil {
container.LogCopier.Close()
Expand Down
17 changes: 10 additions & 7 deletions worker/master/controller/thirdcomponent/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package thirdcomponent

import (
"context"
"os"
"reflect"
"time"

Expand Down Expand Up @@ -59,7 +60,7 @@ type Reconciler struct {
concurrentReconciles int
applyer apply.Applicator
discoverPool *DiscoverPool
discoverNum prometheus.Gauge
discoverNum *prometheus.Desc

informer runtimecache.Informer
lister rainbondlistersv1alpha1.ThirdComponentLister
Expand Down Expand Up @@ -404,7 +405,11 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {

// Collect -
func (r *Reconciler) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(r.discoverNum.Desc(), prometheus.GaugeValue, r.discoverPool.GetSize())
hostname, err := os.Hostname()
if err != nil {
hostname = os.Getenv("POD_IP")
}
ch <- prometheus.MustNewConstMetric(r.discoverNum, prometheus.GaugeValue, r.discoverPool.GetSize(), hostname)
}

// Setup adds a controller that reconciles AppDeployment.
Expand All @@ -422,11 +427,9 @@ func Setup(ctx context.Context, mgr ctrl.Manager) (*Reconciler, error) {
restConfig: mgr.GetConfig(),
Scheme: mgr.GetScheme(),
applyer: apply.NewAPIApplicator(mgr.GetClient()),
discoverNum: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "controller",
Name: "third_component_discover_number",
Help: "Number of running endpoint discover worker of third component.",
}),
discoverNum: prometheus.NewDesc(prometheus.BuildFQName("controller", "", "third_component_discover_number"),
"Number of running endpoint discover worker of third component.",
[]string{"hostname"}, nil),
informer: informer,
lister: lister,
recorder: recorder,
Expand Down