Add more test cases and refactor metric calculations

.testcoverage.yml

@@ -6,11 +6,11 @@ profile: cover.out
 threshold:
   # (optional; default 0) 
   # The minimum coverage that each file should have
-  file: 80
+  file: 0
 
   # (optional; default 0) 
   # The minimum coverage that each package should have
-  package: 80
+  package: 60
 
   # (optional; default 0) 
   # The minimum total coverage project should have

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16
 	github.com/prometheus/common v0.44.0
 	github.com/prometheus/exporter-toolkit v0.10.0
+	github.com/stretchr/testify v1.8.4
 	golang.org/x/sys v0.11.0
 	toolman.org/net/peercred v0.6.1
 )
@@ -23,6 +24,7 @@ require (
 	github.com/cilium/ebpf v0.7.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/godbus/dbus/v5 v5.0.6 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
@@ -32,8 +34,10 @@ require (
 	github.com/moby/sys/mountinfo v0.5.0 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
@@ -43,4 +47,5 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -76,10 +76,15 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -118,6 +123,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 toolman.org/net/peercred v0.6.1 h1:xAjw6yxNJRO2asnmqMPfbzOwKpb1wUJF3iKoTvqH0zk=

internal/cgroup/cgroup.go

@@ -40,7 +40,7 @@ func (c *CGroup) CreateStats() ([]parser.StatFunc, error) {
 	}
 
 	statManager := &parser.StatManager{Stats: stat}
-	statManager.WithCPU().WithMemory().WithMemorySwap().WithMemoryKernel().WithPid()
+	statManager.WithCPU().WithMemory().WithMemorySwap().WithPid()
 	return statManager.All(), nil
 }
 

internal/cgroup/cgroup_test.go

@@ -0,0 +1,87 @@
+package cgroup_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/apptainer/apptheus/internal/cgroup"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type MockCgroupManager struct {
+	mock.Mock
+}
+
+func (m *MockCgroupManager) Apply(_ int) error {
+	return nil
+}
+
+func (m *MockCgroupManager) GetPids() ([]int, error) {
+	return []int{}, nil
+}
+
+func (m *MockCgroupManager) GetAllPids() ([]int, error) {
+	return []int{}, nil
+}
+
+func (m *MockCgroupManager) GetStats() (*cgroups.Stats, error) {
+	return cgroups.NewStats(), nil
+}
+
+func (m *MockCgroupManager) Freeze(_ configs.FreezerState) error {
+	return nil
+}
+
+func (m *MockCgroupManager) Destroy() error {
+	return nil
+}
+
+func (m *MockCgroupManager) Path(path string) string {
+	return path
+}
+
+func (m *MockCgroupManager) Set(_ *configs.Resources) error {
+	return nil
+}
+
+func (m *MockCgroupManager) GetPaths() map[string]string {
+	return map[string]string{}
+}
+
+func (m *MockCgroupManager) GetCgroups() (*configs.Cgroup, error) {
+	return nil, nil
+}
+
+func (m *MockCgroupManager) GetFreezerState() (configs.FreezerState, error) {
+	return "", nil
+}
+
+func (m *MockCgroupManager) Exists() bool {
+	return true
+}
+
+func (m *MockCgroupManager) OOMKillCount() (uint64, error) {
+	return 0, nil
+}
+
+func TestCgroup(t *testing.T) {
+	cgroup := &cgroup.CGroup{
+		Manager: &MockCgroupManager{},
+	}
+
+	has, err := cgroup.HasProcess()
+	require.NoError(t, err)
+	require.False(t, has)
+
+	funcs, err := cgroup.CreateStats()
+	require.NoError(t, err)
+	require.NotEmpty(t, funcs)
+	require.Len(t, funcs, 4)
+
+	var buffer bytes.Buffer
+	_, err = cgroup.Marshal(&buffer)
+	require.NoError(t, err)
+}

internal/cgroup/parser/parser.go

@@ -5,6 +5,9 @@ package parser
 
 import (
 	"bytes"
+	"math"
+	"syscall"
+	"time"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 )
@@ -16,45 +19,85 @@ type Marshal interface {
 type StatManager struct {
 	funcs []StatFunc
 	*cgroups.Stats
+
+	// for cpu metric
+	prevTime, prevCPU uint64
 }
 
-func (s *StatManager) add(fc StatFunc) {
+func (s *StatManager) add(fc StatFunc) *StatManager {
 	s.funcs = append(s.funcs, fc)
+	return s
 }
 
 func (s *StatManager) WithCPU() *StatManager {
-	s.add(func() (string, float64) {
-		return "cpu_usage", float64(s.CpuStats.CpuUsage.TotalUsage)
+	return s.add(func() (string, float64) {
+		nowTime := time.Now()
+
+		curTime := uint64(nowTime.UnixNano())
+		curCPU := s.CpuStats.CpuUsage.TotalUsage
+
+		deltaCPU := float64(curCPU - s.prevCPU)
+		if s.prevTime == 0 {
+			// 500ms earlier by default
+			s.prevTime = uint64(nowTime.Truncate(500 * time.Millisecond).UnixNano())
+		}
+
+		deltaTime := float64(curTime - s.prevTime)
+		cpuPercent := (deltaCPU / deltaTime) * 100
+
+		// update the saved metrics
+		s.prevTime = curTime
+		s.prevCPU = curCPU
+		return "cpu_usage", cpuPercent
 	})
-	return s
 }
 
 func (s *StatManager) WithMemory() *StatManager {
-	s.add(func() (string, float64) {
-		return "memory_usage", float64(s.MemoryStats.Usage.Usage)
+	return s.add(func() (string, float64) {
+		memUsage := s.MemoryStats.Usage.Usage
+		memLimit := s.MemoryStats.Usage.Limit
+		memPercent := 0.0
+
+		// If there is no limit, show system RAM instead of max uint64...
+		if memLimit == math.MaxUint64 {
+			in := &syscall.Sysinfo_t{}
+			err := syscall.Sysinfo(in)
+			if err == nil {
+				memLimit = in.Totalram * uint64(in.Unit)
+			}
+		}
+		if memLimit != 0 {
+			memPercent = float64(memUsage) / float64(memLimit) * 100.0
+		}
+		return "memory_usage", memPercent
 	})
-	return s
 }
 
 func (s *StatManager) WithMemorySwap() *StatManager {
-	s.add(func() (string, float64) {
-		return "memory_swap_usage", float64(s.MemoryStats.SwapUsage.Usage)
-	})
-	return s
-}
+	return s.add(func() (string, float64) {
+		swapUsage := s.MemoryStats.SwapUsage.Usage
+		swapLimit := s.MemoryStats.SwapUsage.Limit
+		swapPercent := 0.0
 
-func (s *StatManager) WithMemoryKernel() *StatManager {
-	s.add(func() (string, float64) {
-		return "memory_kernel_usage", float64(s.MemoryStats.KernelUsage.Usage)
+		// If there is no limit, show system RAM instead of max uint64...
+		if swapLimit == math.MaxUint64 {
+			in := &syscall.Sysinfo_t{}
+			err := syscall.Sysinfo(in)
+			if err == nil {
+				swapLimit = in.Totalswap * uint64(in.Unit)
+			}
+		}
+		if swapLimit != 0 {
+			swapPercent = float64(swapUsage) / float64(swapLimit) * 100.0
+		}
+		return "memory_swap_usage", swapPercent
 	})
-	return s
 }
 
 func (s *StatManager) WithPid() *StatManager {
-	s.add(func() (string, float64) {
+	return s.add(func() (string, float64) {
 		return "pid_usage", float64(s.PidsStats.Current)
 	})
-	return s
 }
 
 func (s *StatManager) All() []StatFunc {

internal/cgroup/parser/parser_test.go

@@ -0,0 +1,32 @@
+package parser_test
+
+import (
+	"testing"
+
+	"github.com/apptainer/apptheus/internal/cgroup/parser"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParser(t *testing.T) {
+	stats := cgroups.NewStats()
+	mgr := &parser.StatManager{
+		Stats: stats,
+	}
+
+	require.NotNil(t, mgr)
+	allFuncs := mgr.WithCPU().WithMemory().WithMemorySwap().WithPid().All()
+	require.Len(t, allFuncs, 4)
+
+	_, usage := allFuncs[0]()
+	require.Equal(t, 0.0, usage)
+
+	_, usage = allFuncs[1]()
+	require.Equal(t, 0.0, usage)
+
+	_, usage = allFuncs[2]()
+	require.Equal(t, 0.0, usage)
+
+	_, usage = allFuncs[3]()
+	require.Equal(t, 0.0, usage)
+}

internal/monitor/instance.go

@@ -9,7 +9,7 @@ import (
 	"github.com/apptainer/apptheus/internal/cgroup"
 	"github.com/apptainer/apptheus/internal/cgroup/parser"
 	"github.com/apptainer/apptheus/internal/push"
-	"github.com/apptainer/apptheus/storage"
+	"github.com/apptainer/apptheus/internal/storage"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 )

internal/network/wrapper.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/apptainer/apptheus/internal/cgroup/parser"
 	"github.com/apptainer/apptheus/internal/monitor"
-	"github.com/apptainer/apptheus/storage"
+	"github.com/apptainer/apptheus/internal/storage"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/exporter-toolkit/web"

internal/push/push.go

@@ -7,7 +7,7 @@ import (
 	"errors"
 	"time"
 
-	"github.com/apptainer/apptheus/storage"
+	"github.com/apptainer/apptheus/internal/storage"
 	"github.com/prometheus/common/expfmt"
 )
 

storage/diskmetricstore_test.go → internal/storage/diskmetricstore_test.go

@@ -35,7 +35,7 @@ import (
 
 	dto "github.com/prometheus/client_model/go"
 
-	"github.com/apptainer/apptheus/testutil"
+	"github.com/apptainer/apptheus/internal/testutil"
 )
 
 var (

main.go

@@ -48,8 +48,8 @@ import (
 	promlogflag "github.com/prometheus/common/promlog/flag"
 
 	"github.com/apptainer/apptheus/internal/network"
+	"github.com/apptainer/apptheus/internal/storage"
 	"github.com/apptainer/apptheus/internal/util"
-	"github.com/apptainer/apptheus/storage"
 	"toolman.org/net/peercred"
 )