feat: improve fetch resource job

core/provider/common.go

@@ -2,8 +2,11 @@ package provider
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/goto/guardian/domain"
 )
 
@@ -37,6 +40,38 @@ func (m PermissionManager) GetPermissions(pc *domain.ProviderConfig, resourceTyp
 	return nil, ErrInvalidResourceType
 }
 
+func normalizeDetails(details map[string]interface{}) (map[string]interface{}, error) {
+	jsonData, err := json.Marshal(details)
+	if err != nil {
+		return nil, err
+	}
+
+	var normalized map[string]interface{}
+	if err := json.Unmarshal(jsonData, &normalized); err != nil {
+		return nil, err
+	}
+
+	return normalized, nil
+}
+
+func compareResources(existingResource, newResource domain.Resource) bool {
+	opts := cmp.Options{
+		cmpopts.IgnoreFields(domain.Resource{}, "ID", "CreatedAt", "UpdatedAt"),
+		cmpopts.SortSlices(func(x, y map[string]any) bool {
+			return x["name"].(string) < y["name"].(string) // Assumes each entry has a unique name field
+		}),
+		cmpopts.EquateEmpty(),
+	}
+	normalizedExistingDetails, _ := normalizeDetails(existingResource.Details)
+	normalizedNewDetails, _ := normalizeDetails(newResource.Details)
+	existingResource.Details = normalizedExistingDetails
+	newResource.Details = normalizedNewDetails
+	if diff := cmp.Diff(existingResource, newResource, opts); diff != "" {
+		return true
+	}
+	return false
+}
+
 type UnimplementedClient struct{}
 
 func (c *UnimplementedClient) CreateConfig(*domain.ProviderConfig) error {

core/provider/service.go

@@ -10,8 +10,6 @@ import (
 	"github.com/goto/guardian/pkg/evaluator"
 
 	"github.com/go-playground/validator/v10"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/goto/guardian/domain"
 	"github.com/goto/guardian/pkg/log"
 	"github.com/goto/guardian/plugins/providers"
@@ -257,14 +255,13 @@ func (s *Service) Update(ctx context.Context, p *domain.Provider) error {
 }
 
 // FetchResources fetches all resources for all registered providers
-// when is this used
 func (s *Service) FetchResources(ctx context.Context) error {
 	providers, err := s.repository.Find(ctx)
 	if err != nil {
 		return err
 	}
-
 	failedProviders := map[string]error{}
+	updatedProviders := 0
 	for _, p := range providers {
 		startTime := time.Now()
 		s.logger.Info(ctx, "fetching resources", "provider_urn", p.URN)
@@ -284,7 +281,10 @@ func (s *Service) FetchResources(ctx context.Context) error {
 			s.logger.Error(ctx, "failed to add resources", "provider_urn", p.URN, "error", err)
 		}
 		s.logger.Info(ctx, "fetching resources completed", "provider_urn", p.URN, "duration", time.Since(startTime))
+		updatedProviders++
 	}
+	s.logger.Info(ctx, "existing provider", "count", len(providers))
+	s.logger.Info(ctx, "updated providers", "count", updatedProviders)
 	if len(failedProviders) > 0 {
 		var urns []string
 		for providerURN, err := range failedProviders {
@@ -612,10 +612,8 @@ func (s *Service) getResources(ctx context.Context, p *domain.Provider) ([]*doma
 	flattenedProviderResources := flattenResources(filteredResources)
 
 	existingProviderResources := map[string]bool{}
-	opts := cmp.Options{
-		cmpopts.IgnoreFields(domain.Resource{}, "ID", "CreatedAt", "UpdatedAt"),
-	}
-	isUpdated := false
+
+	isResourceUpdated := 0
 	for _, newResource := range flattenedProviderResources {
 		for _, existingResource := range existingGuardianResources {
 			if existingResource.Type == newResource.Type && existingResource.URN == newResource.URN {
@@ -629,16 +627,16 @@ func (s *Service) getResources(ctx context.Context, p *domain.Provider) ([]*doma
 					} else {
 						newResource.Details = existingDetails
 					}
-					if diff := cmp.Diff(existingResource, newResource, opts); diff != "" {
-						isUpdated = true
+					if isUpdated := compareResources(*existingResource, *newResource); isUpdated {
+						isResourceUpdated++
 					}
 				}
 				existingProviderResources[existingResource.ID] = true
 				break
 			}
 		}
 	}
-	if !isUpdated {
+	if isResourceUpdated == 0 && len(existingGuardianResources) == len(flattenedProviderResources) {
 		return []*domain.Resource{}, nil
 	}
 

core/provider/service_test.go

@@ -400,6 +400,13 @@ func (s *ServiceTestSuite) TestFetchResources() {
 					},
 				},
 			},
+			{
+				ID:           "12ß",
+				ProviderType: mockProviderType,
+				ProviderURN:  mockProvider,
+				Type:         "test-resource-type",
+				URN:          "test-resource-urn-2",
+			},
 		}
 
 		expectedProvider := providers[0]