Kav/improve inherit attr (#144)

* feat: improve inherit attributes and other small internal improvements

build/compile.mk

@@ -56,5 +56,5 @@ compile-windows32: deps-only
  OUTPUT_FILE="$(BUILD_DIR)windows/$$b.exe" ; \
  echo "=== $(PROJECT_NAME) === [ compile-windows ]: $$OUTPUT_FILE"; \
  BUILD_FILES=`find $(SRCDIR)/cmd/$$b -type f -name "*.go"` ; \
- GOARCH=386 GOOS=windows $(GO_CMD) build -ldflags=$(LDFLAGS) -o $$OUTPUT_FILE $$BUILD_FILES ; \
+ GOARCH=386 CGO_ENABLED=1 GOOS=windows $(GO_CMD) build -ldflags=$(LDFLAGS) -o $$OUTPUT_FILE $$BUILD_FILES ; \
  done

examples/flexConfigs/ecs-task-state-changes-lambda.yml

@@ -1,10 +1,10 @@
-### this file should be renamed to "flex-lambda-ingest.yml" 
+### this file should be renamed to "flex-lambda-ingest.yml"
 ### set a cloudwatch event to trigger the lamda for ECS state changes
 ---
 name: FlexEcsStateChange
 custom_attributes:
  version: 1
-apis: 
+apis:
  - event_type: EcsStateChangeSample
  ingest: true
  inherit_attributes: true
@@ -13,4 +13,4 @@ apis:
  - containers
  rename_keys:
  parent.0: "aws"
- parent.1: "detail"
+ parent.1: "detail"

internal/processor/flattener.go

@@ -444,3 +444,26 @@ func RunLazyFlatten(ds *map[string]interface{}, cfg *load.Config, api int) {
  }
  }
 }
+
+func flattenSlicesAndMaps(data interface{}) map[string]interface{} {
+ switch d := data.(type) {
+ case map[string]interface{}:
+ flattened, err := flatten.Flatten(d, "", flatten.DotStyle)
+ if err == nil {
+ return flattened
+ }
+ case []interface{}:
+ sliceData := map[string]interface{}{}
+ for i, sample := range d {
+ switch sampleData := sample.(type) {
+ case map[string]interface{}:
+ flattened, err := flatten.Flatten(sampleData, "", flatten.DotStyle)
+ if err == nil {
+ sliceData[fmt.Sprintf("%d", i)] = flattened
+ }
+ }
+ }
+ return sliceData
+ }
+ return nil
+}

internal/processor/keys.go

@@ -8,6 +8,7 @@ package processor
 import (
  "fmt"
  "regexp"
+ "strconv"
  "strings"
 
  "github.com/newrelic/nri-flex/internal/formatter"
@@ -103,15 +104,16 @@ func FindStartKey(mainDataset *map[string]interface{}, startKeys []string, inher
  switch mainDs := (*mainDataset)[startSplit[0]].(type) {
  case []interface{}:
  var nestedSlices []interface{}
- for _, nested := range mainDs {
+ for i, nested := range mainDs {
  switch sample := nested.(type) {
  case map[string]interface{}:
  if sample[startSplit[1]] != nil {
  switch nestedSample := sample[startSplit[1]].(type) {
  case map[string]interface{}:
+ nestedSample["flexSliceIndex"] = i
  nestedSlices = append(nestedSlices, nestedSample)
  case []interface{}:
- nestedSlices = append(nestedSlices, nestedSample...)
+ nestedSlices = append(nestedSlices, applyIndexes(i, nestedSample)...)
  }
  }
  }
@@ -138,12 +140,48 @@ func FindStartKey(mainDataset *map[string]interface{}, startKeys []string, inher
  }
 }
 
+func applyIndexes(index int, slices []interface{}) []interface{} {
+ newSlices := []interface{}{}
+ for _, sample := range slices {
+ switch sampleData := sample.(type) {
+ case map[string]interface{}:
+ sampleData["flexSliceIndex"] = index
+ newSlices = append(newSlices, sampleData)
+ default:
+ newSlices = append(newSlices, sample)
+ }
+ }
+ return newSlices
+}
+
 func storeParentAttributes(mainDataset map[string]interface{}, parentAttributes map[string]interface{}, startKey string, level int, inheritAttributes bool) {
  if inheritAttributes {
+ startSplit := strings.Split(startKey, ">")
  for key, val := range mainDataset {
  if key != startKey {
- value := fmt.Sprintf("%v", val)
- if !strings.Contains(value, "map[") {
+ switch valueData := val.(type) {
+ case map[string]interface{}, []interface{}:
+ if len(startSplit) == 2 && key == startSplit[0] {
+ // lazy flatten the slices and maps from the highest level
+ for mapKey, mapVal := range flattenSlicesAndMaps(val) {
+ switch data := mapVal.(type) {
+ case map[string]interface{}:
+ for innerMapKey, innerMapVal := range data {
+ // avoid duplicates from the flattened data
+ if !strings.Contains(innerMapKey, startSplit[1]) {
+ parentAttributes[fmt.Sprintf("parent.%d.%v.%v", level, mapKey, innerMapKey)] = fmt.Sprintf("%v", innerMapVal)
+ }
+ }
+ default:
+ // avoid duplicates from the flattened data
+ if !strings.Contains(mapKey, startSplit[1]) {
+ parentAttributes[fmt.Sprintf("parent.%d.%v", level, mapKey)] = fmt.Sprintf("%v", mapVal)
+ }
+ }
+ }
+ }
+ default:
+ value := fmt.Sprintf("%v", valueData)
  parentAttributes[fmt.Sprintf("parent.%d.%v", level, key)] = value
  }
  }
@@ -161,8 +199,25 @@ func applyParentAttributes(mainDataset map[string]interface{}, datasets []interf
  switch switchDs := dataset.(type) {
  case map[string]interface{}:
  for key, val := range parentAttributes {
- switchDs[key] = val
+ // check if this is a nested parent, and only apply if the index matches
+ matches := formatter.RegMatch(key, "parent\\.(\\d+)\\.(\\d+)\\.(.+)")
+ if len(matches) > 1 {
+ sliceIndex := -1
+ if switchDs["flexSliceIndex"] != nil {
+ sliceIndex = switchDs["flexSliceIndex"].(int)
+ }
+ matchIndex2, err := strconv.Atoi(matches[1])
+ if err == nil {
+ // no need to add the second index into the key as we've unpacked at the matched level
+ if sliceIndex == matchIndex2 {
+ switchDs[fmt.Sprintf("parent.%v.%v", matches[0], matches[2])] = val
+ }
+ }
+ } else {
+ switchDs[key] = val
+ }
  }
+ delete(switchDs, "flexSliceIndex")
  }
  }
  }

internal/processor/keys_test.go

@@ -400,9 +400,8 @@ func TestStartKeys(t *testing.T) {
  },
  },
  },
- startKeys: []string{"def>def2"},
- // TODO: shouldn't def2 be considered as another key and store also xyz as parent attribute?
- expected: `{"def2":[{"def3":{"def4":4},"parent.0.abc":"1"}]}`,
+ startKeys: []string{"def>def2"},
+ expected: `{"def2":[{"def3":{"def4":4},"parent.0.abc":"1","parent.0.xyz":"test"}]}`,
  inheritAttrs: true,
  },
  "NestedKeyWithMapAndArray_InheritAttrs": {