From 344e86e14a433fc4b972d675012da940ddc3f57d Mon Sep 17 00:00:00 2001 From: Bowen Xiao Date: Fri, 25 Oct 2024 17:49:45 -0700 Subject: [PATCH 1/4] Admin_ES_cmds_test Part II --- tools/cli/admin_elastic_search_commands.go | 6 +- .../cli/admin_elastic_search_commands_test.go | 383 ++++++++++++++++++ 2 files changed, 387 insertions(+), 2 deletions(-) diff --git a/tools/cli/admin_elastic_search_commands.go b/tools/cli/admin_elastic_search_commands.go index eb823a505b3..3fad409ca28 100644 --- a/tools/cli/admin_elastic_search_commands.go +++ b/tools/cli/admin_elastic_search_commands.go @@ -340,6 +340,8 @@ func toTimeStr(s interface{}) string { // GenerateReport generate report for an aggregation query to ES func GenerateReport(c *cli.Context) error { + output := getDeps(c).Output() + // use url command argument to create client index, err := getRequiredOption(c, FlagIndex) if err != nil { @@ -380,7 +382,7 @@ func GenerateReport(c *cli.Context) error { } // Show result to terminal - table := tablewriter.NewWriter(os.Stdout) + table := tablewriter.NewWriter(output) var headers []string var groupby, bucket map[string]interface{} var buckets []interface{} @@ -390,7 +392,7 @@ func GenerateReport(c *cli.Context) error { } buckets = groupby["buckets"].([]interface{}) if len(buckets) == 0 { - fmt.Println("no matching bucket") + output.Write([]byte("no matching bucket")) return nil } diff --git a/tools/cli/admin_elastic_search_commands_test.go b/tools/cli/admin_elastic_search_commands_test.go index b9e4579debf..71ff9fb25a0 100644 --- a/tools/cli/admin_elastic_search_commands_test.go +++ b/tools/cli/admin_elastic_search_commands_test.go @@ -30,6 +30,7 @@ import ( "net/http/httptest" "os" "regexp" + "strings" "testing" "time" @@ -700,3 +701,385 @@ func TestGenerateESDoc(t *testing.T) { }) } } + +func TestGenerateReport(t *testing.T) { + tests := []struct { + name string + handler http.HandlerFunc + setupContext func(app *cli.App) *cli.Context + setupMocks func(mockClientFactory *MockClientFactory, esClient *elastic.Client) + expectedOutput string + expectedError string + }{ + { + name: "SuccessCSVReportWithExtraKey", + handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a successful response with extra key not in primaryColsMap + expectedPath := "/test-index/_search" + if r.URL.Path == expectedPath && r.Method == "POST" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{ + "aggregations": { + "groupby": { + "buckets": [ + { + "key": { + "group_DomainID": "domain1", + "group_CustomKey": "custom-value" + }, + "Attr_CustomDatetimeField": { + "value_as_string": "2023-10-01T12:34:56.789Z" + } + } + ] + } + } + }`)) + } else { + w.WriteHeader(http.StatusNotFound) + } + }), + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Define and set flags + set.String(FlagIndex, "", "Index flag") + set.String(FlagListQuery, "", "List query flag") + set.String(FlagOutputFormat, "", "Output format flag") + set.String(FlagOutputFilename, "", "Output file flag") + // Set the actual values + _ = set.Set(FlagIndex, "test-index") + _ = set.Set(FlagListQuery, "SELECT * FROM logs") + _ = set.Set(FlagOutputFormat, "csv") + _ = set.Set(FlagOutputFilename, "test-report.csv") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { + mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) + }, + expectedOutput: `+-------------+--------------+--------------------------+ +| DOMAINID(*) | CUSTOMKEY(*) | ATTR CUSTOMDATETIMEFIELD | ++-------------+--------------+--------------------------+ +| domain1 | custom-value | 2023-10-01T12:34:56.789Z | ++-------------+--------------+--------------------------+ +`, + expectedError: "", + }, + // can't put all keys all together because keys generated in reports are in a random order, thus will fail tests + { + name: "SuccessCSVReportWithOtherExtraKey", + handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a successful response with extra key not in primaryColsMap + expectedPath := "/test-index/_search" + if r.URL.Path == expectedPath && r.Method == "POST" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{ + "aggregations": { + "groupby": { + "buckets": [ + { + "key": { + "group_DomainID": "domain1", + "group_CustomKey": "custom-value" + }, + "CompletionTime": { + "value": 1696203296789 + } + } + ] + } + } + }`)) + } else { + w.WriteHeader(http.StatusNotFound) + } + }), + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Define and set flags + set.String(FlagIndex, "", "Index flag") + set.String(FlagListQuery, "", "List query flag") + set.String(FlagOutputFormat, "", "Output format flag") + set.String(FlagOutputFilename, "", "Output file flag") + // Set the actual values + _ = set.Set(FlagIndex, "test-index") + _ = set.Set(FlagListQuery, "SELECT * FROM logs") + _ = set.Set(FlagOutputFormat, "csv") + _ = set.Set(FlagOutputFilename, "test-report.csv") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { + mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) + }, + expectedOutput: `+-------------+--------------+---------------------------+ +| DOMAINID(*) | CUSTOMKEY(*) | COMPLETIONTIME | ++-------------+--------------+---------------------------+ +| domain1 | custom-value | 1969-12-31T16:28:16-08:00 | ++-------------+--------------+---------------------------+ +`, + expectedError: "", + }, + { + name: "SuccessHTMLReportWithExtraKey", + handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a successful response with extra key not in primaryColsMap + expectedPath := "/test-index/_search" + if r.URL.Path == expectedPath && r.Method == "POST" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{ + "aggregations": { + "groupby": { + "buckets": [ + { + "key": { + "group_DomainID": "domain1", + "group_CustomKey": "custom-value" + }, + "doc_count": 10 + } + ] + } + } + }`)) + } else { + w.WriteHeader(http.StatusNotFound) + } + }), + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Define and set flags + set.String(FlagIndex, "", "Index flag") + set.String(FlagListQuery, "", "List query flag") + set.String(FlagOutputFormat, "", "Output format flag") + set.String(FlagOutputFilename, "", "Output file flag") + // Set the actual values + _ = set.Set(FlagIndex, "test-index") + _ = set.Set(FlagListQuery, "SELECT * FROM logs") + _ = set.Set(FlagOutputFormat, "html") + _ = set.Set(FlagOutputFilename, "test-report.csv") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { + mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) + }, + expectedOutput: `+-------------+--------------+-------+ +| DOMAINID(*) | CUSTOMKEY(*) | COUNT | ++-------------+--------------+-------+ +| domain1 | custom-value | 10 | ++-------------+--------------+-------+ +`, + expectedError: "", + }, + { + name: "EmptyBucket", + handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a successful response with extra key not in primaryColsMap + expectedPath := "/test-index/_search" + if r.URL.Path == expectedPath && r.Method == "POST" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{ + "aggregations": { + "groupby": { + "buckets": [] + } + } + }`)) + } else { + w.WriteHeader(http.StatusNotFound) + } + }), + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Define and set flags + set.String(FlagIndex, "", "Index flag") + set.String(FlagListQuery, "", "List query flag") + set.String(FlagOutputFormat, "", "Output format flag") + set.String(FlagOutputFilename, "", "Output file flag") + // Set the actual values + _ = set.Set(FlagIndex, "test-index") + _ = set.Set(FlagListQuery, "SELECT * FROM logs") + _ = set.Set(FlagOutputFormat, "html") + _ = set.Set(FlagOutputFilename, "test-report.csv") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { + mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) + }, + expectedOutput: `no matching bucket`, + expectedError: "", + }, + { + name: "UnsupportedReportFormat", + handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Elasticsearch request returns successful response + if r.URL.Path == "/test-index/_search" && r.Method == "POST" { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{ + "aggregations": { + "groupby": { + "buckets": [ + { + "key": {"group_DomainID": "domain1"}, + "doc_count": 10 + } + ] + } + } + }`)) + } else { + w.WriteHeader(http.StatusNotFound) + } + }), + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Define and set flags + set.String(FlagIndex, "", "Index flag") + set.String(FlagListQuery, "", "List query flag") + set.String(FlagOutputFormat, "", "Output format flag") + set.String(FlagOutputFilename, "", "Output file flag") + // Set the actual values + _ = set.Set(FlagIndex, "test-index") + _ = set.Set(FlagListQuery, "SELECT * FROM logs") + _ = set.Set(FlagOutputFormat, "unsupported-format") + _ = set.Set(FlagOutputFilename, "test-report.unsupported") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { + mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) + }, + expectedOutput: "", + expectedError: "Report format unsupported-format not supported.", + }, + { + name: "ElasticsearchQueryError", + handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate an error response from Elasticsearch + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{"error": "query failed"}`)) + }), + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Define and set flags + set.String(FlagIndex, "", "Index flag") + set.String(FlagListQuery, "", "List query flag") + // Set the actual values + _ = set.Set(FlagIndex, "test-index") + _ = set.Set(FlagListQuery, "SELECT * FROM logs") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { + mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) + }, + expectedOutput: "", + expectedError: "Fail to talk with ES", + }, + { + name: "MissingRequiredFlagIndex", + handler: nil, // No handler needed since the error occurs before any Elasticsearch interaction + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Only setting FlagListQuery, but missing FlagIndex to trigger the error + set.String(FlagListQuery, "", "List query flag") + _ = set.Set(FlagListQuery, "SELECT * FROM logs") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) {}, + expectedOutput: "", + expectedError: "Required flag not found: ", + }, + { + name: "MissingRequiredFlagListQuery", + handler: nil, // No handler needed since the error occurs before any Elasticsearch interaction + setupContext: func(app *cli.App) *cli.Context { + set := flag.NewFlagSet("test", 0) + // Only setting FlagIndex, but missing FlagListQuery to trigger the error + set.String(FlagIndex, "", "Index flag") + _ = set.Set(FlagIndex, "test-index") + return cli.NewContext(app, set, nil) + }, + setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) {}, + expectedOutput: "", + expectedError: "Required flag not found: ", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create mock Elasticsearch client and server + esClient, testServer := getMockClient(t, tt.handler) + defer testServer.Close() + + // Initialize mock controller + mockCtrl := gomock.NewController(t) + + // Create mock client factory + mockClientFactory := NewMockClientFactory(mockCtrl) + + // Create test IO handler to capture output + ioHandler := &testIOHandler{} + + // Set up the CLI app + app := NewCliApp(mockClientFactory, WithIOHandler(ioHandler)) + + // Expect ElasticSearchClient to return the mock client created by getMockClient + tt.setupMocks(mockClientFactory, esClient) + + // Set up the context for the specific test case + c := tt.setupContext(app) + + // Call GenerateReport + err := GenerateReport(c) + + // Validate results + if tt.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.expectedError) + } else { + assert.NoError(t, err) + // Validate the output captured by testIOHandler + assert.Equal(t, tt.expectedOutput, ioHandler.outputBytes.String()) + } + }) + } +} + +func TestGenerateHTMLReport_RowSpanLogic(t *testing.T) { + // Prepare headers and tableData to trigger the rowspan logic + headers := []string{"Domain", "Status", "Count"} + tableData := [][]string{ + {"domain1", "open", "10"}, + {"domain1", "open", "15"}, + {"domain2", "closed", "20"}, + {"domain2", "closed", "25"}, + } + + // Prepare temp file to write HTML report + tempFile, err := os.CreateTemp("", "test_report_*.html") + assert.NoError(t, err) + defer os.Remove(tempFile.Name()) // Clean up + + // Call generateHTMLReport with numBuckKeys to control the column collapsing + err = generateHTMLReport(tempFile.Name(), 2, false, headers, tableData) + assert.NoError(t, err) + + // Read and validate the generated HTML content + content, err := os.ReadFile(tempFile.Name()) + assert.NoError(t, err) + + // Remove all newlines and spaces to simplify comparison + actualContent := string(content) + actualContent = removeWhitespace(actualContent) + + // Expected HTML content (also simplified by removing whitespace) + expectedHTMLStructure := removeWhitespace(` + domain1open10 + + open15`) + + // Validate the rowspan logic was applied correctly + assert.Contains(t, actualContent, expectedHTMLStructure) +} + +// Helper function to remove all whitespace from a string +func removeWhitespace(input string) string { + return strings.ReplaceAll(strings.ReplaceAll(input, "\n", ""), "\t", "") +} From 93a480ddfcb0a95f105f6b15f689f0cca01a5c7a Mon Sep 17 00:00:00 2001 From: Bowen Xiao Date: Sun, 27 Oct 2024 12:03:39 -0700 Subject: [PATCH 2/4] Use nano second to avoid error --- tools/cli/admin_elastic_search_commands_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/cli/admin_elastic_search_commands_test.go b/tools/cli/admin_elastic_search_commands_test.go index 71ff9fb25a0..f2e6915912e 100644 --- a/tools/cli/admin_elastic_search_commands_test.go +++ b/tools/cli/admin_elastic_search_commands_test.go @@ -782,7 +782,7 @@ func TestGenerateReport(t *testing.T) { "group_CustomKey": "custom-value" }, "CompletionTime": { - "value": 1696203296789 + "value": 1727962032967890000 } } ] @@ -813,7 +813,7 @@ func TestGenerateReport(t *testing.T) { expectedOutput: `+-------------+--------------+---------------------------+ | DOMAINID(*) | CUSTOMKEY(*) | COMPLETIONTIME | +-------------+--------------+---------------------------+ -| domain1 | custom-value | 1969-12-31T16:28:16-08:00 | +| domain1 | custom-value | 2024-10-03T06:27:12-07:00 | +-------------+--------------+---------------------------+ `, expectedError: "", From 53445ff8086a69902581019d292d44fdcd3e7919 Mon Sep 17 00:00:00 2001 From: Bowen Xiao Date: Sun, 27 Oct 2024 12:34:50 -0700 Subject: [PATCH 3/4] change a test case to pass build --- tools/cli/admin_elastic_search_commands_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/cli/admin_elastic_search_commands_test.go b/tools/cli/admin_elastic_search_commands_test.go index f2e6915912e..0af20acea75 100644 --- a/tools/cli/admin_elastic_search_commands_test.go +++ b/tools/cli/admin_elastic_search_commands_test.go @@ -781,8 +781,8 @@ func TestGenerateReport(t *testing.T) { "group_DomainID": "domain1", "group_CustomKey": "custom-value" }, - "CompletionTime": { - "value": 1727962032967890000 + "Attr_CustomStringField": { + "value": "test-string" } } ] @@ -810,11 +810,11 @@ func TestGenerateReport(t *testing.T) { setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) }, - expectedOutput: `+-------------+--------------+---------------------------+ -| DOMAINID(*) | CUSTOMKEY(*) | COMPLETIONTIME | -+-------------+--------------+---------------------------+ -| domain1 | custom-value | 2024-10-03T06:27:12-07:00 | -+-------------+--------------+---------------------------+ + expectedOutput: `+-------------+--------------+------------------------+ +| DOMAINID(*) | CUSTOMKEY(*) | ATTR CUSTOMSTRINGFIELD | ++-------------+--------------+------------------------+ +| domain1 | custom-value | test-string | ++-------------+--------------+------------------------+ `, expectedError: "", }, From 746085fb6c50ec9e11bdbf54135182aa8dc0734e Mon Sep 17 00:00:00 2001 From: Bowen Xiao Date: Mon, 28 Oct 2024 09:45:44 -0700 Subject: [PATCH 4/4] Update test case --- tools/cli/admin_elastic_search_commands.go | 2 +- tools/cli/admin_elastic_search_commands_test.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/cli/admin_elastic_search_commands.go b/tools/cli/admin_elastic_search_commands.go index 3fad409ca28..9ecf382ecba 100644 --- a/tools/cli/admin_elastic_search_commands.go +++ b/tools/cli/admin_elastic_search_commands.go @@ -392,7 +392,7 @@ func GenerateReport(c *cli.Context) error { } buckets = groupby["buckets"].([]interface{}) if len(buckets) == 0 { - output.Write([]byte("no matching bucket")) + output.Write([]byte("no matching bucket\n")) return nil } diff --git a/tools/cli/admin_elastic_search_commands_test.go b/tools/cli/admin_elastic_search_commands_test.go index 0af20acea75..059af005708 100644 --- a/tools/cli/admin_elastic_search_commands_test.go +++ b/tools/cli/admin_elastic_search_commands_test.go @@ -904,8 +904,9 @@ func TestGenerateReport(t *testing.T) { setupMocks: func(mockClientFactory *MockClientFactory, esClient *elastic.Client) { mockClientFactory.EXPECT().ElasticSearchClient(gomock.Any()).Return(esClient, nil).Times(1) }, - expectedOutput: `no matching bucket`, - expectedError: "", + expectedOutput: `no matching bucket +`, + expectedError: "", }, { name: "UnsupportedReportFormat",