Merge pull request #20 from marklogic-community/develop

1.1.1 release

LICENSE

@@ -1,4 +1,4 @@
-Copyright © 2019 MarkLogic Corporation. 
+Copyright © 2020 MarkLogic Corporation.
 
 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
 

NOTICE

@@ -1,6 +1,6 @@
 MarkLogic® Monitoring App for Splunk
 
-Copyright © 2019 MarkLogic Corporation.
+Copyright © 2020 MarkLogic Corporation.
 
 This project and its code and functionality is not representative of MarkLogic Server and is not supported by MarkLogic.
 

README.md

@@ -1,7 +1,7 @@
 # MarkLogic Monitoring for Splunk
 
-MarkLogic Monitoring for Splunk provides configurations and pre-built dashboards that deliver real-time visibility into Error, Access, and Audit log events 
- to monitor and analyze MarkLogic logs with Splunk.
+MarkLogic Monitoring for Splunk provides configurations and pre-built dashboards that deliver real-time visibility into Error, Access, Request, 
+and Audit log events to monitor and analyze MarkLogic logs with Splunk.
 
 ![Monitoring dashboard](src/main/resources/appserver/static/screenshot.png "Monitoring dashboard")
 
@@ -46,10 +46,11 @@ Below are example stanzas that can be applied to your *inputs.conf* in order to
 The MarkLogic Monitoring dashboard queries make use of *macros* in order to construct base queries that target 
 MarkLogic log events from the MarkLogic sourcetypes for Error, Access, and Audit logs
 
-- *marklogic_index* - search criteria limited to `(index=main OR index=marklogic)`
+- *marklogic_index* - search criteria limited to `(index=default OR index=main OR index=marklogic)`
 - *marklogic_access* - restricts searches to the MarkLogic *_AccessLog.txt events
 - *marklogic_audit* - restricts searches to the MarkLogic AuditLog.txt events
 - *marklogic_error* - restricts searches to the *ErrorLog.txt events
+- *marklogic_request* - restricts searches to the *_RequestrLog.txt events
 
 You can change these by modifying the *marklogic*, *marklogic_index*, *marklogic_error*, *marklogic_access*, 
 and *marklogic_audit* macros under *Settings > Advanced search > Search macros*.

build.gradle

@@ -1,20 +1,24 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 import com.marklogic.client.DatabaseClient
-import sun.misc.BASE64Encoder
 
 plugins {
     id 'base'
     id 'net.saliman.properties' version '1.5.1'
-    id 'com.marklogic.ml-gradle' version '3.16.2'
+    id 'com.marklogic.ml-gradle' version '4.5.3'
 }
 
 class GenerateLookups extends com.marklogic.gradle.task.MarkLogicTask {
+    @Input
     String modulesDir = 'src/main/ml-modules/'
+    @Input
     String featureQuery = new File("${modulesDir}get-marklogic-features.xqy").getText('UTF-8')
+    @Input
     String errorQuery = new File("${modulesDir}get-marklogic-error-codes.xqy").getText('UTF-8')
-
+    @Input
     String lookupsDir = 'src/main/resources/lookups/'
+    @Input
     String featureFile = 'marklogic_feature.csv'
+    @Input
     String errorFile = 'marklogic_error.csv'
 
     @TaskAction
@@ -104,9 +108,8 @@ def addAuthorizationToken(connection, token){
 
 def appInpsectLogin() {
     def connection = buildAppInspectConnection( 'https://api.splunk.com/2.0/rest/login/splunk', 'GET')
-    BASE64Encoder enc = new sun.misc.BASE64Encoder();
     String userpassword = project.appInspectUsername + ':' + project.appInspectPassword;
-    String encodedAuthorization = enc.encode( userpassword.getBytes() );
+    String encodedAuthorization = new String(java.util.Base64.getMimeEncoder().encode( userpassword.getBytes() ), java.nio.charset.StandardCharsets.UTF_8);
     connection.setRequestProperty('Authorization', 'Basic ' + encodedAuthorization);
     new groovy.json.JsonSlurper().parse(connection.getInputStream())
 }

gradle.properties

@@ -1,7 +1,7 @@
 name=marklogic-monitoring-for-splunk
 group=com.marklogic
 # Splunk requires major.minor.release
-version=1.1.0
+version=1.1.1
 appName=marklogic
 
 splunk_home=/Applications/Splunk

gradle/wrapper/gradle-wrapper.properties

@@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-all.zip

src/main/ml-modules/get-marklogic-error-codes.xqy

@@ -18,9 +18,14 @@ string-join(
     (
       "error_code,feature,error_description,error_cause,error_response,error_url",
 
-      for $feature in tokenize($FEATURES, ",")
-      let $error-doc := xdmp:http-get("http://docs.marklogic.com/guide/messages/"||$feature||"-en?print=yes")
+      for $feature in distinct-values((tokenize($FEATURES, ","), "HEALTH", "JSEARCH"))
+      let $error-doc := xdmp:http-get("https://docs.marklogic.com/guide/messages/"||$feature||"-en?print=yes",
+          <options xmlns="xdmp:http">
+            <verify-cert>false</verify-cert>
+          </options>
+      )
       let $content := xdmp:unquote($error-doc[2])/html:html/html:body/*/*/html:div[@class="message"]
+      order by $feature
       return (
         for $message in $content
         let $code := $message/html:h3/html:a/string()

src/main/ml-modules/get-marklogic-features.xqy

@@ -14,7 +14,11 @@ declare function local:cell($val) {
 string-join(
   (
     "feature,feature_description",
-    let $feature-doc := xdmp:http-get("http://docs.marklogic.com/guide/messages/intro")
+    let $feature-doc := xdmp:http-get("https://docs.marklogic.com/guide/messages/intro",
+        <options xmlns="xdmp:http">
+          <verify-cert>false</verify-cert>
+        </options>
+    )
     let $rows := xdmp:unquote($feature-doc[2])/html:html/html:body/*//html:div[html:p/html:a/@id="id_pgfId-1059358"]/html:table[@class="api_generic_table"]/html:tr[html:td]
     return (
       for $row in $rows

src/main/resources/LICENSE

@@ -1,6 +1,6 @@
 MarkLogic® Monitoring App for Splunk
 
-Copyright © 2019 MarkLogic Corporation.
+Copyright © 2020 MarkLogic Corporation.
 
 This project and its code and functionality is not representative of MarkLogic Server and is not supported by MarkLogic.
 

src/main/resources/README.md

@@ -1,12 +1,12 @@
 # MarkLogic Monitoring for Splunk
 
-MarkLogic Monitoring for Splunk provides configurations and pre-built dashboards that deliver real-time visibility into Error, Access, and Audit log events 
- to monitor and analyze MarkLogic logs with Splunk.
+MarkLogic Monitoring for Splunk provides configurations and pre-built dashboards that deliver real-time visibility into 
+Error, Access, Request, and Audit log events to monitor and analyze MarkLogic logs with Splunk.
 
 The app includes:
 - Monitoring dashboard providing an overview of cluster activity and highlighting significant error events
 - Log file dashboards with filters for Error, Access, and Audit log fields
-- Pre-configured soucetypes and fields for MarkLogic Error, Access, and Audit log messages
+- Pre-configured soucetypes and fields for MarkLogic Error, Access, Request, and Audit log messages
 - Lookups configured with MarkLogic [features, error codes](https://docs.marklogic.com/guide/messages/intro), and [log levels](https://docs.marklogic.com/guide/admin/logfiles#id_37841)
 
 ## Configuration
@@ -41,13 +41,14 @@ Below are example stanzas that can be applied to your *inputs.conf* in order to
 The MarkLogic Monitoring dashboard queries make use of *macros* in order to construct base queries that target 
 MarkLogic log events from the MarkLogic sourcetypes for Error, Access, and Audit logs
 
-- *marklogic_index* - search criteria limited to `(index=main OR index=marklogic)`
+- *marklogic_index* - search criteria limited to `(index=default OR index=main OR index=marklogic)`
 - *marklogic_access* - restricts searches to the MarkLogic *_AccessLog.txt events
 - *marklogic_audit* - restricts searches to the MarkLogic AuditLog.txt events
 - *marklogic_error* - restricts searches to the *ErrorLog.txt events
+- *marklogic_request* - restricts searches to the *_RequestLog.txt events
 
 You can change these by modifying the *marklogic*, *marklogic_index*, *marklogic_error*, *marklogic_access*, 
-and *marklogic_audit* macros under *Settings > Advanced search > Search macros*.
+*marklogic_request*, and *marklogic_audit* macros under *Settings > Advanced search > Search macros*.
 
 ## Getting Help
 Submit issues or feature requests at https://github.com/marklogic-community/marklogic-monitoring-for-splunk/issues 

src/main/resources/default/app.conf

@@ -1,5 +1,6 @@
 [install]
 is_configured = 0
+build = @VERSION@
 
 [ui]
 is_visible = 1

src/main/resources/default/data/ui/views/access_logs.xml

@@ -1,4 +1,4 @@
-<form>
+<form version="1.1">
   <label>Access Logs</label>
   <fieldset submitButton="false" autoRun="true">
     <input type="time" token="date-range">

src/main/resources/default/data/ui/views/audit_logs.xml

@@ -1,4 +1,4 @@
-<form theme="light">
+<form theme="light" version="1.1">
   <label>Audit Logs</label>
   <fieldset submitButton="false" autoRun="true">
     <input type="time" token="date-range" searchWhenChanged="true">

src/main/resources/default/data/ui/views/backups.xml

@@ -1,11 +1,11 @@
-<form>
+<form version="1.1">
   <label>Backups</label>
   <fieldset submitButton="false" autoRun="true">
     <input type="time" token="date-range" searchWhenChanged="true">
       <label></label>
       <default>
-        <earliestTime>-30d</earliestTime>
-        <latestTime></latestTime>
+        <earliest>-30d</earliest>
+        <latest></latest>
       </default>
     </input>
   </fieldset>

src/main/resources/default/data/ui/views/error_logs.xml

@@ -1,4 +1,4 @@
-<form>
+<form version="1.1">
     <label>Error Logs</label>
     <fieldset submitButton="false" autoRun="true">
         <input type="time" token="date-range" searchWhenChanged="true">

src/main/resources/default/data/ui/views/memory.xml

@@ -1,4 +1,4 @@
-<form>
+<form version="1.1">
     <label>Memory</label>
     <fieldset submitButton="false" autoRun="true">
         <input type

src/main/resources/default/data/ui/views/monitoring.xml

@@ -1,4 +1,4 @@
-<form>
+<form version="1.1">
     <label>Monitoring</label>
     <fieldset submitButton="false" autoRun="true">
         <input type="time" token="date-range" searchWhenChanged="true">

src/main/resources/default/inputs.conf

@@ -9,6 +9,11 @@
 #index = marklogic
 #sourcetype = access_combined
 
+#[monitor:///var/opt/MarkLogic/.../*_RequestLog.txt]
+#disabled = true
+#index = marklogic
+#sourcetype = marklogic_request
+
 #[monitor:///var/opt/MarkLogic/.../AuditLog.txt]
 #disabled = true
 #index = marklogic
@@ -25,6 +30,11 @@
 #index = marklogic
 #sourcetype = access_combined
 
+#[monitor://C:\Program Files\MarkLogic\Data\Logs\*_RequestLog.txt]
+#disabled = true
+#index = marklogic
+#sourcetype = marklogic_request
+
 #[monitor://C:\Program Files\MarkLogic\Data\Logs\AuditLog.txt]
 #disabled = true
 #index = marklogic

src/main/resources/default/macros.conf

@@ -16,4 +16,8 @@ iseval = 0
 
 [marklogic_error]
 definition = `marklogic_index` sourcetype=marklogic_error
+iseval = 0
+
+[marklogic_request]
+definition = `marklogic_index` sourcetype=marklogic_request
 iseval = 0

src/main/resources/default/props.conf

@@ -1,5 +1,4 @@
 [marklogic_audit]
-DATETIME_CONFIG =
 NO_BINARY_CHECK = true
 PREFIX_SOURCETYPE = false
 SHOULD_LINEMERGE = false
@@ -14,7 +13,6 @@ EXTRACT-roles = roles=(?P<roles>[^;]+);
 EVAL-roles = split(roles,",")
 
 [marklogic_error]
-DATETIME_CONFIG =
 NO_BINARY_CHECK = true
 PREFIX_SOURCETYPE = false
 SHOULD_LINEMERGE = false
@@ -64,4 +62,8 @@ priority = 99
 
 [source::...\*ErrorLog.txt]
 sourcetype = marklogic_error
+priority = 99
+
+[source::...\*_RequestLog.txt]
+sourcetype = marklogic_request
 priority = 99