Changes to install and start logstash on ec2 instances

install-logstash.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+
+ echo "Starting logstash install ..."
+    sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
+    if [ -f ~/logstash.repo ];
+    then
+       echo "File logstash.repo exist."
+       sudo mv ~/logstash.repo /etc/yum.repos.d/
+       sudo chown root:root /etc/yum.repos.d/logstash.repo
+       sudo chmod 644 /etc/yum.repos.d/logstash.repo
+    else
+       echo "File logstash.repo does not exist"
+       echo "[logstash-6.x]" >> ~/logstash.repo
+       echo "name=Elastic repository for 6.x packages" >> ~/logstash.repo
+       echo "baseurl=https://artifacts.elastic.co/packages/6.x/yum" >> ~/logstash.repo
+       echo "gpgcheck=1" >> ~/logstash.repo
+       echo "gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch" >> ~/logstash.repo
+       echo "enabled=1" >> ~/logstash.repo
+       echo "autorefresh=1" >> ~/logstash.repo
+       echo "type=rpm-md" >> ~/logstash.repo
+
+       sudo mv ~/logstash.repo /etc/yum.repos.d/
+       sudo chown root:root /etc/yum.repos.d/logstash.repo
+       sudo chmod 644 /etc/yum.repos.d/logstash.repo
+    fi
+
+
+    sudo yum -y install logstash
+
+
+    if [ -f ~/logstash.conf ];
+    then
+       echo "File  logstash.conf exist."a
+       sudo mv ~/logstash.conf /etc/logstash/conf.d/
+       sudo chown root:root /etc/logstash/conf.d/logstash.conf
+       sudo chmod 644 /etc/logstash/conf.d/logstash.conf
+    else
+       echo "File logstash.conf does not exist"
+
+    fi
+
+    sudo nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf &
+
+ps -ef | grep logstash >> /tmp/logstash.ps
+
+ echo "Ended intall"
+ cat "Ended install" >> /tmp/logstash.logfile

jmeter-ec2.sh

@@ -718,10 +718,66 @@ function runsetup() {
       echo -n "done...."
     fi
 
+    # scp install-logstash.sh
+    if [ -r $LOCAL_HOME/install-logstash.sh ] ; then
+      echo -n "logstash-install.sh.."
+      for host in ${hosts[@]} ; do
+          (scp -q -C -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+                                        -i "$PEM_PATH/$PEM_FILE" -P $REMOTE_PORT \
+                                        $LOCAL_HOME/install-logstash.sh \
+                                        $USER@$host:$REMOTE_HOME) &
+      done
+      wait
+      echo -n "done...."
+    fi
+
+    # scp logstash.conf
+    if [ -r $LOCAL_HOME/logstash.conf ] ; then
+      echo -n "logstash.conf.."
+      for host in ${hosts[@]} ; do
+          (scp -q -C -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+                                        -i "$PEM_PATH/$PEM_FILE" -P $REMOTE_PORT \
+                                        $LOCAL_HOME/logstash.conf \
+                                        $USER@$host:$REMOTE_HOME) &
+      done
+      wait
+      echo -n "done...."
+    fi
+
+
+
     echo "all files uploaded"
     echo
   fi
 
+
+ # Install logstash
+  echo "starting install logstash:"
+  for host in ${hosts[@]} ; do
+    echo $host
+  done
+  for counter in ${!hosts[@]} ; do
+      ( ssh -nq -o StrictHostKeyChecking=no \
+      -p $REMOTE_PORT \
+      -i "$PEM_PATH/$PEM_FILE" $USER@${hosts[$counter]} \
+      $REMOTE_HOME/install-logstash.sh &>/dev/null
+      )
+  done
+
+
+ # Run logstash
+  echo "starting running logstash:"
+  for host in ${hosts[@]} ; do
+    echo $host
+  done
+  for counter in ${!hosts[@]} ; do
+      ( nohup ssh -nq -o StrictHostKeyChecking=no \
+      -p $REMOTE_PORT \
+      -i "$PEM_PATH/$PEM_FILE" $USER@${hosts[$counter]} \
+      $REMOTE_HOME/run-logstash.sh &>/dev/null &
+      )
+  done
+
   # Start JMeter
   echo "starting jmeter on:"
   for host in ${hosts[@]} ; do

logstash.conf

@@ -0,0 +1,48 @@
+input {
+file {  # point to jmeter result file created using simple csv writer
+        path => ["/home/ec2-user/data/resultfile.csv"]
+        start_position => "beginning"
+     }
+}
+
+filter {
+
+  if ([message] =~ "responseCode") {
+    drop { }
+  } else {
+    csv {
+        columns => ["timestamp", "elapsed", "label", "responseCode", "responseMessage", "threadName",
+                    "dataType", "success", "failureMessage", "bytes", "grpThreads", "allThreads", "URL", "Latency",
+                    "SampleCount", "ErrorCount", "Hostname"]
+    }
+    date {
+      match => [ "timestamp", "UNIX_MS" ]
+    }
+
+
+    mutate {
+      convert => [ "elapsed", "integer" ]
+      convert => [ "label", "string" ]
+      convert => [ "responseCode", "integer" ]
+      convert => [ "responseMessage", "string" ]
+      convert => [ "success", "boolean" ]
+      convert => [ "failureMessage", "string" ]
+      convert => [ "bytes", "integer" ]
+      convert => [ "URL", "string" ]
+      convert => [ "Latency", "integer" ]
+      convert => [ "SampleCount", "integer" ]
+      convert => [ "ErrorCount", "integer" ]
+      convert => [ "Hostname", "string" ]
+
+      remove_field => [ "threadName", "dataType", "grpThreads", "allThreads" ]
+    }
+  }
+}
+output {
+#stdout { codec => rubydebug }
+  elasticsearch {
+# point to Elastic search server configured to accept incoming traffic
+    hosts => ["111.22.33.44:9200"]
+    index => "jmeter-result-dash-%{+YYYY.MM.dd}"
+  }
+}

run-logstash.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+nohup sudo /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf > /dev/null 2>&1 &