summaryrefslogtreecommitdiffstats
path: root/scripts/elasticsearch
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/elasticsearch')
-rw-r--r--scripts/elasticsearch/auditdataConfigSettings.json82
-rw-r--r--scripts/elasticsearch/commands.txt205
-rw-r--r--scripts/elasticsearch/prepareElasticSearchBulkImport.pl41
-rw-r--r--scripts/elasticsearch/sampleAuditLog5.csv12
4 files changed, 340 insertions, 0 deletions
diff --git a/scripts/elasticsearch/auditdataConfigSettings.json b/scripts/elasticsearch/auditdataConfigSettings.json
new file mode 100644
index 0000000..912295b
--- /dev/null
+++ b/scripts/elasticsearch/auditdataConfigSettings.json
@@ -0,0 +1,82 @@
+{
+ "mappings": {
+ "default": {
+ "properties": {
+ "entityId": {
+ "type": "nested"
+ },
+ "entityLink": {
+ "type": "string"
+ },
+ "entityType": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "resourceVersion": {
+ "type": "string"
+ },
+ "validationId": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "validationTimestamp": {
+ "type": "date",
+ "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy||yyyyMMdd'T'HHmmssZ"
+ },
+ "violations": {
+ "type": "nested",
+ "properties": {
+ "category": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "errorMessage": {
+ "type": "string"
+ },
+ "modelName": {
+ "type": "string"
+ },
+ "severity": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "validationRule": {
+ "type": "string"
+ },
+ "violationDetails": {
+ "type": "nested",
+ "properties": {
+ "MISSING_REL": {
+ "type": "string"
+ },
+ "entityId": {
+ "type": "nested"
+ },
+ "entityType": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "modelName": {
+ "type": "string"
+ }
+ }
+ },
+ "violationId": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "violationTimestamp": {
+ "type": "date",
+ "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy||yyyyMMdd'T'HHmmssZ"
+ },
+ "violationType": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+ }
+
+ }
+}
diff --git a/scripts/elasticsearch/commands.txt b/scripts/elasticsearch/commands.txt
new file mode 100644
index 0000000..2e2284a
--- /dev/null
+++ b/scripts/elasticsearch/commands.txt
@@ -0,0 +1,205 @@
+
+ElasticSearch Configuration:
+============================
+
+1. Verify that your ElasticSearch instance configuation contains the following cors parameters in the elasticsearch.yaml file. The CORS
+ workaround has proven to work for Firefox, Chrome, and Opera.
+
+http.cors.enabled: true
+http.cors.allow-origin: "/.*/"
+http.cors.allow-headers: ["X-Requested-With", "Content-Type", "Content-Length"]
+http.cors.allow-credentials: true
+
+2. Start up Elastic Search by running running the elasticsearch.bat in the elasticsearch 2.3.1 bin folder.
+
+If you want to do a cleanup before running these instructions, then you can execute this
+optional command which will destroy the index settings and data.
+
+curl -XDELETE "http://localhost:9200/auditdata?pretty"
+
+Expected Result:
+{
+ "acknowledged" : true
+}
+
+
+ElasticSearch Index Setup and Bulk Load Instructions:
+=====================================================
+
+1. Configure Elastic Search Index
+
+curl -XPUT localhost:9200/auditdata?pretty --data-binary @auditdataConfigSettings.json
+
+Expected Result:
+{
+ "acknowledged" : true
+}
+
+2. Prepare elastic search bulk import:
+
+prepareElasticSearchBulkImport.pl sampleAuditLog5.csv auditBulkLoad.json
+
+curl -XPUT localhost:9200/_bulk?pretty --data-binary @auditBulkLoad.json
+
+At the top of the output verify if there any import errors by looking at the errors field.
+
+Expected Result:
+{
+ "took" : 103,
+ "errors" : false, <-------- this field is important. if true you need to look at the output, otherwise you can ignore it
+ "items" : [ {
+ "create" : {
+ "_index" : "auditdata",
+ "_type" : "everything",
+ "_id" : "AVXN0g6Ve6sNoEtMKGxy",
+ "_version" : 1,
+ "_shards" : {
+ "total" : 2,
+ "successful" : 1,
+ "failed" : 0
+ },
+ "status" : 201
+ }
+
+
+
+3. Verify that auditdata index contains data
+
+curl -XGET http://localhost:9200/_cat/indices?v
+
+Expected Result:
+
+health status index pri rep docs.count docs.deleted store.size pri.store.size
+yellow open auditdata 5 1 250 0 85.2kb 85.2kb
+
+4. Verify configuration of elastic search index parameter settings:
+
+curl -XGET http://localhost:9200/auditdata?pretty
+
+Expected Result:
+{
+ "auditdata" : {
+ "aliases" : { },
+ "mappings" : {
+ "everything" : {
+ "properties" : {
+ "date" : {
+ "type" : "date",
+ "format" : "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy"
+ },
+ "entityKey" : {
+ "type" : "string"
+ },
+ "entityType" : {
+ "type" : "string"
+ },
+ "message" : {
+ "type" : "string"
+ },
+ "severity" : {
+ "type" : "string"
+ },
+ "status" : {
+ "type" : "string"
+ }
+ }
+ },
+ "auditdata" : {
+ "properties" : {
+ "date" : {
+ "type" : "date",
+ "format" : "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy"
+ },
+ "entityKey" : {
+ "type" : "string"
+ },
+ "entityType" : {
+ "type" : "string"
+ },
+ "message" : {
+ "type" : "string"
+ },
+ "severity" : {
+ "type" : "string"
+ },
+ "status" : {
+ "type" : "string"
+ }
+ }
+ }
+ },
+ "settings" : {
+ "index" : {
+ "creation_date" : "1468250773569",
+ "number_of_shards" : "5",
+ "number_of_replicas" : "1",
+ "uuid" : "IgJe5PZyQmSfCLzuxm3Ulw",
+ "version" : {
+ "created" : "2030199"
+ }
+ }
+ },
+ "warmers" : { }
+ }
+}
+
+5. Test that you can retrieve data:
+
+curl -XGET "http://localhost:9200/auditdata/_search/?size=3&pretty"
+
+Expected Result:
+{
+ "took" : 8,
+ "timed_out" : false,
+ "_shards" : {
+ "total" : 5,
+ "successful" : 5,
+ "failed" : 0
+ },
+ "hits" : {
+ "total" : 250,
+ "max_score" : 1.0,
+ "hits" : [ {
+ "_index" : "auditdata",
+ "_type" : "everything",
+ "_id" : "AVXakQNNe6sNoEtMKG1y",
+ "_score" : 1.0,
+ "_source" : {
+ "date" : "May 26 2016 15:24:13",
+ "severity" : "CRITICAL",
+ "entityType" : "vpls-pe",
+ "entityKey" : "sfcca303vr1",
+ "status" : "prov-status=[ACTIVE]",
+ "message" : "Invalid prov-status value. Must have a value not equal to ACTIVE/active."
+ }
+ },{
+ "_index" : "auditdata",
+ "_type" : "everything",
+ "_id" : "AVXakQNNe6sNoEtMKG13",
+ "_score" : 1.0,
+ "_source" : {
+ "date" : "May 26 2016 15:24:13",
+ "severity" : "CRITICAL",
+ "entityType" : "vpe",
+ "entityKey" : "VPESAT-ashah401me6",
+ "status" : "prov-status=[NULL]",
+ "message" : "Invalid prov-status value. Must have a value not equal to ACTIVE/active."
+ }
+ },{
+ "_index" : "auditdata",
+ "_type" : "everything",
+ "_id" : "AVXakQNNe6sNoEtMKG17",
+ "_score" : 1.0,
+ "_source" : {
+ "date" : "May 26 2016 15:24:13",
+ "severity" : "CRITICAL",
+ "entityType" : "vpe",
+ "entityKey" : "VPESAT-eshah401me6",
+ "status" : "prov-status=[]",
+ "message" : "Invalid prov-status value. Must have a value not equal to ACTIVE/active."
+ }
+ } ]
+ }
+}
+
+
diff --git a/scripts/elasticsearch/prepareElasticSearchBulkImport.pl b/scripts/elasticsearch/prepareElasticSearchBulkImport.pl
new file mode 100644
index 0000000..e3fc690
--- /dev/null
+++ b/scripts/elasticsearch/prepareElasticSearchBulkImport.pl
@@ -0,0 +1,41 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+my $filename = $ARGV[0];
+my $outputfile= $ARGV[1];
+
+open my $fh_input, '<', $filename or die "Cannot open $filename: $!";
+open my $fh_output, '>', $outputfile or die "Cannot open $outputfile: $!";
+
+while ( my $line = <$fh_input> ) {
+ chomp ($line);
+
+ if ( $line =~ /(.*)(\".*\")(.*)/ ) {
+
+ # we have seen examples of the status field containing quoted comma-delimited
+ # strings which is messing up parsing of the record data which is supposed to be
+ # comma-separated at the field level. This little block converts sections of
+ # this type of data into a single-quoted-string with a semi-colon delimiter instead.
+
+ my $beforeBadStr = $1;
+ my $badStr = $2;
+ my $afterBadStr = $3;
+
+ $badStr =~ s/,/;/g;
+ $badStr =~ s/"/'/g;
+
+ $line = $beforeBadStr . $badStr . $afterBadStr ;
+
+ }
+
+ my @row = split(",", $line);
+ print $fh_output "{\"index\":{\"_index\":\"auditdata\",\"_type\":\"default\"}\n";
+ print $fh_output "{\"entityType\": \"$row[0]\", \"errorMessage\": \"$row[1]\", \"violations\": [{ \"violationTimestamp\": \"$row[2]\", \"severity\": \"$row[3]\", \"violationType\": \"$row[4]\", \"violationDetails\": { \"MISSING_REL\": \"$row[5]\", \"entityType\": \"$row[6]\", \"entityId\": { \"vdc-id\": \"$row[7]\" } }, \"category\": \"$row[8]\" }, { \"violationTimestamp\": \"$row[9]\", \"severity\": \"$row[10]\", \"violationType\": \"$row[11]\", \"violationDetails\": { \"MISSING_REL\": \"$row[12]\", \"entityType\": \"$row[13]\", \"entityId\": { \"vdc-id\": \"$row[14]\" } }, \"category\": \"$row[15]\" }]}\n";
+
+}
+
+close($fh_input);
+close($fh_output);
+
diff --git a/scripts/elasticsearch/sampleAuditLog5.csv b/scripts/elasticsearch/sampleAuditLog5.csv
new file mode 100644
index 0000000..deb4c6d
--- /dev/null
+++ b/scripts/elasticsearch/sampleAuditLog5.csv
@@ -0,0 +1,12 @@
+virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161219T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],newvce,vnf-id-team-auto-11,INVALID OBJ1,20161219T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-112,INVALID OBJ1
+newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161219T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],newvce,vnf-id-team-auto-11,INVALID OBJ2,20161219T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],newvce2,vnf-id-team-auto-112,INVALID OBJ2
+virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161219T091529Z,MINOR,Rule3,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-11,INVALID OBJ3,20161219T091529Z,MINOR,Rule3,prov-status=[ACTIVE],newvce2,vnf-id-team-auto-112,INVALID OBJ3
+newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161218T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],vserver,vnf-id-team-auto-11,INVALID OBJ4,20161218T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],vserver2,vnf-id-team-auto-112,INVALID OBJ4
+newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161218T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],vserver,vnf-id-team-auto-11,INVALID OBJ5,20161218T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],vserver2,vnf-id-team-auto-112,INVALID OBJ5
+virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161218T091529Z,MINOR,Rule3,prov-status=[ACTIVE],vserver,vnf-id-team-auto-11,INVALID OBJ6,20161218T091529Z,MINOR,Rule3,prov-status=[ACTIVE],vserver2,vnf-id-team-auto-112,INVALID OBJ6
+newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161217T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ7,20161217T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ7
+newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-11,INVALID OBJ8,20161217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ8
+virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161217T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ9,20161217T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ9
+newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20160917T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ1,20160917T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ1
+newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20160217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-11,INVALID OBJ2,20160217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ2
+virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20151017T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ3,20151017T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ3