From a38f3d6bb17a478d08016e49d6c2a667ac483d4a Mon Sep 17 00:00:00 2001 From: Arul Date: Mon, 15 Jan 2018 11:04:25 -0500 Subject: Introduction of external URLs Introduce external URLs to get AAI UI to show a specific graph Change-Id: Ibc10dab32540f2c8347df1be535e48b88308b9ec Signed-off-by: Arul Issue-ID: AAI-626 --- scripts/elasticsearch/auditdataConfigSettings.json | 82 +++++++++ scripts/elasticsearch/commands.txt | 205 +++++++++++++++++++++ .../prepareElasticSearchBulkImport.pl | 41 +++++ scripts/elasticsearch/sampleAuditLog5.csv | 12 ++ 4 files changed, 340 insertions(+) create mode 100644 scripts/elasticsearch/auditdataConfigSettings.json create mode 100644 scripts/elasticsearch/commands.txt create mode 100644 scripts/elasticsearch/prepareElasticSearchBulkImport.pl create mode 100644 scripts/elasticsearch/sampleAuditLog5.csv (limited to 'scripts') diff --git a/scripts/elasticsearch/auditdataConfigSettings.json b/scripts/elasticsearch/auditdataConfigSettings.json new file mode 100644 index 0000000..912295b --- /dev/null +++ b/scripts/elasticsearch/auditdataConfigSettings.json @@ -0,0 +1,82 @@ +{ + "mappings": { + "default": { + "properties": { + "entityId": { + "type": "nested" + }, + "entityLink": { + "type": "string" + }, + "entityType": { + "type": "string", + "index": "not_analyzed" + }, + "resourceVersion": { + "type": "string" + }, + "validationId": { + "type": "string", + "index": "not_analyzed" + }, + "validationTimestamp": { + "type": "date", + "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy||yyyyMMdd'T'HHmmssZ" + }, + "violations": { + "type": "nested", + "properties": { + "category": { + "type": "string", + "index": "not_analyzed" + }, + "errorMessage": { + "type": "string" + }, + "modelName": { + "type": "string" + }, + "severity": { + "type": "string", + "index": "not_analyzed" + }, + "validationRule": { + "type": "string" + }, + "violationDetails": { + "type": "nested", + "properties": { + "MISSING_REL": { + "type": "string" + }, + "entityId": { + "type": "nested" + }, + "entityType": { + "type": "string", + "index": "not_analyzed" + }, + "modelName": { + "type": "string" + } + } + }, + "violationId": { + "type": "string", + "index": "not_analyzed" + }, + "violationTimestamp": { + "type": "date", + "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy||yyyyMMdd'T'HHmmssZ" + }, + "violationType": { + "type": "string", + "index": "not_analyzed" + } + } + } + } + } + + } +} diff --git a/scripts/elasticsearch/commands.txt b/scripts/elasticsearch/commands.txt new file mode 100644 index 0000000..2e2284a --- /dev/null +++ b/scripts/elasticsearch/commands.txt @@ -0,0 +1,205 @@ + +ElasticSearch Configuration: +============================ + +1. Verify that your ElasticSearch instance configuation contains the following cors parameters in the elasticsearch.yaml file. The CORS + workaround has proven to work for Firefox, Chrome, and Opera. + +http.cors.enabled: true +http.cors.allow-origin: "/.*/" +http.cors.allow-headers: ["X-Requested-With", "Content-Type", "Content-Length"] +http.cors.allow-credentials: true + +2. Start up Elastic Search by running running the elasticsearch.bat in the elasticsearch 2.3.1 bin folder. + +If you want to do a cleanup before running these instructions, then you can execute this +optional command which will destroy the index settings and data. + +curl -XDELETE "http://localhost:9200/auditdata?pretty" + +Expected Result: +{ + "acknowledged" : true +} + + +ElasticSearch Index Setup and Bulk Load Instructions: +===================================================== + +1. Configure Elastic Search Index + +curl -XPUT localhost:9200/auditdata?pretty --data-binary @auditdataConfigSettings.json + +Expected Result: +{ + "acknowledged" : true +} + +2. Prepare elastic search bulk import: + +prepareElasticSearchBulkImport.pl sampleAuditLog5.csv auditBulkLoad.json + +curl -XPUT localhost:9200/_bulk?pretty --data-binary @auditBulkLoad.json + +At the top of the output verify if there any import errors by looking at the errors field. + +Expected Result: +{ + "took" : 103, + "errors" : false, <-------- this field is important. if true you need to look at the output, otherwise you can ignore it + "items" : [ { + "create" : { + "_index" : "auditdata", + "_type" : "everything", + "_id" : "AVXN0g6Ve6sNoEtMKGxy", + "_version" : 1, + "_shards" : { + "total" : 2, + "successful" : 1, + "failed" : 0 + }, + "status" : 201 + } + + + +3. Verify that auditdata index contains data + +curl -XGET http://localhost:9200/_cat/indices?v + +Expected Result: + +health status index pri rep docs.count docs.deleted store.size pri.store.size +yellow open auditdata 5 1 250 0 85.2kb 85.2kb + +4. Verify configuration of elastic search index parameter settings: + +curl -XGET http://localhost:9200/auditdata?pretty + +Expected Result: +{ + "auditdata" : { + "aliases" : { }, + "mappings" : { + "everything" : { + "properties" : { + "date" : { + "type" : "date", + "format" : "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy" + }, + "entityKey" : { + "type" : "string" + }, + "entityType" : { + "type" : "string" + }, + "message" : { + "type" : "string" + }, + "severity" : { + "type" : "string" + }, + "status" : { + "type" : "string" + } + } + }, + "auditdata" : { + "properties" : { + "date" : { + "type" : "date", + "format" : "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM/dd/yyyy" + }, + "entityKey" : { + "type" : "string" + }, + "entityType" : { + "type" : "string" + }, + "message" : { + "type" : "string" + }, + "severity" : { + "type" : "string" + }, + "status" : { + "type" : "string" + } + } + } + }, + "settings" : { + "index" : { + "creation_date" : "1468250773569", + "number_of_shards" : "5", + "number_of_replicas" : "1", + "uuid" : "IgJe5PZyQmSfCLzuxm3Ulw", + "version" : { + "created" : "2030199" + } + } + }, + "warmers" : { } + } +} + +5. Test that you can retrieve data: + +curl -XGET "http://localhost:9200/auditdata/_search/?size=3&pretty" + +Expected Result: +{ + "took" : 8, + "timed_out" : false, + "_shards" : { + "total" : 5, + "successful" : 5, + "failed" : 0 + }, + "hits" : { + "total" : 250, + "max_score" : 1.0, + "hits" : [ { + "_index" : "auditdata", + "_type" : "everything", + "_id" : "AVXakQNNe6sNoEtMKG1y", + "_score" : 1.0, + "_source" : { + "date" : "May 26 2016 15:24:13", + "severity" : "CRITICAL", + "entityType" : "vpls-pe", + "entityKey" : "sfcca303vr1", + "status" : "prov-status=[ACTIVE]", + "message" : "Invalid prov-status value. Must have a value not equal to ACTIVE/active." + } + },{ + "_index" : "auditdata", + "_type" : "everything", + "_id" : "AVXakQNNe6sNoEtMKG13", + "_score" : 1.0, + "_source" : { + "date" : "May 26 2016 15:24:13", + "severity" : "CRITICAL", + "entityType" : "vpe", + "entityKey" : "VPESAT-ashah401me6", + "status" : "prov-status=[NULL]", + "message" : "Invalid prov-status value. Must have a value not equal to ACTIVE/active." + } + },{ + "_index" : "auditdata", + "_type" : "everything", + "_id" : "AVXakQNNe6sNoEtMKG17", + "_score" : 1.0, + "_source" : { + "date" : "May 26 2016 15:24:13", + "severity" : "CRITICAL", + "entityType" : "vpe", + "entityKey" : "VPESAT-eshah401me6", + "status" : "prov-status=[]", + "message" : "Invalid prov-status value. Must have a value not equal to ACTIVE/active." + } + } ] + } +} + + diff --git a/scripts/elasticsearch/prepareElasticSearchBulkImport.pl b/scripts/elasticsearch/prepareElasticSearchBulkImport.pl new file mode 100644 index 0000000..e3fc690 --- /dev/null +++ b/scripts/elasticsearch/prepareElasticSearchBulkImport.pl @@ -0,0 +1,41 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +my $filename = $ARGV[0]; +my $outputfile= $ARGV[1]; + +open my $fh_input, '<', $filename or die "Cannot open $filename: $!"; +open my $fh_output, '>', $outputfile or die "Cannot open $outputfile: $!"; + +while ( my $line = <$fh_input> ) { + chomp ($line); + + if ( $line =~ /(.*)(\".*\")(.*)/ ) { + + # we have seen examples of the status field containing quoted comma-delimited + # strings which is messing up parsing of the record data which is supposed to be + # comma-separated at the field level. This little block converts sections of + # this type of data into a single-quoted-string with a semi-colon delimiter instead. + + my $beforeBadStr = $1; + my $badStr = $2; + my $afterBadStr = $3; + + $badStr =~ s/,/;/g; + $badStr =~ s/"/'/g; + + $line = $beforeBadStr . $badStr . $afterBadStr ; + + } + + my @row = split(",", $line); + print $fh_output "{\"index\":{\"_index\":\"auditdata\",\"_type\":\"default\"}\n"; + print $fh_output "{\"entityType\": \"$row[0]\", \"errorMessage\": \"$row[1]\", \"violations\": [{ \"violationTimestamp\": \"$row[2]\", \"severity\": \"$row[3]\", \"violationType\": \"$row[4]\", \"violationDetails\": { \"MISSING_REL\": \"$row[5]\", \"entityType\": \"$row[6]\", \"entityId\": { \"vdc-id\": \"$row[7]\" } }, \"category\": \"$row[8]\" }, { \"violationTimestamp\": \"$row[9]\", \"severity\": \"$row[10]\", \"violationType\": \"$row[11]\", \"violationDetails\": { \"MISSING_REL\": \"$row[12]\", \"entityType\": \"$row[13]\", \"entityId\": { \"vdc-id\": \"$row[14]\" } }, \"category\": \"$row[15]\" }]}\n"; + +} + +close($fh_input); +close($fh_output); + diff --git a/scripts/elasticsearch/sampleAuditLog5.csv b/scripts/elasticsearch/sampleAuditLog5.csv new file mode 100644 index 0000000..deb4c6d --- /dev/null +++ b/scripts/elasticsearch/sampleAuditLog5.csv @@ -0,0 +1,12 @@ +virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161219T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],newvce,vnf-id-team-auto-11,INVALID OBJ1,20161219T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-112,INVALID OBJ1 +newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161219T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],newvce,vnf-id-team-auto-11,INVALID OBJ2,20161219T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],newvce2,vnf-id-team-auto-112,INVALID OBJ2 +virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161219T091529Z,MINOR,Rule3,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-11,INVALID OBJ3,20161219T091529Z,MINOR,Rule3,prov-status=[ACTIVE],newvce2,vnf-id-team-auto-112,INVALID OBJ3 +newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161218T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],vserver,vnf-id-team-auto-11,INVALID OBJ4,20161218T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],vserver2,vnf-id-team-auto-112,INVALID OBJ4 +newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161218T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],vserver,vnf-id-team-auto-11,INVALID OBJ5,20161218T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],vserver2,vnf-id-team-auto-112,INVALID OBJ5 +virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161218T091529Z,MINOR,Rule3,prov-status=[ACTIVE],vserver,vnf-id-team-auto-11,INVALID OBJ6,20161218T091529Z,MINOR,Rule3,prov-status=[ACTIVE],vserver2,vnf-id-team-auto-112,INVALID OBJ6 +newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161217T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ7,20161217T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ7 +newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-11,INVALID OBJ8,20161217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ8 +virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20161217T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ9,20161217T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ9 +newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20160917T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ1,20160917T091529Z,CRITICAL,Rule1,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ1 +newvce,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20160217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],virtual-data-center,vnf-id-team-auto-11,INVALID OBJ2,20160217T091529Z,MAJOR,Rule2,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ2 +virtual-data-center,Invalid prov-status value. Must have a value not equal to ACTIVE/active.,20151017T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver,vnf-id-team-auto-11,INVALID OBJ3,20151017T091529Z,MINOR,Rule3,prov-status=[ACTIVE],pserver2,vnf-id-team-auto-112,INVALID OBJ3 -- cgit 1.2.3-korg