EELF and email notification

Issue-ID: DCAEGEN2-554
Change-Id: Ibcc19aa84c4bd687fb258b4e1effddedb0328871
Signed-off-by: Au, Bill (wa998j) <wa998j@att.com>

12 files changed, 220 insertions, 43 deletions

diff --git a/Dockerfile b/Dockerfile
index 1245a2d..7b4d2bc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,5 @@
 FROM maven:3-jdk-8
+RUN apt-get update && apt-get install -y sendmail
 
 WORKDIR /opt/sch
 ADD . /opt/sch
@@ -7,4 +8,6 @@ RUN mvn clean package
 # TODO: This is bogus. This is simply to be used for Registrator registration.
 EXPOSE 65000
 
-CMD ["java", "-jar", "/opt/sch/target/dcae-service-change-handler.jar", "prod", "http://consul:8500/v1/kv/service-change-handler?raw=true"]
+COPY startSCH.sh /opt/sch
+RUN chmod +x /opt/sch/startSCH.sh
+CMD ["/opt/sch/startSCH.sh"]
diff --git a/logback.xml b/logback.xml
new file mode 100644
index 0000000..6a28b70
--- /dev/null
+++ b/logback.xml
@@ -0,0 +1,108 @@
+<configuration debug="true" scan="true" scanPeriod="30 seconds">
+
+    <property name="LOG_DIR" value="logs" />
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <Pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</Pattern>
+        </encoder>
+    </appender>
+    
+    <appender name="metricsAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>DEBUG</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+        <file>${LOG_DIR}/metrics.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <!-- rollover daily -->
+            <fileNamePattern>${LOG_DIR}/metrics-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <!-- each file should be at most 100MB, keep 2 days worth of history, but at most 200MB -->
+            <maxFileSize>100MB</maxFileSize>
+            <maxHistory>2</maxHistory>
+            <totalSizeCap>200MB</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSS+00:00}|%d{yyyy-MM-dd'T'HH:mm:ss.SSS+00.00}|NULL|UNKONWN|%thread||service-change-handler||||COMPLETE|0|OK||%level||||||%class||||%method|%line|||%msg%n</pattern>
+        </encoder>
+    </appender>
+	
+    <appender name="debugAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>DEBUG</level>
+        </filter>
+        <file>${LOG_DIR}/debug.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <!-- rollover daily -->
+            <fileNamePattern>${LOG_DIR}/debug-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <!-- each file should be at most 100MB, keep 2 days worth of history, but at most 200MB -->
+            <maxFileSize>100MB</maxFileSize>
+            <maxHistory>2</maxHistory>
+            <totalSizeCap>200MB</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSS+00.00}|NULL|%thread:%level:%class:%method:%line:%msg%n</pattern>
+        </encoder>
+    </appender>
+	
+    <appender name="auditAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>INFO</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+        <file>${LOG_DIR}/audit.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_DIR}/audit-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <maxFileSize>100MB</maxFileSize>
+            <maxHistory>2</maxHistory>
+            <totalSizeCap>200MB</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSS+00:00}|%d{yyyy-MM-dd'T'HH:mm:ss.SSS+00:00}|NULL|UNKNOWN|%thread||service-change-handler||COMPLETE|0|OK||%level||||||%class|||%method|%line|||%msg%n</pattern>
+        </encoder>
+    </appender>
+	
+    <appender name="errorAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>ERROR</level>
+        </filter>
+        <file>${LOG_DIR}/error.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <!-- rollover daily -->
+            <fileNamePattern>${LOG_DIR}/error-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <!-- each file should be at most 100MB, keep 2 days worth of history, but at most 200MB -->
+            <maxFileSize>100MB</maxFileSize>
+            <maxHistory>2</maxHistory>
+            <totalSizeCap>200MB</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSS+00.00}|NULL|%thread|service-change-handler||||%level|500|ERROR|%class:%method:%line:%msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="metricsLogger" level="info" additivity="false">
+        <appender-ref ref="metricsAppender" />
+    </logger>
+
+    <logger name="debugLogger" level="debug" additivity="false">
+        <appender-ref ref="debugAppender" />
+    </logger>
+
+    <logger name="auditLogger" level="info" additivity="false">
+        <appender-ref ref="auditAppender" />
+    </logger> -->
+
+    <logger name="errorLogger" level="error" additivity="false">
+        <appender-ref ref="errorAppender" />
+    </logger>
+
+    <root level="debug">
+        <appender-ref ref="metricsAppender" />
+        <appender-ref ref="debugAppender" />
+        <appender-ref ref="auditAppender" />
+        <appender-ref ref="errorAppender" />
+    </root>
+
+</configuration>
diff --git a/pom.xml b/pom.xml
index 40c599a..7e348a7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,20 +65,19 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property.
             <version>5.8.0</version>
         </dependency>
         <dependency>
+            <groupId>com.draines</groupId>
+            <artifactId>postal</artifactId>
+            <version>2.0.2</version>
+        </dependency>
+        <dependency>
             <groupId>org.onap.sdc.sdc-distribution-client</groupId>
             <artifactId>sdc-distribution-client</artifactId>
             <version>1.3.0</version>
         </dependency>
         <dependency>
-            <groupId>com.taoensso</groupId>
-            <artifactId>timbre</artifactId>
-            <version>4.10.0</version>
-        </dependency>
-        <!-- Does magic so traditional Java loggers show up -->
-        <dependency>
-            <groupId>com.fzakaria</groupId>
-            <artifactId>slf4j-timbre</artifactId>
-            <version>0.3.8</version>
+            <groupId>org.clojure</groupId>
+            <artifactId>tools.logging</artifactId>
+            <version>0.4.0</version>
         </dependency>
         <dependency>
             <groupId>clj-http</groupId>
@@ -96,6 +95,11 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property.
             <version>0.4.0</version>
         </dependency>
         <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+                <version>1.2.1</version>
+        </dependency>
+        <dependency>
             <groupId>clj-fakes</groupId>
             <artifactId>clj-fakes</artifactId>
             <version>0.9.0</version>
diff --git a/project.clj b/project.clj
index 67a294a..3731a39 100644
--- a/project.clj
+++ b/project.clj
@@ -25,12 +25,12 @@
   :description "Service change handler"
   :dependencies [[org.clojure/clojure "1.8.0"]
                  [cheshire/cheshire "5.8.0"]
-                 [com.taoensso/timbre "4.7.4"]
-                 [com.fzakaria/slf4j-timbre "0.3.2"]
+                 [org.clojure/tools.logging "0.4.0"]
                  [clj-http/clj-http "3.3.0"]
                  [org.bovinegenius/exploding-fish "0.3.4"]
                  [clj-yaml/clj-yaml "0.4.0"]
                  [org.onap.sdc.sdc-distribution-client/sdc-distribution-client "1.3.0"]
+                 [ch.qos.logback/logback-classic "1.2.1"]
                  ]
 
   ; TODO: Fill in the onap maven repository info
diff --git a/src/sch/asdc_client.clj b/src/sch/asdc_client.clj
index b859f03..83b5017 100644
--- a/src/sch/asdc_client.clj
+++ b/src/sch/asdc_client.clj
@@ -20,7 +20,7 @@
 
 (ns sch.asdc-client
   (:require [clj-http.client :as client]
-            [taoensso.timbre :as timbre :refer [error]]
+            [clojure.tools.logging :as logger :refer [debug error]]
             [cheshire.core :refer [parse-string]]
             [org.bovinegenius.exploding-fish :refer [uri param]])
   (:gen-class))
@@ -47,6 +47,7 @@
 
 (defn get-artifact!
   [connection artifact-path]
+  (debug (str "Entering get-artifact - artiface-path:" artifact-path))
   (let [[asdc-uri user password instance-id insecure?] connection
         target-uri (assoc asdc-uri :path artifact-path)
         resp (client/get (str target-uri) { :basic-auth [user password]
@@ -61,6 +62,7 @@
 
 (defn get-service-metadata!
   [connection service-uuid]
+  (debug (str "Entering get-service-metadta - service-uuid:" service-uuid))
   (let [[asdc-uri user password instance-id insecure?] connection
         target-uri (assoc asdc-uri :path (construct-service-path service-uuid))
         resp (client/get (str target-uri) { :basic-auth [user password]
diff --git a/src/sch/core.clj b/src/sch/core.clj
index 1852c76..1cd2621 100644
--- a/src/sch/core.clj
+++ b/src/sch/core.clj
@@ -21,8 +21,7 @@
 (ns sch.core
   (:require [clojure.java.io :refer :all]
             [cheshire.core :refer [parse-stream parse-string]]
-            [taoensso.timbre :as timbre :refer [info error]]
-            [taoensso.timbre.appenders.3rd-party.rolling :refer [rolling-appender]]
+            [clojure.tools.logging :as logger :refer [debug info error]]
             [sch.handle :refer [handle-change-event! download-artifacts! deploy-artifacts!
                                 deployed-ok deployed-error deployed-already]]
             [sch.asdc-client :refer [get-service-metadata! create-asdc-conn get-consumer-id]]
@@ -30,10 +29,11 @@
             [sch.parse :refer [get-dcae-artifact-types pick-out-artifact]]
             [sch.util :refer [read-config]]
             [clojure.string :as strlib]
+            [postal.core :refer [send-message]]
             )
   (:import (org.onap.sdc.impl DistributionClientFactory)
            (org.onap.sdc.api.consumer IConfiguration INotificationCallback
-                                            IDistributionStatusMessage)
+                                            IDistributionStatusMessage IComponentDoneStatusMessage)
            (org.onap.sdc.utils DistributionActionResultEnum DistributionStatusEnum)
            (com.google.gson Gson)
            )
@@ -66,24 +66,74 @@
                   (str (.getDistributionMessageResult resp)))))
     ))
 
+(defn send-component-done-status!
+  "Convenience function used to send component done status messages"
+  [dist-client distribution-id consumer-id msg artifact status]
+  (let [dist-message (proxy [IComponentDoneStatusMessage] []
+                       (getDistributionID [] distribution-id)
+                       (getConsumerID [] consumer-id)
+                       (getTimestamp []
+                         (. java.lang.System currentTimeMillis))
+                       (getArtifactURL [] (:artifactURL artifact))
+                       (getComponentName [] "service-change-handler")
+                       (getStatus [] status))
+        resp (if (strlib/blank? msg)
+               (.sendComponentDoneStatus dist-client dist-message)
+               (.sendComponentDoneStatus dist-client dist-message "failed to deploy to inventory"))
+        ]
+    (if (not= (.getDistributionActionResult resp) (. DistributionActionResultEnum SUCCESS))
+      (error (str "Problem sending status: " (:artifactName artifact) ", "
+                  (str (.getDistributionMessageResult resp)))))
+    ))
+
 (defn deploy-artifacts-ex!
   "Enhanced deploy artifacts function
 
   After calling deploy-artifacts!, this method takes the results and sends out
   appropriate distribution status messages per artifact processed"
-  [inventory-uri service-metadata requests send-dist-status]
+  [inventory-uri service-metadata requests send-dist-status send-comp-done-status fromEmail toEmail]
   (let [[to-post posted to-delete deleted] (deploy-artifacts! inventory-uri service-metadata
                                                               requests)
         pick-out-artifact (partial pick-out-artifact service-metadata)]
 
-    (dorun (map #(send-dist-status (pick-out-artifact %)
+    (dorun (map #(do
+                   (send-dist-status (pick-out-artifact %)
                                    (. DistributionStatusEnum DEPLOY_OK))
+                   (send-comp-done-status "" (pick-out-artifact %)
+                                   (. DistributionStatusEnum COMPONENT_DONE_OK))
+                   (if (and (not (strlib/blank? fromEmail)) (not (strlib/blank? toEmail)))
+                     (do
+                       (debug (str "Sending notification from " fromEmail " to " toEmail))
+                       (try
+                         (send-message {:from fromEmail
+                                        :to [toEmail]
+                                        :subject "DCAE inventory blueprint downloaded from ASDC and inserted into inventory DB"
+                                        :body (str
+                                                "ASDC blueprint has been inserted into inventory <"
+                                                inventory-uri
+                                                ">.\n"
+                                                (pick-out-artifact %)
+                                              )
+                                       })
+                         (catch Exception e (error (str "caught exception from send-message" (.getMessage e))))
+                       )
+                     )
+                   )
+                 )
                 (deployed-ok to-post posted)))
-    (dorun (map #(send-dist-status (pick-out-artifact %)
+    (dorun (map #(do
+                   (send-dist-status (pick-out-artifact %)
                                    (. DistributionStatusEnum DEPLOY_ERROR))
+                   (send-comp-done-status "failed to deploy to inventory" (pick-out-artifact %)
+                                   (. DistributionStatusEnum COMPONENT_DONE_ERROR))
+                   )
                 (deployed-error to-post posted)))
-    (dorun (map #(send-dist-status (pick-out-artifact %)
+    (dorun (map #(do
+                   (send-dist-status (pick-out-artifact %)
                                    (. DistributionStatusEnum ALREADY_DEPLOYED))
+                   (send-comp-done-status "" (pick-out-artifact %)
+                                   (. DistributionStatusEnum COMPONENT_DONE_OK))
+                   )
                 (deployed-already requests to-post)))
     ; REVIEW: How about the deleted service types?
     ))
@@ -113,7 +163,7 @@
       (getKeyStorePassword [] (str (:keyStorePassword config-asdc)))
       (activateServerTLSAuth [] (boolean (:activateServerTLSAuth config-asdc)))
       (isFilterInEmptyResources [] (boolean (:isFilterInEmptyResources config-asdc)))
-      (isUseHttpsWithDmaap [] (boolean (:useHttpsWithDmaap config-asdc false)))
+      (isUseHttpsWithDmaap [] (boolean (:useHttpsWithDmaap config-asdc true)))
       (isConsumeProduceStatusTopic [] (boolean (:isConsumeProduceStatusTopic config-asdc false)))
       )))
 
@@ -122,7 +172,8 @@
 
   Uses the asdc distribution client and to poll for notification events and makes calls
   to handle those events"
-  [dist-client-config inventory-uri asdc-conn]
+  [dist-client-config inventory-uri asdc-conn fromEmail toEmail]
+  (debug "Entering run-distribution-client")
   (let [dist-client (. DistributionClientFactory createDistributionClient)
         dist-client-callback (proxy [INotificationCallback] []
                                (activateCallback [data]
@@ -136,6 +187,7 @@
                                  Discovered that the notification event and the service metadata
                                  data models are different. Use service metadata because its
                                  richer."
+                                 (debug "Entering dist-client-callback")
                                  (let [change-event (parse-string (.toJson (Gson.) data) true)
                                        service-id (:serviceUUID change-event)
                                        distribution-id (:distributionID change-event)
@@ -144,6 +196,9 @@
                                        send-dist-status (partial send-distribution-status!
                                                                  dist-client distribution-id
                                                                  (get-consumer-id asdc-conn))
+                                       send-comp-done-status (partial send-component-done-status!
+                                                                 dist-client distribution-id
+                                                                 (get-consumer-id asdc-conn))
                                        ]
 
                                    (info (str "Handle change event: " (:serviceName change-event)
@@ -159,7 +214,8 @@
                                                     % (. DistributionStatusEnum DOWNLOAD_OK))
                                                  artifacts))
                                      (deploy-artifacts-ex! inventory-uri service-metadata
-                                                           requests send-dist-status)
+                                                           requests send-dist-status send-comp-done-status
+                                                           fromEmail toEmail)
                                      )
 
                                    )))
@@ -171,26 +227,18 @@
         (error dist-client-init-result))
       )))
 
-(defn- setup-logging-rolling
-  "Setup logging with the rolling appender"
-  [{ {:keys [currentLogFilename rotationFrequency]} :logging }]
-  (let [rolling-params (when currentLogFilename { :path currentLogFilename })
-        rolling-params (when rotationFrequency
-                         (assoc rolling-params :pattern (keyword rotationFrequency)))]
-    (timbre/merge-config! { :level :debug :appenders { :rolling (rolling-appender rolling-params) } })
-    (info "Setup logging: Rolling appender" (if rolling-params rolling-params "DEFAULT"))
-  ))
-
-
 (defn -main [& args]
   (let [[mode config-path event-path] args
         config (read-config config-path)
         inventory-uri (create-inventory-conn config)
         asdc-conn (create-asdc-conn config)
+        fromEmail (get-in config [:notification :fromEmail])
+        toEmail (get-in config [:notification :toEmail])
         ]
 
-    (setup-logging-rolling config)
-
+    (if (and (not (strlib/blank? fromEmail)) (not (strlib/blank? toEmail)))
+      (debug "Email notification enabled")
+    )
     (if (= "DEV" (clojure.string/upper-case mode))
       (do
         (info "Development mode")
@@ -199,7 +247,7 @@
         )
       (run-distribution-client!
         (create-distribution-client-config config)
-        inventory-uri asdc-conn)
+        inventory-uri asdc-conn fromEmail toEmail)
       )
 
     (info "Done"))
diff --git a/src/sch/handle.clj b/src/sch/handle.clj
index 453b586..d537aa3 100644
--- a/src/sch/handle.clj
+++ b/src/sch/handle.clj
@@ -20,7 +20,7 @@
 
 (ns sch.handle
   (:require [clojure.java.io :refer :all]
-            [taoensso.timbre :as timbre :refer [info error]]
+            [clojure.tools.logging :as logger :refer [info error]]
             [sch.inventory-client :refer [get-service-types! post-service-type!
                                           delete-service-type!]]
             [sch.asdc-client :refer [get-artifact!]]
diff --git a/src/sch/inventory_client.clj b/src/sch/inventory_client.clj
index a38b693..75907df 100644
--- a/src/sch/inventory_client.clj
+++ b/src/sch/inventory_client.clj
@@ -20,7 +20,7 @@
 
 (ns sch.inventory-client
   (:require [clj-http.client :as client]
-            [taoensso.timbre :as timbre :refer [error]]
+            [clojure.tools.logging :as logger :refer [error]]
             [cheshire.core :refer [parse-string]]
             [org.bovinegenius.exploding-fish :refer [uri param]])
   (:gen-class))
diff --git a/src/sch/parse.clj b/src/sch/parse.clj
index 668dbf4..c60c878 100644
--- a/src/sch/parse.clj
+++ b/src/sch/parse.clj
@@ -20,7 +20,7 @@
 
 (ns sch.parse
   (:require [clojure.java.io :refer :all]
-            [taoensso.timbre :as timbre :refer [info error]]
+            [clojure.tools.logging :as logger]
             [sch.asdc-client :refer [construct-service-path]]
             [cheshire.core :refer [parse-string]]
             )
diff --git a/startSCH.sh b/startSCH.sh
new file mode 100644
index 0000000..951b1eb
--- /dev/null
+++ b/startSCH.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+grep "^nameserver" /opt/sch/etc/resolv.conf >> /etc/resolv.conf
+
+service sendmail start
+
+java -Dlogback.configurationFile=logback.xml -jar /opt/sch/target/dcae-service-change-handler.jar prod http://consul:8500/v1/kv/service-change-handler?raw=true
+
diff --git a/test/sch/core_test.clj b/test/sch/core_test.clj
index 4d0ee2b..8688eb3 100644
--- a/test/sch/core_test.clj
+++ b/test/sch/core_test.clj
@@ -33,6 +33,10 @@
           (send-dist-status-only-ok [artifact status]
             (if (not= (. DistributionStatusEnum ALREADY_DEPLOYED) status)
               (throw (Exception. "Distribution status should be ALREADY DEPLOYED"))
+              ))
+          (send-comp-done-status-only-ok [msg artifact status]
+            (if (not= (. DistributionStatusEnum COMPONENT_DONE_OK) status)
+              (throw (Exception. "Distribution status should be COMPONENT DONE OK"))
               ))]
     (let [service-metadata [{:resources [{:resourceInvariantUUID "123"
                                           :artifacts [:artifactName "type-foo"]
@@ -43,7 +47,7 @@
       (f/with-fakes
         (f/patch! #'sch.handle/deploy-artifacts! deploy-artifacts)
         (is (= nil (deploy-artifacts-ex! "http://inventory" service-metadata requests
-                                         send-dist-status-only-ok))))
+                                         send-dist-status-only-ok send-comp-done-status-only-ok "" ""))))
       )))
 
 
diff --git a/version.properties b/version.properties
index dba81ff..07578e5 100644
--- a/version.properties
+++ b/version.properties
@@ -1,6 +1,6 @@
 major=1

-minor=1

-patch=5

+minor=2

+patch=0

 base_version=${major}.${minor}.${patch}

 release_version=${base_version}

 snapshot_version=${base_version}-SNAPSHOT