Add logging and tests and build using mvn

An initial version of adding logs to chameleon
An initial version of adding specs (tests) to chameleon.
Adding pom.xml so the project can be build using maven.
Updating README.md for instructions on running it locally.

Issue-ID: AAI-1220
Change-Id: I85f46fd7f625c83b84f211d6766970431e6d91eb
Signed-off-by: Shwetank Dave <shwetank.dave@amdocs.com>

7 files changed, 288 insertions, 80 deletions

diff --git a/src/chameleon/aai_processor.clj b/src/chameleon/aai_processor.clj
index d1a7d25..4243b39 100644
--- a/src/chameleon/aai_processor.clj
+++ b/src/chameleon/aai_processor.clj
@@ -1,9 +1,8 @@
 (ns chameleon.aai-processor
-  (:require
-   [chameleon.route :refer :all]
-   [cheshire.core :refer :all]
-   [integrant.core :as ig]
-   [clojure.set :refer :all]))
+  (:require [chameleon.route :refer :all]
+            [cheshire.core :refer :all]
+            [integrant.core :as ig]
+            [clojure.set :refer :all]))
 
 (defonce ^:private p-attr (atom nil))
 (defonce ^:private t-attr (atom nil))
@@ -33,8 +32,7 @@
     {"id" id
      "type" type
      "source" {"id" src-id "type" src-type}
-     "target" {"id" target-id "type" target-type}})
-  )
+     "target" {"id" target-id "type" target-type}}))
 
 (defn from-gallifrey
   "Transforms Gallifrey response payloads into a format consumable by AAI-centric clients"
@@ -42,27 +40,21 @@
   (let [resource-type (get-in body ["properties" "_type"])
         id (body "_id")
         type (get-in body ["properties" "type"])
-        properties (body "properties")]
+        properties (body "properties")
+        entity-response {"id" id
+                         "type" type
+                         "properties" (dissoc properties "_type" "type")}]
     (if (= resource-type "entity")
-                                        ; Transform into an entity type
+      ;; Transform into an entity type
       (let [relationships (body "relationships")]
-        {
-         "id" id
-         "type" type
-         "properties" (dissoc properties "_type" "type")
-         "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships)))
-         "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships)))
-         })
-                                        ; Transform into a relationship type
-      {
-       "id" id
-       "type" type
-       "properties" (dissoc properties "_type" "type")
-       })))
+        (assoc entity-response
+               "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships)))
+               "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships)))))
+      entity-response)))
 
 (defn from-spike
   "Transforms Spike-based event payloads to a format accepted by Gallifrey for vertices and relationships"
-  [gallifrey-host payload]
+  [gallifrey-host payload & [error-logger audit-logger]]
   (let [txpayload (map-keywords (parse-string payload))
         operation (:operation txpayload)
         parse-type (if (contains? txpayload :vertex)
@@ -74,13 +66,14 @@
         entity (map-keywords (parse-type txpayload))
         key (:key entity)
         properties (assoc (:properties entity) :type (:type entity))
-        truth-time (if (not (nil?  (get properties @t-attr))) {:t-t (get properties @t-attr)})
-        assertion  {:meta {:key key
-                           :operation operation
-                           :time truth-time}}
+        truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)})
+        assertion {:meta {:key key
+                          :operation operation
+                          :time truth-time}}
         provenance (get properties @p-attr "aai")]
-    (assert-gallifrey gallifrey-host provenance (name entity-type) (if (= entity-type :entity)
-                                                                     (assoc assertion  :body (generate-string {:properties properties}))
-                                                                     (assoc assertion :body (generate-string (conj {:properties properties}
-                                                                                                                   {:source  (rename-keys (:source entity) {"key" "id"})}
-                                                                                                                   {:target (rename-keys (:target entity) {"key" "id"})})))))))
+    (assert-gallifrey! gallifrey-host provenance (name entity-type)
+                       (if (= entity-type :entity)
+                         (assoc assertion :body (generate-string {:properties properties}))
+                         (assoc assertion :body (generate-string (conj {:properties properties}
+                                                                       {:source  (rename-keys (:source entity) {"key" "id"})}
+                                                                       {:target (rename-keys (:target entity) {"key" "id"})})))) error-logger audit-logger)))
diff --git a/src/chameleon/config.clj b/src/chameleon/config.clj
index bd99096..10324e2 100644
--- a/src/chameleon/config.clj
+++ b/src/chameleon/config.clj
@@ -4,14 +4,16 @@
 
 (defn config
   [app-config]
-  (let [conf {
+  (let [conf {:chameleon/loggers (:log-config app-config)
               :chameleon/event
               {:event-config (assoc-in (:event-config app-config)
                                        [:aai :processor] from-spike)
-               :gallifrey-host (:gallifrey-host app-config)}
+               :gallifrey-host (:gallifrey-host app-config)
+               :loggers (ig/ref :chameleon/loggers)}
               :chameleon/handler
               {:gallifrey-host (:gallifrey-host app-config)
-               :gallifrey-transformer from-gallifrey}
+               :gallifrey-transformer from-gallifrey
+               :loggers (ig/ref :chameleon/loggers)}
               :chameleon/aai-processor
               {:provenance-attr "last-mod-source-of-truth"
                :truth-attr "truth-time"}
diff --git a/src/chameleon/event.clj b/src/chameleon/event.clj
index c4bec8e..92f4211 100644
--- a/src/chameleon/event.clj
+++ b/src/chameleon/event.clj
@@ -1,19 +1,25 @@
 (ns chameleon.event
   (:require [integrant.core :as ig]
-            [clojure.string :refer [starts-with?]])
+            [clojure.string :refer [starts-with?]]
+            [chameleon.logging :as log])
   (:import [org.onap.aai.event.client DMaaPEventConsumer]))
 
 (defmethod ig/init-key :chameleon/event
-  [_ {:keys [event-config gallifrey-host]}]
+  [_ {:keys [event-config gallifrey-host loggers]}]
   (let [{:keys [host topic motsid pass consumer-group consumer-id timeout batch-size type processor]} (:aai event-config)
+        [error-logger audit-logger] loggers
         event-processor (DMaaPEventConsumer. host topic motsid pass consumer-group consumer-id timeout batch-size type)]
-    (println "Event processor for AAI  created. Starting event polling on " host  topic)
+    (log/info error-logger "EVENT_PROCESSOR" [(format "AAI  created. Starting event polling on %s %s" host topic) ])
     (.start (Thread. (fn [] (while true
                              (let  [it (.iterator (.consume event-processor))]
-                               (println "Polling...")
+                               (log/info error-logger "EVENT_PROCESSOR" ["Polling ..."])
                                (while (.hasNext it)
+                                 (log/mdc-init! "SPIKE-EVENT" "CHAMELEON" "" "" gallifrey-host)
                                  (try (let [event (.next it)]
-                                        (if (not (starts-with? event "DMAAP")) ;Temporarily added for current version of dmaap client
-                                          (processor gallifrey-host event)))
-                                      (catch Exception e (println (str "Unexpected exception during processing: " (.getMessage e)))))))))))
-    ))
+                                        ;;Temporarily added for current version of dmaap client
+                                        (when-not (starts-with? event "DMAAP")
+                                          (log/info error-logger "EVENT_PROCESSOR" [event])
+                                          (processor gallifrey-host event error-logger audit-logger)
+                                          (log/mdc-clear!)))
+                                      (catch Exception e
+                                        (println (str "Unexpected exception during processing: " (.getMessage e)))))))))))))
diff --git a/src/chameleon/handler.clj b/src/chameleon/handler.clj
index fd97bf1..675a34b 100644
--- a/src/chameleon/handler.clj
+++ b/src/chameleon/handler.clj
@@ -10,17 +10,18 @@
             [ring.middleware.session :refer [wrap-session]]
             [cheshire.core :as json]
             [clj-time.format :as tf]
-            [integrant.core :as ig]))
+            [integrant.core :as ig]
+            [chameleon.logging :as log]))
 
 (declare handler)
 
 (defonce ^:private g-host (atom nil))
 (defonce ^:private g-transformer nil)
 
-(defmethod ig/init-key :chameleon/handler  [_ {:keys [gallifrey-host gallifrey-transformer]}]
+(defmethod ig/init-key :chameleon/handler  [_ {:keys [gallifrey-host loggers gallifrey-transformer]}]
   (reset! g-host gallifrey-host)
   (def g-transformer gallifrey-transformer)
-  handler)
+  (handler loggers))
 
 (defmethod ig/halt-key! :chameleon/handler  [_ _]
   (reset! g-host nil)
@@ -32,10 +33,10 @@
   :allowed-methods [:get]
   :available-media-types ["application/json"]
   :exists? (fn [ctx]
-             (let [resource (c-route/query @g-host id type (-> ctx
-                                                               :request
-                                                               :params
-                                                               (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys
+             (let [resource (c-route/query @g-host id type  (-> ctx
+                                                                :request
+                                                                :params
+                                                                (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys
                (when (= (:status resource) 200)
                  {::resource (-> resource
                                  :body
@@ -56,9 +57,27 @@
   (GET "/relationship/:id" [id] (resource-endpoint "relationship"  id))
   (resources "/"))
 
-(def handler
+(defn log-reqs
+  [handler loggers]
+  (let [[error-logger audit-logger] loggers]
+    (fn [request]
+      (log/mdc-init! (get-in request [:headers "X-TransactionId"]) "CHAMELEON"
+                     "CHAMELEON_SERVICE" "ONAP" (:remote-addr request))
+      (log/info error-logger "CHAMELEON_REQUEST" (mapv str ((juxt (comp name :request-method) :uri :remote-addr) request)))
+      (let [resp (handler request)
+            fields (->> ((juxt :status :body) resp)
+                        (into ((juxt (comp name :request-method) :uri) request))
+                        (mapv str))]
+        (log/info error-logger "RESPONSE" fields)
+        (log/info audit-logger "RESPONSE" fields)
+        (log/mdc-clear!)
+        resp))))
+
+(defn handler
+  [loggers]
   (-> app-routes
-      (wrap-defaults api-defaults)))
+      (wrap-defaults api-defaults)
+      (log-reqs loggers)))
 
 
 ;;; Implementation
diff --git a/src/chameleon/logging.clj b/src/chameleon/logging.clj
new file mode 100644
index 0000000..15e1e7b
--- /dev/null
+++ b/src/chameleon/logging.clj
@@ -0,0 +1,123 @@
+(ns chameleon.logging
+  (:require [camel-snake-kebab.core :as cs]
+            [camel-snake-kebab.extras :refer [transform-keys]]
+            [clojure.java.io :as io]
+            [integrant.core :as ig]
+            [clojure.spec.alpha :as s])
+  (:import [org.onap.aai.cl.api Logger LogFields LogLine]
+           [org.onap.aai.cl.eelf LoggerFactory LogMessageEnum AaiLoggerAdapter AuditLogLine]
+           [org.onap.aai.cl.mdc MdcContext MdcOverride]
+           [com.att.eelf.i18n EELFResourceManager]
+           [clojure.lang.IFn]
+           [org.slf4j MDC]))
+
+(declare LOGLINE_DEFINED_FIELDS error-logger audit-logger ->java string->enum logfields)
+
+(defmethod ig/init-key :chameleon/loggers
+  [_ {:keys [logback logmsgs] :or {logback (.getPath (io/file "chameleon_logback.xml"))
+                                   logmsgs "log/ChameleonMsgs"}}]
+  (System/setProperty "logback.configurationFile" logback)
+  (EELFResourceManager/loadMessageBundle logmsgs)
+  [(error-logger "chameleon.loggging") (audit-logger "chameleon.loggging")])
+
+(defn conform-multiple
+  [& spec-form-pair]
+  (if (s/valid? :chameleon.specs/spec-form-pair spec-form-pair)
+    (->> spec-form-pair
+         (partition 2)
+         (map (fn [[sp form]]
+                (when (s/invalid? (s/conform sp form))
+                  (s/explain-data sp form))))
+         (remove nil?))
+    (s/explain-data :chameleon.specs/spec-form-pair spec-form-pair)))
+
+(defn mdc-set!
+  "Sets the global MDC context for the current thread."
+  [m]
+  (doseq [[k v] m] (MDC/put k v)))
+
+(defn mdc-init!
+  "Sets the global MDC context as required by the EELF logging library"
+  [transaction-id service instance partner client-address]
+  (MdcContext/initialize transaction-id service instance partner client-address))
+
+(defn mdc-clear! [] (MDC/clear))
+
+(defmacro with-mdc
+  "Will set the global MDC context with the options (will convert from
+  keywords to PascalCase), execute the log, and clear the MDC
+  context."
+  [opts & body]
+  `(do
+     (mdc-set! (transform-keys cs/->PascalCaseString ~opts))
+     ~@body
+     (mdc-clear)))
+
+(defn mdc-override
+  [m]
+  (->java (fn [j k v] (.addAttribute j k v)) (new MdcOverride)
+          (transform-keys cs/->PascalCaseString m)))
+
+(defn error-logger
+  [name]
+  (.getLogger (LoggerFactory/getInstance) name))
+
+(defn audit-logger
+  [name]
+  (.getAuditLogger (LoggerFactory/getInstance) name))
+
+(defn logger?
+  [logger]
+  (instance? AaiLoggerAdapter logger))
+
+(defn info
+  [^AaiLoggerAdapter logger ^String enum msgs & {:keys [fields] :or {fields {}}}]
+  (let [confirmed-specs (conform-multiple :logging/valid-fields fields :logging/msgs msgs
+                                          :chameleon.specs/logger logger)]
+    (if (empty? confirmed-specs)
+      (.info logger (string->enum enum) (logfields fields) (into-array java.lang.String msgs))
+      confirmed-specs)))
+
+(defn debug
+  [^AaiLoggerAdapter logger ^String enum  msgs]
+  (.debug logger (string->enum enum) (into-array java.lang.String msgs)))
+
+(defn valid-logfields?
+  [m]
+  (->> (keys m)
+       (map cs/->SCREAMING_SNAKE_CASE_STRING)
+       set
+       (clojure.set/superset? (-> LOGLINE_DEFINED_FIELDS keys set))))
+
+(def ^{:private true
+       :doc "Adding these fields from \"org.onap.aai.cl.api.LogLine\"
+  class. Right now there isn't a known way to use the ENUMs of an
+  abstract JAVA class if they are not STATIC.  The field order is very
+  specific and it should be maintained for the common logging library
+  version \"1.2.2\".  For a better understanding, please look at the
+  \"DefinedFields\" ENUMs in the org.onap.aai.cl.api.LogLine
+  class"}
+
+  LOGLINE_DEFINED_FIELDS
+  (zipmap ["STATUS_CODE" "RESPONSE_CODE" "RESPONSE_DESCRIPTION" "INSTANCE_UUID"
+           "SEVERITY" "SERVER_IP" "CLIENT_IP" "CLASS_NAME" "PROCESS_KEY"
+           "TARGET_SVC_NAME" "TARGET_ENTITY" "ERROR_CODE" "ERROR_DESCRIPTION"
+           "CUSTOM_1" "CUSTOM_2""CUSTOM_3" "CUSTOM_4"]
+          (range)))
+
+(defn- string->enum
+  ([^String enum] (proxy [Enum LogMessageEnum] [enum (hash enum)]))
+  ([^String enum ordinal] (proxy [Enum LogMessageEnum] [enum ordinal])))
+
+(defn- ->java
+  [f jc m]
+  (reduce (fn [a [k v]] (f a k v) a)
+          jc m))
+
+(defn- logfields
+  "Generate a \"LogFields\" object with all the fields set."
+  [m]
+  (->> m
+       (transform-keys cs/->SCREAMING_SNAKE_CASE_STRING)
+       (transform-keys #(string->enum % (LOGLINE_DEFINED_FIELDS %)))
+       (->java (fn [j field v] (.setField j field v)) (new LogFields))))
diff --git a/src/chameleon/route.clj b/src/chameleon/route.clj
index e8f3a3c..0918da4 100644
--- a/src/chameleon/route.clj
+++ b/src/chameleon/route.clj
@@ -1,12 +1,7 @@
 (ns chameleon.route
-  (:require [org.httpkit.client :as kitclient]))
-
-(defn- interpret-response
-  "Print out the response from the Gallifrey server"
-  [key response]
-  (let [{:keys [status body]}@response]
-    (println "Response for request with key " key " resulted in status " status
-             " with body " body )))
+  (:require [org.httpkit.client :as kitclient]
+            [chameleon.logging :as log]
+            [ring.util.http-status :as hs]))
 
 (defn query
   "Retrieve an entity referenced by id at the provided host. Optionally provide
@@ -21,11 +16,10 @@
                        :keepalive 300
                        :timeout 20000}))
 
-(defn assert-create
+(defn assert-create!
   "Creates an entity in Gallifrey with an initial set of assertions coming from the provided payload"
   [host actor type key payload & [time-dimensions]]
-  (kitclient/request {
-                      :url (str "https://" host "/" type "/" key)
+  (kitclient/request {:url (str "https://" host "/" type "/" key)
                       :method :put
                       :query-params (into {"actor" actor "create" "true"} time-dimensions)
                       :body payload
@@ -33,11 +27,10 @@
                       :keepalive 300
                       :timeout 1000}))
 
-(defn assert-update
+(defn assert-update!
   "Update an entity in Gallifrey with a set of assertions coming from the provided payload"
   [host actor type key payload & [time-dimensions]]
-  (kitclient/request {
-                      :url (str "https://" host "/" type "/" key)
+  (kitclient/request {:url (str "https://" host "/" type "/" key)
                       :method :put
                       :query-params (into  {"actor" actor "changes-only" "true"} time-dimensions)
                       :body payload
@@ -45,26 +38,27 @@
                       :keepalive 300
                       :timeout 1000}))
 
-(defn assert-delete
+(defn assert-delete!
   "Assert a deletion for an entity in Gallifrey based on the provided key."
   [host actor type key & [time-dimensions]]
-  (kitclient/request {
-                      :url (str "https://" host "/" type "/" key)
+  (kitclient/request {:url (str "https://" host "/" type "/" key)
                       :method :delete
-                      :query-params (into  {"actor" actor} time-dimensions)
+                      :query-params (into {"actor" actor} time-dimensions)
                       :insecure? true
                       :keepalive 300
                       :timeout 1000}))
 
-(defn assert-gallifrey [host actor type payload]
+(defn assert-gallifrey!
+  [host actor type payload & [e-logger a-logger]]
   "Propagates an assertion to Gallifrey based off of an event payload coming in from the event service."
   (let [{:keys [meta body]} payload
-        {:keys [key operation time]} meta]
-    (println operation " "  type "  with key " key)
-    (interpret-response key (case operation
-                              "CREATE"
-                              (assert-create host actor type key body time)
-                              "UPDATE"
-                              (assert-update host actor type key body time)
-                              "DELETE"
-                              (assert-delete host actor type key time)))))
+        {:keys [key operation time]} meta
+        _ (log/info e-logger "GALLIFREY_ASSERTION" (mapv str [operation type key]))
+        g-assert (case operation
+                   "CREATE" (assert-create! host actor type key body time)
+                   "UPDATE" (assert-update! host actor type key body time)
+                   "DELETE" (assert-delete! host actor type key time))
+        {:keys [status body]} @g-assert]
+    (log/info e-logger "GALLIFREY_ASSERTED" [(str type) (str key)])
+    (log/info a-logger "RESPONSE" (mapv str [operation key status body])
+              :fields {:response-code status :response-description (hs/get-name status)})))
diff --git a/src/chameleon/specs.clj b/src/chameleon/specs.clj
new file mode 100644
index 0000000..40d5768
--- /dev/null
+++ b/src/chameleon/specs.clj
@@ -0,0 +1,71 @@
+(ns chameleon.specs
+  (:require [clojure.spec.alpha :as s]
+            [chameleon.logging :as log]
+            [clojure.spec.gen.alpha :as gen]
+            [cheshire.core :as c]
+            [clojure.string :as str]))
+
+(s/def ::host string?)
+(s/def ::provenance string?)
+(s/def ::payload map?)
+(s/def ::string (s/spec (s/and string? (complement str/blank?)) :gen gen/string-alphanumeric))
+(s/def ::type (s/spec ::string :gen #(gen/elements ["vserver" "pserver" "generic-vnf"])))
+(s/def ::id uuid?)
+(s/def ::source (s/keys :req-un [::type ::id]))
+(s/def ::target (s/keys :req-un [::type ::id]))
+(s/def ::_id ::id)
+(s/def ::spec-form-pair (s/* (s/cat :spec #(% (s/registry)) :form any?)))
+
+;; spike event
+(s/def :spike/transaction-id uuid?)
+(s/def :spike/schema-version (s/spec #(re-matches #"v\d+" %) :gen #(->> (s/int-in 7 15)
+                                                                        s/gen
+                                                                        (gen/fmap (partial str "v")))))
+(s/def :spike/key uuid?)
+(s/def :spike/name ::string)
+(s/def :spike/last-mod-source-of-truth ::string)
+(s/def :spike/source-of-truth ::string)
+(s/def :spike/vserver-selflink ::string)
+(s/def :spike/is-closed-loop-disabled boolean?)
+(s/def :spike/in-maint boolean?)
+(s/def :spike/timestamp inst?)
+(s/def :spike/operation (s/spec ::string :gen #(gen/elements ["UPDATE" "CREATE" "DELETE"])))
+(s/def :spike/properties (s/keys :req-un [:spike/name ::id :spike/last-mod-source-of-truth
+                                          :spike/source-of-truth :spike/vserver-selflink
+                                          :spike/is-closed-loop-disabled :spike/in-maint]))
+(s/def :spike/vertex (s/keys :req-un [:spike/schema-version ::type :spike/key :spike/properties]))
+(s/def :spike/event (s/keys :req-un [:spike/transaction-id :spike/vertex
+                                     :spike/operation :spike/timestamp]))
+(s/def :spike/payload (s/spec string? :gen #(gen/fmap (partial c/generate-string)
+                                                      (s/gen :spike/event))))
+(s/def :gallifrey/k-end-actor ::string)
+
+;; gallifrey response
+(s/def :gallifrey/k-start-actor ::string)
+(s/def :gallifrey/k-end inst?)
+(s/def :gallifrey/k-start inst?)
+(s/def :gallifrey/history (s/keys :req-un [:gallifrey/k-end-actor :gallifrey/k-end
+                                           :gallifrey/k-start-actor :gallifrey/k-start]))
+(s/def :relationship/_meta (s/map-of ::string :gallifrey/history))
+(s/def :relationship/_type (s/spec string? :gen #(gen/return "relationship")))
+(s/def :gallifrey/_type (s/spec ::string :gen #(gen/return "entity")))
+(s/def :gallifrey/properties (s/keys :req-un [:gallifrey/_type ::type]))
+(s/def :relationship/type (s/spec string? :gen #(->> (gen/string-alphanumeric)
+                                                     (gen/such-that (complement str/blank?))
+                                                     (gen/fmap (partial str "tosca.relationship.")))))
+(s/def :relationship/properties (s/keys :req-un [:relationship/_type :relationship/type]))
+(s/def ::relationship (s/keys :req-un [:relationship/properties ::source
+                                       ::target :relationship/_meta ::_id]))
+(s/def ::relationships (s/coll-of ::relationship :gen-max 8))
+(s/def :gallifrey/payload (s/spec map?
+                                  :gen #(->> [::_id :gallifrey/properties
+                                              :gallifrey/properties ::relationships]
+                                             (s/keys :req-un)
+                                             s/gen
+                                             (gen/fmap (partial clojure.walk/stringify-keys)))))
+
+;; Logger specs
+(s/def ::logger (s/spec log/logger? :gen #(gen/return (log/error-logger "chameleon.specs"))))
+(s/def ::loggers (s/cat :e :chameleon.specs/logger :a :chameleon.specs/logger))
+(s/def :logging/msgs (s/* ::string))
+(s/def :logging/valid-fields log/valid-logfields?)