aboutsummaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/chameleon/aai_processor.clj57
-rw-r--r--src/chameleon/config.clj8
-rw-r--r--src/chameleon/event.clj22
-rw-r--r--src/chameleon/handler.clj37
-rw-r--r--src/chameleon/logging.clj123
-rw-r--r--src/chameleon/route.clj50
-rw-r--r--src/chameleon/specs.clj71
7 files changed, 288 insertions, 80 deletions
diff --git a/src/chameleon/aai_processor.clj b/src/chameleon/aai_processor.clj
index d1a7d25..4243b39 100644
--- a/src/chameleon/aai_processor.clj
+++ b/src/chameleon/aai_processor.clj
@@ -1,9 +1,8 @@
(ns chameleon.aai-processor
- (:require
- [chameleon.route :refer :all]
- [cheshire.core :refer :all]
- [integrant.core :as ig]
- [clojure.set :refer :all]))
+ (:require [chameleon.route :refer :all]
+ [cheshire.core :refer :all]
+ [integrant.core :as ig]
+ [clojure.set :refer :all]))
(defonce ^:private p-attr (atom nil))
(defonce ^:private t-attr (atom nil))
@@ -33,8 +32,7 @@
{"id" id
"type" type
"source" {"id" src-id "type" src-type}
- "target" {"id" target-id "type" target-type}})
- )
+ "target" {"id" target-id "type" target-type}}))
(defn from-gallifrey
"Transforms Gallifrey response payloads into a format consumable by AAI-centric clients"
@@ -42,27 +40,21 @@
(let [resource-type (get-in body ["properties" "_type"])
id (body "_id")
type (get-in body ["properties" "type"])
- properties (body "properties")]
+ properties (body "properties")
+ entity-response {"id" id
+ "type" type
+ "properties" (dissoc properties "_type" "type")}]
(if (= resource-type "entity")
- ; Transform into an entity type
+ ;; Transform into an entity type
(let [relationships (body "relationships")]
- {
- "id" id
- "type" type
- "properties" (dissoc properties "_type" "type")
- "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships)))
- "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships)))
- })
- ; Transform into a relationship type
- {
- "id" id
- "type" type
- "properties" (dissoc properties "_type" "type")
- })))
+ (assoc entity-response
+ "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships)))
+ "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships)))))
+ entity-response)))
(defn from-spike
"Transforms Spike-based event payloads to a format accepted by Gallifrey for vertices and relationships"
- [gallifrey-host payload]
+ [gallifrey-host payload & [error-logger audit-logger]]
(let [txpayload (map-keywords (parse-string payload))
operation (:operation txpayload)
parse-type (if (contains? txpayload :vertex)
@@ -74,13 +66,14 @@
entity (map-keywords (parse-type txpayload))
key (:key entity)
properties (assoc (:properties entity) :type (:type entity))
- truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)})
- assertion {:meta {:key key
- :operation operation
- :time truth-time}}
+ truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)})
+ assertion {:meta {:key key
+ :operation operation
+ :time truth-time}}
provenance (get properties @p-attr "aai")]
- (assert-gallifrey gallifrey-host provenance (name entity-type) (if (= entity-type :entity)
- (assoc assertion :body (generate-string {:properties properties}))
- (assoc assertion :body (generate-string (conj {:properties properties}
- {:source (rename-keys (:source entity) {"key" "id"})}
- {:target (rename-keys (:target entity) {"key" "id"})})))))))
+ (assert-gallifrey! gallifrey-host provenance (name entity-type)
+ (if (= entity-type :entity)
+ (assoc assertion :body (generate-string {:properties properties}))
+ (assoc assertion :body (generate-string (conj {:properties properties}
+ {:source (rename-keys (:source entity) {"key" "id"})}
+ {:target (rename-keys (:target entity) {"key" "id"})})))) error-logger audit-logger)))
diff --git a/src/chameleon/config.clj b/src/chameleon/config.clj
index bd99096..10324e2 100644
--- a/src/chameleon/config.clj
+++ b/src/chameleon/config.clj
@@ -4,14 +4,16 @@
(defn config
[app-config]
- (let [conf {
+ (let [conf {:chameleon/loggers (:log-config app-config)
:chameleon/event
{:event-config (assoc-in (:event-config app-config)
[:aai :processor] from-spike)
- :gallifrey-host (:gallifrey-host app-config)}
+ :gallifrey-host (:gallifrey-host app-config)
+ :loggers (ig/ref :chameleon/loggers)}
:chameleon/handler
{:gallifrey-host (:gallifrey-host app-config)
- :gallifrey-transformer from-gallifrey}
+ :gallifrey-transformer from-gallifrey
+ :loggers (ig/ref :chameleon/loggers)}
:chameleon/aai-processor
{:provenance-attr "last-mod-source-of-truth"
:truth-attr "truth-time"}
diff --git a/src/chameleon/event.clj b/src/chameleon/event.clj
index c4bec8e..92f4211 100644
--- a/src/chameleon/event.clj
+++ b/src/chameleon/event.clj
@@ -1,19 +1,25 @@
(ns chameleon.event
(:require [integrant.core :as ig]
- [clojure.string :refer [starts-with?]])
+ [clojure.string :refer [starts-with?]]
+ [chameleon.logging :as log])
(:import [org.onap.aai.event.client DMaaPEventConsumer]))
(defmethod ig/init-key :chameleon/event
- [_ {:keys [event-config gallifrey-host]}]
+ [_ {:keys [event-config gallifrey-host loggers]}]
(let [{:keys [host topic motsid pass consumer-group consumer-id timeout batch-size type processor]} (:aai event-config)
+ [error-logger audit-logger] loggers
event-processor (DMaaPEventConsumer. host topic motsid pass consumer-group consumer-id timeout batch-size type)]
- (println "Event processor for AAI created. Starting event polling on " host topic)
+ (log/info error-logger "EVENT_PROCESSOR" [(format "AAI created. Starting event polling on %s %s" host topic) ])
(.start (Thread. (fn [] (while true
(let [it (.iterator (.consume event-processor))]
- (println "Polling...")
+ (log/info error-logger "EVENT_PROCESSOR" ["Polling ..."])
(while (.hasNext it)
+ (log/mdc-init! "SPIKE-EVENT" "CHAMELEON" "" "" gallifrey-host)
(try (let [event (.next it)]
- (if (not (starts-with? event "DMAAP")) ;Temporarily added for current version of dmaap client
- (processor gallifrey-host event)))
- (catch Exception e (println (str "Unexpected exception during processing: " (.getMessage e)))))))))))
- ))
+ ;;Temporarily added for current version of dmaap client
+ (when-not (starts-with? event "DMAAP")
+ (log/info error-logger "EVENT_PROCESSOR" [event])
+ (processor gallifrey-host event error-logger audit-logger)
+ (log/mdc-clear!)))
+ (catch Exception e
+ (println (str "Unexpected exception during processing: " (.getMessage e)))))))))))))
diff --git a/src/chameleon/handler.clj b/src/chameleon/handler.clj
index fd97bf1..675a34b 100644
--- a/src/chameleon/handler.clj
+++ b/src/chameleon/handler.clj
@@ -10,17 +10,18 @@
[ring.middleware.session :refer [wrap-session]]
[cheshire.core :as json]
[clj-time.format :as tf]
- [integrant.core :as ig]))
+ [integrant.core :as ig]
+ [chameleon.logging :as log]))
(declare handler)
(defonce ^:private g-host (atom nil))
(defonce ^:private g-transformer nil)
-(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host gallifrey-transformer]}]
+(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host loggers gallifrey-transformer]}]
(reset! g-host gallifrey-host)
(def g-transformer gallifrey-transformer)
- handler)
+ (handler loggers))
(defmethod ig/halt-key! :chameleon/handler [_ _]
(reset! g-host nil)
@@ -32,10 +33,10 @@
:allowed-methods [:get]
:available-media-types ["application/json"]
:exists? (fn [ctx]
- (let [resource (c-route/query @g-host id type (-> ctx
- :request
- :params
- (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys
+ (let [resource (c-route/query @g-host id type (-> ctx
+ :request
+ :params
+ (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys
(when (= (:status resource) 200)
{::resource (-> resource
:body
@@ -56,9 +57,27 @@
(GET "/relationship/:id" [id] (resource-endpoint "relationship" id))
(resources "/"))
-(def handler
+(defn log-reqs
+ [handler loggers]
+ (let [[error-logger audit-logger] loggers]
+ (fn [request]
+ (log/mdc-init! (get-in request [:headers "X-TransactionId"]) "CHAMELEON"
+ "CHAMELEON_SERVICE" "ONAP" (:remote-addr request))
+ (log/info error-logger "CHAMELEON_REQUEST" (mapv str ((juxt (comp name :request-method) :uri :remote-addr) request)))
+ (let [resp (handler request)
+ fields (->> ((juxt :status :body) resp)
+ (into ((juxt (comp name :request-method) :uri) request))
+ (mapv str))]
+ (log/info error-logger "RESPONSE" fields)
+ (log/info audit-logger "RESPONSE" fields)
+ (log/mdc-clear!)
+ resp))))
+
+(defn handler
+ [loggers]
(-> app-routes
- (wrap-defaults api-defaults)))
+ (wrap-defaults api-defaults)
+ (log-reqs loggers)))
;;; Implementation
diff --git a/src/chameleon/logging.clj b/src/chameleon/logging.clj
new file mode 100644
index 0000000..15e1e7b
--- /dev/null
+++ b/src/chameleon/logging.clj
@@ -0,0 +1,123 @@
+(ns chameleon.logging
+ (:require [camel-snake-kebab.core :as cs]
+ [camel-snake-kebab.extras :refer [transform-keys]]
+ [clojure.java.io :as io]
+ [integrant.core :as ig]
+ [clojure.spec.alpha :as s])
+ (:import [org.onap.aai.cl.api Logger LogFields LogLine]
+ [org.onap.aai.cl.eelf LoggerFactory LogMessageEnum AaiLoggerAdapter AuditLogLine]
+ [org.onap.aai.cl.mdc MdcContext MdcOverride]
+ [com.att.eelf.i18n EELFResourceManager]
+ [clojure.lang.IFn]
+ [org.slf4j MDC]))
+
+(declare LOGLINE_DEFINED_FIELDS error-logger audit-logger ->java string->enum logfields)
+
+(defmethod ig/init-key :chameleon/loggers
+ [_ {:keys [logback logmsgs] :or {logback (.getPath (io/file "chameleon_logback.xml"))
+ logmsgs "log/ChameleonMsgs"}}]
+ (System/setProperty "logback.configurationFile" logback)
+ (EELFResourceManager/loadMessageBundle logmsgs)
+ [(error-logger "chameleon.loggging") (audit-logger "chameleon.loggging")])
+
+(defn conform-multiple
+ [& spec-form-pair]
+ (if (s/valid? :chameleon.specs/spec-form-pair spec-form-pair)
+ (->> spec-form-pair
+ (partition 2)
+ (map (fn [[sp form]]
+ (when (s/invalid? (s/conform sp form))
+ (s/explain-data sp form))))
+ (remove nil?))
+ (s/explain-data :chameleon.specs/spec-form-pair spec-form-pair)))
+
+(defn mdc-set!
+ "Sets the global MDC context for the current thread."
+ [m]
+ (doseq [[k v] m] (MDC/put k v)))
+
+(defn mdc-init!
+ "Sets the global MDC context as required by the EELF logging library"
+ [transaction-id service instance partner client-address]
+ (MdcContext/initialize transaction-id service instance partner client-address))
+
+(defn mdc-clear! [] (MDC/clear))
+
+(defmacro with-mdc
+ "Will set the global MDC context with the options (will convert from
+ keywords to PascalCase), execute the log, and clear the MDC
+ context."
+ [opts & body]
+ `(do
+ (mdc-set! (transform-keys cs/->PascalCaseString ~opts))
+ ~@body
+ (mdc-clear)))
+
+(defn mdc-override
+ [m]
+ (->java (fn [j k v] (.addAttribute j k v)) (new MdcOverride)
+ (transform-keys cs/->PascalCaseString m)))
+
+(defn error-logger
+ [name]
+ (.getLogger (LoggerFactory/getInstance) name))
+
+(defn audit-logger
+ [name]
+ (.getAuditLogger (LoggerFactory/getInstance) name))
+
+(defn logger?
+ [logger]
+ (instance? AaiLoggerAdapter logger))
+
+(defn info
+ [^AaiLoggerAdapter logger ^String enum msgs & {:keys [fields] :or {fields {}}}]
+ (let [confirmed-specs (conform-multiple :logging/valid-fields fields :logging/msgs msgs
+ :chameleon.specs/logger logger)]
+ (if (empty? confirmed-specs)
+ (.info logger (string->enum enum) (logfields fields) (into-array java.lang.String msgs))
+ confirmed-specs)))
+
+(defn debug
+ [^AaiLoggerAdapter logger ^String enum msgs]
+ (.debug logger (string->enum enum) (into-array java.lang.String msgs)))
+
+(defn valid-logfields?
+ [m]
+ (->> (keys m)
+ (map cs/->SCREAMING_SNAKE_CASE_STRING)
+ set
+ (clojure.set/superset? (-> LOGLINE_DEFINED_FIELDS keys set))))
+
+(def ^{:private true
+ :doc "Adding these fields from \"org.onap.aai.cl.api.LogLine\"
+ class. Right now there isn't a known way to use the ENUMs of an
+ abstract JAVA class if they are not STATIC. The field order is very
+ specific and it should be maintained for the common logging library
+ version \"1.2.2\". For a better understanding, please look at the
+ \"DefinedFields\" ENUMs in the org.onap.aai.cl.api.LogLine
+ class"}
+
+ LOGLINE_DEFINED_FIELDS
+ (zipmap ["STATUS_CODE" "RESPONSE_CODE" "RESPONSE_DESCRIPTION" "INSTANCE_UUID"
+ "SEVERITY" "SERVER_IP" "CLIENT_IP" "CLASS_NAME" "PROCESS_KEY"
+ "TARGET_SVC_NAME" "TARGET_ENTITY" "ERROR_CODE" "ERROR_DESCRIPTION"
+ "CUSTOM_1" "CUSTOM_2""CUSTOM_3" "CUSTOM_4"]
+ (range)))
+
+(defn- string->enum
+ ([^String enum] (proxy [Enum LogMessageEnum] [enum (hash enum)]))
+ ([^String enum ordinal] (proxy [Enum LogMessageEnum] [enum ordinal])))
+
+(defn- ->java
+ [f jc m]
+ (reduce (fn [a [k v]] (f a k v) a)
+ jc m))
+
+(defn- logfields
+ "Generate a \"LogFields\" object with all the fields set."
+ [m]
+ (->> m
+ (transform-keys cs/->SCREAMING_SNAKE_CASE_STRING)
+ (transform-keys #(string->enum % (LOGLINE_DEFINED_FIELDS %)))
+ (->java (fn [j field v] (.setField j field v)) (new LogFields))))
diff --git a/src/chameleon/route.clj b/src/chameleon/route.clj
index e8f3a3c..0918da4 100644
--- a/src/chameleon/route.clj
+++ b/src/chameleon/route.clj
@@ -1,12 +1,7 @@
(ns chameleon.route
- (:require [org.httpkit.client :as kitclient]))
-
-(defn- interpret-response
- "Print out the response from the Gallifrey server"
- [key response]
- (let [{:keys [status body]}@response]
- (println "Response for request with key " key " resulted in status " status
- " with body " body )))
+ (:require [org.httpkit.client :as kitclient]
+ [chameleon.logging :as log]
+ [ring.util.http-status :as hs]))
(defn query
"Retrieve an entity referenced by id at the provided host. Optionally provide
@@ -21,11 +16,10 @@
:keepalive 300
:timeout 20000}))
-(defn assert-create
+(defn assert-create!
"Creates an entity in Gallifrey with an initial set of assertions coming from the provided payload"
[host actor type key payload & [time-dimensions]]
- (kitclient/request {
- :url (str "https://" host "/" type "/" key)
+ (kitclient/request {:url (str "https://" host "/" type "/" key)
:method :put
:query-params (into {"actor" actor "create" "true"} time-dimensions)
:body payload
@@ -33,11 +27,10 @@
:keepalive 300
:timeout 1000}))
-(defn assert-update
+(defn assert-update!
"Update an entity in Gallifrey with a set of assertions coming from the provided payload"
[host actor type key payload & [time-dimensions]]
- (kitclient/request {
- :url (str "https://" host "/" type "/" key)
+ (kitclient/request {:url (str "https://" host "/" type "/" key)
:method :put
:query-params (into {"actor" actor "changes-only" "true"} time-dimensions)
:body payload
@@ -45,26 +38,27 @@
:keepalive 300
:timeout 1000}))
-(defn assert-delete
+(defn assert-delete!
"Assert a deletion for an entity in Gallifrey based on the provided key."
[host actor type key & [time-dimensions]]
- (kitclient/request {
- :url (str "https://" host "/" type "/" key)
+ (kitclient/request {:url (str "https://" host "/" type "/" key)
:method :delete
- :query-params (into {"actor" actor} time-dimensions)
+ :query-params (into {"actor" actor} time-dimensions)
:insecure? true
:keepalive 300
:timeout 1000}))
-(defn assert-gallifrey [host actor type payload]
+(defn assert-gallifrey!
+ [host actor type payload & [e-logger a-logger]]
"Propagates an assertion to Gallifrey based off of an event payload coming in from the event service."
(let [{:keys [meta body]} payload
- {:keys [key operation time]} meta]
- (println operation " " type " with key " key)
- (interpret-response key (case operation
- "CREATE"
- (assert-create host actor type key body time)
- "UPDATE"
- (assert-update host actor type key body time)
- "DELETE"
- (assert-delete host actor type key time)))))
+ {:keys [key operation time]} meta
+ _ (log/info e-logger "GALLIFREY_ASSERTION" (mapv str [operation type key]))
+ g-assert (case operation
+ "CREATE" (assert-create! host actor type key body time)
+ "UPDATE" (assert-update! host actor type key body time)
+ "DELETE" (assert-delete! host actor type key time))
+ {:keys [status body]} @g-assert]
+ (log/info e-logger "GALLIFREY_ASSERTED" [(str type) (str key)])
+ (log/info a-logger "RESPONSE" (mapv str [operation key status body])
+ :fields {:response-code status :response-description (hs/get-name status)})))
diff --git a/src/chameleon/specs.clj b/src/chameleon/specs.clj
new file mode 100644
index 0000000..40d5768
--- /dev/null
+++ b/src/chameleon/specs.clj
@@ -0,0 +1,71 @@
+(ns chameleon.specs
+ (:require [clojure.spec.alpha :as s]
+ [chameleon.logging :as log]
+ [clojure.spec.gen.alpha :as gen]
+ [cheshire.core :as c]
+ [clojure.string :as str]))
+
+(s/def ::host string?)
+(s/def ::provenance string?)
+(s/def ::payload map?)
+(s/def ::string (s/spec (s/and string? (complement str/blank?)) :gen gen/string-alphanumeric))
+(s/def ::type (s/spec ::string :gen #(gen/elements ["vserver" "pserver" "generic-vnf"])))
+(s/def ::id uuid?)
+(s/def ::source (s/keys :req-un [::type ::id]))
+(s/def ::target (s/keys :req-un [::type ::id]))
+(s/def ::_id ::id)
+(s/def ::spec-form-pair (s/* (s/cat :spec #(% (s/registry)) :form any?)))
+
+;; spike event
+(s/def :spike/transaction-id uuid?)
+(s/def :spike/schema-version (s/spec #(re-matches #"v\d+" %) :gen #(->> (s/int-in 7 15)
+ s/gen
+ (gen/fmap (partial str "v")))))
+(s/def :spike/key uuid?)
+(s/def :spike/name ::string)
+(s/def :spike/last-mod-source-of-truth ::string)
+(s/def :spike/source-of-truth ::string)
+(s/def :spike/vserver-selflink ::string)
+(s/def :spike/is-closed-loop-disabled boolean?)
+(s/def :spike/in-maint boolean?)
+(s/def :spike/timestamp inst?)
+(s/def :spike/operation (s/spec ::string :gen #(gen/elements ["UPDATE" "CREATE" "DELETE"])))
+(s/def :spike/properties (s/keys :req-un [:spike/name ::id :spike/last-mod-source-of-truth
+ :spike/source-of-truth :spike/vserver-selflink
+ :spike/is-closed-loop-disabled :spike/in-maint]))
+(s/def :spike/vertex (s/keys :req-un [:spike/schema-version ::type :spike/key :spike/properties]))
+(s/def :spike/event (s/keys :req-un [:spike/transaction-id :spike/vertex
+ :spike/operation :spike/timestamp]))
+(s/def :spike/payload (s/spec string? :gen #(gen/fmap (partial c/generate-string)
+ (s/gen :spike/event))))
+(s/def :gallifrey/k-end-actor ::string)
+
+;; gallifrey response
+(s/def :gallifrey/k-start-actor ::string)
+(s/def :gallifrey/k-end inst?)
+(s/def :gallifrey/k-start inst?)
+(s/def :gallifrey/history (s/keys :req-un [:gallifrey/k-end-actor :gallifrey/k-end
+ :gallifrey/k-start-actor :gallifrey/k-start]))
+(s/def :relationship/_meta (s/map-of ::string :gallifrey/history))
+(s/def :relationship/_type (s/spec string? :gen #(gen/return "relationship")))
+(s/def :gallifrey/_type (s/spec ::string :gen #(gen/return "entity")))
+(s/def :gallifrey/properties (s/keys :req-un [:gallifrey/_type ::type]))
+(s/def :relationship/type (s/spec string? :gen #(->> (gen/string-alphanumeric)
+ (gen/such-that (complement str/blank?))
+ (gen/fmap (partial str "tosca.relationship.")))))
+(s/def :relationship/properties (s/keys :req-un [:relationship/_type :relationship/type]))
+(s/def ::relationship (s/keys :req-un [:relationship/properties ::source
+ ::target :relationship/_meta ::_id]))
+(s/def ::relationships (s/coll-of ::relationship :gen-max 8))
+(s/def :gallifrey/payload (s/spec map?
+ :gen #(->> [::_id :gallifrey/properties
+ :gallifrey/properties ::relationships]
+ (s/keys :req-un)
+ s/gen
+ (gen/fmap (partial clojure.walk/stringify-keys)))))
+
+;; Logger specs
+(s/def ::logger (s/spec log/logger? :gen #(gen/return (log/error-logger "chameleon.specs"))))
+(s/def ::loggers (s/cat :e :chameleon.specs/logger :a :chameleon.specs/logger))
+(s/def :logging/msgs (s/* ::string))
+(s/def :logging/valid-fields log/valid-logfields?)