diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/chameleon/aai_processor.clj | 57 | ||||
| -rw-r--r-- | src/chameleon/config.clj | 8 | ||||
| -rw-r--r-- | src/chameleon/event.clj | 22 | ||||
| -rw-r--r-- | src/chameleon/handler.clj | 37 | ||||
| -rw-r--r-- | src/chameleon/logging.clj | 123 | ||||
| -rw-r--r-- | src/chameleon/route.clj | 50 | ||||
| -rw-r--r-- | src/chameleon/specs.clj | 71 |
7 files changed, 288 insertions, 80 deletions
diff --git a/src/chameleon/aai_processor.clj b/src/chameleon/aai_processor.clj index d1a7d25..4243b39 100644 --- a/src/chameleon/aai_processor.clj +++ b/src/chameleon/aai_processor.clj @@ -1,9 +1,8 @@ (ns chameleon.aai-processor - (:require - [chameleon.route :refer :all] - [cheshire.core :refer :all] - [integrant.core :as ig] - [clojure.set :refer :all])) + (:require [chameleon.route :refer :all] + [cheshire.core :refer :all] + [integrant.core :as ig] + [clojure.set :refer :all])) (defonce ^:private p-attr (atom nil)) (defonce ^:private t-attr (atom nil)) @@ -33,8 +32,7 @@ {"id" id "type" type "source" {"id" src-id "type" src-type} - "target" {"id" target-id "type" target-type}}) - ) + "target" {"id" target-id "type" target-type}})) (defn from-gallifrey "Transforms Gallifrey response payloads into a format consumable by AAI-centric clients" @@ -42,27 +40,21 @@ (let [resource-type (get-in body ["properties" "_type"]) id (body "_id") type (get-in body ["properties" "type"]) - properties (body "properties")] + properties (body "properties") + entity-response {"id" id + "type" type + "properties" (dissoc properties "_type" "type")}] (if (= resource-type "entity") - ; Transform into an entity type + ;; Transform into an entity type (let [relationships (body "relationships")] - { - "id" id - "type" type - "properties" (dissoc properties "_type" "type") - "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships))) - "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships))) - }) - ; Transform into a relationship type - { - "id" id - "type" type - "properties" (dissoc properties "_type" "type") - }))) + (assoc entity-response + "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships))) + "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships))))) + entity-response))) (defn from-spike "Transforms Spike-based event payloads to a format accepted by Gallifrey for vertices and relationships" - [gallifrey-host payload] + [gallifrey-host payload & [error-logger audit-logger]] (let [txpayload (map-keywords (parse-string payload)) operation (:operation txpayload) parse-type (if (contains? txpayload :vertex) @@ -74,13 +66,14 @@ entity (map-keywords (parse-type txpayload)) key (:key entity) properties (assoc (:properties entity) :type (:type entity)) - truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)}) - assertion {:meta {:key key - :operation operation - :time truth-time}} + truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)}) + assertion {:meta {:key key + :operation operation + :time truth-time}} provenance (get properties @p-attr "aai")] - (assert-gallifrey gallifrey-host provenance (name entity-type) (if (= entity-type :entity) - (assoc assertion :body (generate-string {:properties properties})) - (assoc assertion :body (generate-string (conj {:properties properties} - {:source (rename-keys (:source entity) {"key" "id"})} - {:target (rename-keys (:target entity) {"key" "id"})}))))))) + (assert-gallifrey! gallifrey-host provenance (name entity-type) + (if (= entity-type :entity) + (assoc assertion :body (generate-string {:properties properties})) + (assoc assertion :body (generate-string (conj {:properties properties} + {:source (rename-keys (:source entity) {"key" "id"})} + {:target (rename-keys (:target entity) {"key" "id"})})))) error-logger audit-logger))) diff --git a/src/chameleon/config.clj b/src/chameleon/config.clj index bd99096..10324e2 100644 --- a/src/chameleon/config.clj +++ b/src/chameleon/config.clj @@ -4,14 +4,16 @@ (defn config [app-config] - (let [conf { + (let [conf {:chameleon/loggers (:log-config app-config) :chameleon/event {:event-config (assoc-in (:event-config app-config) [:aai :processor] from-spike) - :gallifrey-host (:gallifrey-host app-config)} + :gallifrey-host (:gallifrey-host app-config) + :loggers (ig/ref :chameleon/loggers)} :chameleon/handler {:gallifrey-host (:gallifrey-host app-config) - :gallifrey-transformer from-gallifrey} + :gallifrey-transformer from-gallifrey + :loggers (ig/ref :chameleon/loggers)} :chameleon/aai-processor {:provenance-attr "last-mod-source-of-truth" :truth-attr "truth-time"} diff --git a/src/chameleon/event.clj b/src/chameleon/event.clj index c4bec8e..92f4211 100644 --- a/src/chameleon/event.clj +++ b/src/chameleon/event.clj @@ -1,19 +1,25 @@ (ns chameleon.event (:require [integrant.core :as ig] - [clojure.string :refer [starts-with?]]) + [clojure.string :refer [starts-with?]] + [chameleon.logging :as log]) (:import [org.onap.aai.event.client DMaaPEventConsumer])) (defmethod ig/init-key :chameleon/event - [_ {:keys [event-config gallifrey-host]}] + [_ {:keys [event-config gallifrey-host loggers]}] (let [{:keys [host topic motsid pass consumer-group consumer-id timeout batch-size type processor]} (:aai event-config) + [error-logger audit-logger] loggers event-processor (DMaaPEventConsumer. host topic motsid pass consumer-group consumer-id timeout batch-size type)] - (println "Event processor for AAI created. Starting event polling on " host topic) + (log/info error-logger "EVENT_PROCESSOR" [(format "AAI created. Starting event polling on %s %s" host topic) ]) (.start (Thread. (fn [] (while true (let [it (.iterator (.consume event-processor))] - (println "Polling...") + (log/info error-logger "EVENT_PROCESSOR" ["Polling ..."]) (while (.hasNext it) + (log/mdc-init! "SPIKE-EVENT" "CHAMELEON" "" "" gallifrey-host) (try (let [event (.next it)] - (if (not (starts-with? event "DMAAP")) ;Temporarily added for current version of dmaap client - (processor gallifrey-host event))) - (catch Exception e (println (str "Unexpected exception during processing: " (.getMessage e))))))))))) - )) + ;;Temporarily added for current version of dmaap client + (when-not (starts-with? event "DMAAP") + (log/info error-logger "EVENT_PROCESSOR" [event]) + (processor gallifrey-host event error-logger audit-logger) + (log/mdc-clear!))) + (catch Exception e + (println (str "Unexpected exception during processing: " (.getMessage e))))))))))))) diff --git a/src/chameleon/handler.clj b/src/chameleon/handler.clj index fd97bf1..675a34b 100644 --- a/src/chameleon/handler.clj +++ b/src/chameleon/handler.clj @@ -10,17 +10,18 @@ [ring.middleware.session :refer [wrap-session]] [cheshire.core :as json] [clj-time.format :as tf] - [integrant.core :as ig])) + [integrant.core :as ig] + [chameleon.logging :as log])) (declare handler) (defonce ^:private g-host (atom nil)) (defonce ^:private g-transformer nil) -(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host gallifrey-transformer]}] +(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host loggers gallifrey-transformer]}] (reset! g-host gallifrey-host) (def g-transformer gallifrey-transformer) - handler) + (handler loggers)) (defmethod ig/halt-key! :chameleon/handler [_ _] (reset! g-host nil) @@ -32,10 +33,10 @@ :allowed-methods [:get] :available-media-types ["application/json"] :exists? (fn [ctx] - (let [resource (c-route/query @g-host id type (-> ctx - :request - :params - (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys + (let [resource (c-route/query @g-host id type (-> ctx + :request + :params + (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys (when (= (:status resource) 200) {::resource (-> resource :body @@ -56,9 +57,27 @@ (GET "/relationship/:id" [id] (resource-endpoint "relationship" id)) (resources "/")) -(def handler +(defn log-reqs + [handler loggers] + (let [[error-logger audit-logger] loggers] + (fn [request] + (log/mdc-init! (get-in request [:headers "X-TransactionId"]) "CHAMELEON" + "CHAMELEON_SERVICE" "ONAP" (:remote-addr request)) + (log/info error-logger "CHAMELEON_REQUEST" (mapv str ((juxt (comp name :request-method) :uri :remote-addr) request))) + (let [resp (handler request) + fields (->> ((juxt :status :body) resp) + (into ((juxt (comp name :request-method) :uri) request)) + (mapv str))] + (log/info error-logger "RESPONSE" fields) + (log/info audit-logger "RESPONSE" fields) + (log/mdc-clear!) + resp)))) + +(defn handler + [loggers] (-> app-routes - (wrap-defaults api-defaults))) + (wrap-defaults api-defaults) + (log-reqs loggers))) ;;; Implementation diff --git a/src/chameleon/logging.clj b/src/chameleon/logging.clj new file mode 100644 index 0000000..15e1e7b --- /dev/null +++ b/src/chameleon/logging.clj @@ -0,0 +1,123 @@ +(ns chameleon.logging + (:require [camel-snake-kebab.core :as cs] + [camel-snake-kebab.extras :refer [transform-keys]] + [clojure.java.io :as io] + [integrant.core :as ig] + [clojure.spec.alpha :as s]) + (:import [org.onap.aai.cl.api Logger LogFields LogLine] + [org.onap.aai.cl.eelf LoggerFactory LogMessageEnum AaiLoggerAdapter AuditLogLine] + [org.onap.aai.cl.mdc MdcContext MdcOverride] + [com.att.eelf.i18n EELFResourceManager] + [clojure.lang.IFn] + [org.slf4j MDC])) + +(declare LOGLINE_DEFINED_FIELDS error-logger audit-logger ->java string->enum logfields) + +(defmethod ig/init-key :chameleon/loggers + [_ {:keys [logback logmsgs] :or {logback (.getPath (io/file "chameleon_logback.xml")) + logmsgs "log/ChameleonMsgs"}}] + (System/setProperty "logback.configurationFile" logback) + (EELFResourceManager/loadMessageBundle logmsgs) + [(error-logger "chameleon.loggging") (audit-logger "chameleon.loggging")]) + +(defn conform-multiple + [& spec-form-pair] + (if (s/valid? :chameleon.specs/spec-form-pair spec-form-pair) + (->> spec-form-pair + (partition 2) + (map (fn [[sp form]] + (when (s/invalid? (s/conform sp form)) + (s/explain-data sp form)))) + (remove nil?)) + (s/explain-data :chameleon.specs/spec-form-pair spec-form-pair))) + +(defn mdc-set! + "Sets the global MDC context for the current thread." + [m] + (doseq [[k v] m] (MDC/put k v))) + +(defn mdc-init! + "Sets the global MDC context as required by the EELF logging library" + [transaction-id service instance partner client-address] + (MdcContext/initialize transaction-id service instance partner client-address)) + +(defn mdc-clear! [] (MDC/clear)) + +(defmacro with-mdc + "Will set the global MDC context with the options (will convert from + keywords to PascalCase), execute the log, and clear the MDC + context." + [opts & body] + `(do + (mdc-set! (transform-keys cs/->PascalCaseString ~opts)) + ~@body + (mdc-clear))) + +(defn mdc-override + [m] + (->java (fn [j k v] (.addAttribute j k v)) (new MdcOverride) + (transform-keys cs/->PascalCaseString m))) + +(defn error-logger + [name] + (.getLogger (LoggerFactory/getInstance) name)) + +(defn audit-logger + [name] + (.getAuditLogger (LoggerFactory/getInstance) name)) + +(defn logger? + [logger] + (instance? AaiLoggerAdapter logger)) + +(defn info + [^AaiLoggerAdapter logger ^String enum msgs & {:keys [fields] :or {fields {}}}] + (let [confirmed-specs (conform-multiple :logging/valid-fields fields :logging/msgs msgs + :chameleon.specs/logger logger)] + (if (empty? confirmed-specs) + (.info logger (string->enum enum) (logfields fields) (into-array java.lang.String msgs)) + confirmed-specs))) + +(defn debug + [^AaiLoggerAdapter logger ^String enum msgs] + (.debug logger (string->enum enum) (into-array java.lang.String msgs))) + +(defn valid-logfields? + [m] + (->> (keys m) + (map cs/->SCREAMING_SNAKE_CASE_STRING) + set + (clojure.set/superset? (-> LOGLINE_DEFINED_FIELDS keys set)))) + +(def ^{:private true + :doc "Adding these fields from \"org.onap.aai.cl.api.LogLine\" + class. Right now there isn't a known way to use the ENUMs of an + abstract JAVA class if they are not STATIC. The field order is very + specific and it should be maintained for the common logging library + version \"1.2.2\". For a better understanding, please look at the + \"DefinedFields\" ENUMs in the org.onap.aai.cl.api.LogLine + class"} + + LOGLINE_DEFINED_FIELDS + (zipmap ["STATUS_CODE" "RESPONSE_CODE" "RESPONSE_DESCRIPTION" "INSTANCE_UUID" + "SEVERITY" "SERVER_IP" "CLIENT_IP" "CLASS_NAME" "PROCESS_KEY" + "TARGET_SVC_NAME" "TARGET_ENTITY" "ERROR_CODE" "ERROR_DESCRIPTION" + "CUSTOM_1" "CUSTOM_2""CUSTOM_3" "CUSTOM_4"] + (range))) + +(defn- string->enum + ([^String enum] (proxy [Enum LogMessageEnum] [enum (hash enum)])) + ([^String enum ordinal] (proxy [Enum LogMessageEnum] [enum ordinal]))) + +(defn- ->java + [f jc m] + (reduce (fn [a [k v]] (f a k v) a) + jc m)) + +(defn- logfields + "Generate a \"LogFields\" object with all the fields set." + [m] + (->> m + (transform-keys cs/->SCREAMING_SNAKE_CASE_STRING) + (transform-keys #(string->enum % (LOGLINE_DEFINED_FIELDS %))) + (->java (fn [j field v] (.setField j field v)) (new LogFields)))) diff --git a/src/chameleon/route.clj b/src/chameleon/route.clj index e8f3a3c..0918da4 100644 --- a/src/chameleon/route.clj +++ b/src/chameleon/route.clj @@ -1,12 +1,7 @@ (ns chameleon.route - (:require [org.httpkit.client :as kitclient])) - -(defn- interpret-response - "Print out the response from the Gallifrey server" - [key response] - (let [{:keys [status body]}@response] - (println "Response for request with key " key " resulted in status " status - " with body " body ))) + (:require [org.httpkit.client :as kitclient] + [chameleon.logging :as log] + [ring.util.http-status :as hs])) (defn query "Retrieve an entity referenced by id at the provided host. Optionally provide @@ -21,11 +16,10 @@ :keepalive 300 :timeout 20000})) -(defn assert-create +(defn assert-create! "Creates an entity in Gallifrey with an initial set of assertions coming from the provided payload" [host actor type key payload & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :put :query-params (into {"actor" actor "create" "true"} time-dimensions) :body payload @@ -33,11 +27,10 @@ :keepalive 300 :timeout 1000})) -(defn assert-update +(defn assert-update! "Update an entity in Gallifrey with a set of assertions coming from the provided payload" [host actor type key payload & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :put :query-params (into {"actor" actor "changes-only" "true"} time-dimensions) :body payload @@ -45,26 +38,27 @@ :keepalive 300 :timeout 1000})) -(defn assert-delete +(defn assert-delete! "Assert a deletion for an entity in Gallifrey based on the provided key." [host actor type key & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :delete - :query-params (into {"actor" actor} time-dimensions) + :query-params (into {"actor" actor} time-dimensions) :insecure? true :keepalive 300 :timeout 1000})) -(defn assert-gallifrey [host actor type payload] +(defn assert-gallifrey! + [host actor type payload & [e-logger a-logger]] "Propagates an assertion to Gallifrey based off of an event payload coming in from the event service." (let [{:keys [meta body]} payload - {:keys [key operation time]} meta] - (println operation " " type " with key " key) - (interpret-response key (case operation - "CREATE" - (assert-create host actor type key body time) - "UPDATE" - (assert-update host actor type key body time) - "DELETE" - (assert-delete host actor type key time))))) + {:keys [key operation time]} meta + _ (log/info e-logger "GALLIFREY_ASSERTION" (mapv str [operation type key])) + g-assert (case operation + "CREATE" (assert-create! host actor type key body time) + "UPDATE" (assert-update! host actor type key body time) + "DELETE" (assert-delete! host actor type key time)) + {:keys [status body]} @g-assert] + (log/info e-logger "GALLIFREY_ASSERTED" [(str type) (str key)]) + (log/info a-logger "RESPONSE" (mapv str [operation key status body]) + :fields {:response-code status :response-description (hs/get-name status)}))) diff --git a/src/chameleon/specs.clj b/src/chameleon/specs.clj new file mode 100644 index 0000000..40d5768 --- /dev/null +++ b/src/chameleon/specs.clj @@ -0,0 +1,71 @@ +(ns chameleon.specs + (:require [clojure.spec.alpha :as s] + [chameleon.logging :as log] + [clojure.spec.gen.alpha :as gen] + [cheshire.core :as c] + [clojure.string :as str])) + +(s/def ::host string?) +(s/def ::provenance string?) +(s/def ::payload map?) +(s/def ::string (s/spec (s/and string? (complement str/blank?)) :gen gen/string-alphanumeric)) +(s/def ::type (s/spec ::string :gen #(gen/elements ["vserver" "pserver" "generic-vnf"]))) +(s/def ::id uuid?) +(s/def ::source (s/keys :req-un [::type ::id])) +(s/def ::target (s/keys :req-un [::type ::id])) +(s/def ::_id ::id) +(s/def ::spec-form-pair (s/* (s/cat :spec #(% (s/registry)) :form any?))) + +;; spike event +(s/def :spike/transaction-id uuid?) +(s/def :spike/schema-version (s/spec #(re-matches #"v\d+" %) :gen #(->> (s/int-in 7 15) + s/gen + (gen/fmap (partial str "v"))))) +(s/def :spike/key uuid?) +(s/def :spike/name ::string) +(s/def :spike/last-mod-source-of-truth ::string) +(s/def :spike/source-of-truth ::string) +(s/def :spike/vserver-selflink ::string) +(s/def :spike/is-closed-loop-disabled boolean?) +(s/def :spike/in-maint boolean?) +(s/def :spike/timestamp inst?) +(s/def :spike/operation (s/spec ::string :gen #(gen/elements ["UPDATE" "CREATE" "DELETE"]))) +(s/def :spike/properties (s/keys :req-un [:spike/name ::id :spike/last-mod-source-of-truth + :spike/source-of-truth :spike/vserver-selflink + :spike/is-closed-loop-disabled :spike/in-maint])) +(s/def :spike/vertex (s/keys :req-un [:spike/schema-version ::type :spike/key :spike/properties])) +(s/def :spike/event (s/keys :req-un [:spike/transaction-id :spike/vertex + :spike/operation :spike/timestamp])) +(s/def :spike/payload (s/spec string? :gen #(gen/fmap (partial c/generate-string) + (s/gen :spike/event)))) +(s/def :gallifrey/k-end-actor ::string) + +;; gallifrey response +(s/def :gallifrey/k-start-actor ::string) +(s/def :gallifrey/k-end inst?) +(s/def :gallifrey/k-start inst?) +(s/def :gallifrey/history (s/keys :req-un [:gallifrey/k-end-actor :gallifrey/k-end + :gallifrey/k-start-actor :gallifrey/k-start])) +(s/def :relationship/_meta (s/map-of ::string :gallifrey/history)) +(s/def :relationship/_type (s/spec string? :gen #(gen/return "relationship"))) +(s/def :gallifrey/_type (s/spec ::string :gen #(gen/return "entity"))) +(s/def :gallifrey/properties (s/keys :req-un [:gallifrey/_type ::type])) +(s/def :relationship/type (s/spec string? :gen #(->> (gen/string-alphanumeric) + (gen/such-that (complement str/blank?)) + (gen/fmap (partial str "tosca.relationship."))))) +(s/def :relationship/properties (s/keys :req-un [:relationship/_type :relationship/type])) +(s/def ::relationship (s/keys :req-un [:relationship/properties ::source + ::target :relationship/_meta ::_id])) +(s/def ::relationships (s/coll-of ::relationship :gen-max 8)) +(s/def :gallifrey/payload (s/spec map? + :gen #(->> [::_id :gallifrey/properties + :gallifrey/properties ::relationships] + (s/keys :req-un) + s/gen + (gen/fmap (partial clojure.walk/stringify-keys))))) + +;; Logger specs +(s/def ::logger (s/spec log/logger? :gen #(gen/return (log/error-logger "chameleon.specs")))) +(s/def ::loggers (s/cat :e :chameleon.specs/logger :a :chameleon.specs/logger)) +(s/def :logging/msgs (s/* ::string)) +(s/def :logging/valid-fields log/valid-logfields?) |