Initial Commit of Chameleon

Commit the initial set of code for the Chameleon offering to ONAP

Change-Id: Ia58bd49eafc0a3702c17c9cab34d666ed1627ba5
Issue-ID: AAI-797
Signed-off-by: Adrian Batos-Parac <adrian.batos-parac@amdocs.com>

24 files changed, 480 insertions, 0 deletions

diff --git a/.gitreview b/.gitreview
new file mode 100644
index 0000000..8fd4a05
--- /dev/null
+++ b/.gitreview
@@ -0,0 +1,4 @@
+[gerrit]
+host=gerrit.openecomp.org
+port=29418
+project=chameleon.git
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0895c05
--- /dev/null
+++ b/README.md
@@ -0,0 +1,3 @@
+# chameleon
+
+Feeds the Gallifrey time database with entity events
diff --git a/chameleon b/chameleon
new file mode 160000
+Subproject 2610df2fc29a9d16bb869f652d4d21caeae4c15
diff --git a/dev/dev.clj b/dev/dev.clj
new file mode 100644
index 0000000..880f540
--- /dev/null
+++ b/dev/dev.clj
@@ -0,0 +1,28 @@
+(ns dev
+  "Tools for interactive development with the REPL. This file should
+  not be included in a production build of the application."
+  (:require [chameleon.config :refer [config]]
+            [chameleon.handler :refer [handler]]
+            [integrant.core :as ig]
+            [integrant.repl :refer [clear go halt init reset reset-all]]
+            [integrant.repl.state :refer [system]]
+            [clojure.tools.namespace.repl :refer [refresh refresh-all disable-reload!]]
+            [clojure.repl :refer [apropos dir doc find-doc pst source]]
+            [clojure.test :refer [run-tests run-all-tests]]
+            [clojure.pprint :refer [pprint]]
+            [clojure.reflect :refer [reflect]]))
+
+(disable-reload! (find-ns 'integrant.core))
+
+(integrant.repl/set-prep! (constantly (config {
+                                               :event-config {:aai {:host "localhost:3904"
+                                                                    :topic "spikeEvents"
+                                                                    :motsid ""
+                                                                    :pass ""
+                                                                    :consumer-group"chameleon1"
+                                                                    :consumer-id"chameleon1"
+                                                                    :timeout 15000
+                                                                    :batch-size 8
+                                                                    :type "HTTPAUTH"}}
+                                               :gallifrey-host "localhost:443"
+                                               :http-port 3449})))
diff --git a/dev/user.clj b/dev/user.clj
new file mode 100644
index 0000000..9a2f701
--- /dev/null
+++ b/dev/user.clj
@@ -0,0 +1,7 @@
+(ns user)
+
+(defn dev
+  "Load and switch to the 'dev' namespace."
+  []
+  (require 'dev)
+  (in-ns 'dev))
diff --git a/devops/chameleon-config.edn b/devops/chameleon-config.edn
new file mode 100644
index 0000000..32e830e
--- /dev/null
+++ b/devops/chameleon-config.edn
@@ -0,0 +1,10 @@
+{:event-config {:host "localhost:3904"
+                :topic "spikeEvents"
+                :motsid ""
+                :pass ""
+                :consumer-group "chameleon"
+                :consumer-id "chameleon"
+                :timeout 15000
+                :batch-size 8
+                :type "HTTPAUTH"}
+ :gallifrey-host "localhost:443"}
diff --git a/devops/chameleon.service b/devops/chameleon.service
new file mode 100644
index 0000000..0c4b937
--- /dev/null
+++ b/devops/chameleon.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Chameleon container
+After=docker.socket early-docker.target network.target network-online.target
+Wants=network-online.target
+BindsTo=docker.service
+
+[Service]
+Restart=always
+ExecStart=/usr/bin/docker-compose -f /opt/chameleon/docker-compose.yml -p chameleon up
+ExecStop=/usr/bin/docker-compose -f /opt/chameleon/docker-compose.yml -p chameleon down
+
+[Install]
+WantedBy=multi-user.target
diff --git a/devops/chameleon/Dockerfile b/devops/chameleon/Dockerfile
new file mode 100644
index 0000000..97e9c3b
--- /dev/null
+++ b/devops/chameleon/Dockerfile
@@ -0,0 +1,45 @@
+FROM alpine:3.6
+
+# Java Version
+ENV JAVA_VERSION_MAJOR 8
+ENV JAVA_VERSION_MINOR 131
+ENV JAVA_VERSION_BUILD 11
+ENV JAVA_PACKAGE       jre
+ENV GLIBC_VERSION      2.25-r0
+
+ENV JAVA_8_BASE_URL http://download.oracle.com/otn-pub/java/jdk/${JAVA_VERSION_MAJOR}u${JAVA_VERSION_MINOR}-b${JAVA_VERSION_BUILD}/d54c1d3a095b4ff2b6607d096fa80163/${JAVA_PACKAGE}-${JAVA_VERSION_MAJOR}u${JAVA_VERSION_MINOR}
+
+RUN apk update
+RUN apk add curl
+
+# Install glibc (required for java)
+RUN curl -jsSL -o /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \
+    curl -jsSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk && \
+    apk add glibc-${GLIBC_VERSION}.apk && \
+    rm -f glibc-${GLIBC_VERSION}.apk
+
+# Install Java
+RUN mkdir /opt && \
+    curl -jsSL -H "Cookie: oraclelicense=accept-securebackup-cookie" ${JAVA_8_BASE_URL}-linux-x64.tar.gz | tar -xzf - -C /opt && \
+    ln -s /opt/${JAVA_PACKAGE}1.${JAVA_VERSION_MAJOR}.0_${JAVA_VERSION_MINOR} /opt/${JAVA_PACKAGE} && \
+    cd /opt/${JAVA_PACKAGE}/ && rm -rf *src.zip lib/missioncontrol lib/visualvm lib/plugin.jar plugin lib/*javafx* lib/*jfx* lib/ext/jfxrt.jar bin/javaws lib/javaws.jar lib/desktop lib/deploy* lib/amd64/libdecora_sse.so lib/amd64/libprism_*.so lib/amd64/libfxplugins.so lib/amd64/libglass.so lib/amd64/libgstreamer-lite.so lib/amd64/libjavafx*.so lib/amd64/libjfx*.
+
+# Install Java Cryptography Extension (JCE) Unlimited Strength
+RUN curl -jksSLH "Cookie: oraclelicense=accept-securebackup-cookie" -o /tmp/jce_policy.zip \
+    http://download.oracle.com/otn-pub/java/jce/${JAVA_VERSION_MAJOR}/jce_policy-${JAVA_VERSION_MAJOR}.zip && \
+    unzip -o -d /opt/${JAVA_PACKAGE}/lib/security /tmp/jce_policy.zip && rm -f /tmp/jce_policy.zip
+
+RUN apk del curl
+
+ENV PATH /opt/jre/bin:${PATH}
+
+RUN apk add zip openssh-keygen openssh-client
+
+EXPOSE 80
+
+RUN mkdir -p /opt/chameleon
+COPY chameleon.jar /opt/chameleon
+
+COPY ./docker-entrypoint.sh /
+RUN chmod 700 /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]
diff --git a/devops/chameleon/build-chameleon b/devops/chameleon/build-chameleon
new file mode 100644
index 0000000..4421407
--- /dev/null
+++ b/devops/chameleon/build-chameleon
@@ -0,0 +1,2 @@
+lein uberjar
+cp -f ../../target/chameleon.jar .
diff --git a/devops/chameleon/docker-entrypoint.sh b/devops/chameleon/docker-entrypoint.sh
new file mode 100644
index 0000000..4a0b529
--- /dev/null
+++ b/devops/chameleon/docker-entrypoint.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -e
+
+java -jar /opt/chameleon/chameleon.jar
diff --git a/devops/docker-compose.yml b/devops/docker-compose.yml
new file mode 100644
index 0000000..b17a98b
--- /dev/null
+++ b/devops/docker-compose.yml
@@ -0,0 +1,16 @@
+chameleon:
+  build: ./chameleon
+  container_name: chameleon
+  environment:
+    - CONFIG_LOCATION=/opt/chameleon/chameleon-config.edn
+  volumes:
+      - ./chameleon-config.edn:/opt/chameleon/chameleon-config.edn
+
+nginx:
+  build: ./nginx
+  container_name: nginx
+  links:
+    - chameleon
+  ports:
+    - 80:80
+    - 443:443
diff --git a/devops/install-service b/devops/install-service
new file mode 100644
index 0000000..a54b884
--- /dev/null
+++ b/devops/install-service
@@ -0,0 +1,6 @@
+sudo cp chameleon.service /etc/systemd/system
+sudo chown root:root /etc/systemd/system/chameleon.service
+sudo chmod 644 /etc/systemd/system/chameleon.service
+
+sudo systemctl enable chameleon
+sudo systemctl daemon-reload
diff --git a/devops/nginx/Dockerfile b/devops/nginx/Dockerfile
new file mode 100644
index 0000000..4f2ba9f
--- /dev/null
+++ b/devops/nginx/Dockerfile
@@ -0,0 +1,9 @@
+FROM nginx:alpine
+
+COPY ssl-cert-snakeoil.pem /etc/ssl/certs/
+COPY ssl-cert-snakeoil.key /etc/ssl/private/
+RUN chown -R nginx:nginx /etc/ssl
+RUN chmod 640 /etc/ssl/private/ssl-cert-snakeoil.key
+RUN chmod 750 /etc/ssl/private
+
+COPY default.conf /etc/nginx/conf.d/
diff --git a/devops/nginx/default.conf b/devops/nginx/default.conf
new file mode 100644
index 0000000..541f5db
--- /dev/null
+++ b/devops/nginx/default.conf
@@ -0,0 +1,23 @@
+
+server {
+# Listen on 80 and 443
+listen 80;
+listen 443 ssl;
+# Self-signed certificate.
+ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
+ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
+
+# Redirect all non-SSL traffic to SSL.
+if ($ssl_protocol = "") {
+rewrite ^ https://$host$request_uri? permanent;
+}
+
+# Split off traffic to chameleon, and make sure that websockets
+# are managed correctly.
+location / {
+proxy_pass http://chameleon:8082;
+proxy_http_version 1.1;
+proxy_set_header Upgrade websocket;
+proxy_set_header Connection upgrade;
+}
+}
diff --git a/devops/nginx/nginx.conf b/devops/nginx/nginx.conf
new file mode 100644
index 0000000..3ebc618
--- /dev/null
+++ b/devops/nginx/nginx.conf
@@ -0,0 +1,33 @@
+
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+    include /etc/nginx/sites-available/*.conf;
+}
diff --git a/prod/chameleon/server.clj b/prod/chameleon/server.clj
new file mode 100644
index 0000000..b9b38db
--- /dev/null
+++ b/prod/chameleon/server.clj
@@ -0,0 +1,18 @@
+(ns chameleon.server
+  (:require [chameleon.config :refer [config]]
+            [chameleon.handler :refer [handler]]
+            [config.core :refer [env]]
+            [org.httpkit.server :refer [run-server]]
+            [integrant.core :as ig])
+  (:gen-class))
+
+(defn -main [& args]
+  (let [port (Integer/parseInt (or (env :http-port) "8082"))
+        system-config (read-string (slurp (System/getenv "CONFIG_LOCATION" )))
+        event-config (:event-config system-config)
+        route-config (:gallifrey-host system-config)]
+    (println "Listening on port" port)
+    (ig/init (config {
+                      :event-config event-config
+                      :gallifrey-host route-config
+                      :http-port port}))))
diff --git a/project.clj b/project.clj
new file mode 100644
index 0000000..aaab854
--- /dev/null
+++ b/project.clj
@@ -0,0 +1,31 @@
+(defproject chameleon "0.1.0"
+  :dependencies [[org.clojure/clojure "1.8.0"]
+                 [com.7theta/utilis "1.0.4"]
+                 [http-kit "2.2.0"]
+                 [ring/ring-core "1.6.3"]
+                 [ring/ring-defaults "0.3.1"]
+                 [ring/ring-anti-forgery "1.1.0"]
+                 [compojure "1.6.0"]
+                 [liberator "0.15.1"]
+                 [cheshire "5.7.1"]
+                 [inflections "0.13.0"]
+                 [clj-time "0.14.2"]
+                 [integrant "0.6.2"]
+                 [clojure-future-spec "1.9.0-beta4"]
+                 [yogthos/config "0.9"]
+                 [org.onap.aai.event-client/event-client-dmaap "1.2.0"]
+                 ]
+  :repositories [["onap-releases" {:url "https://nexus.onap.org/content/repositories/releases/"}]
+                 ["onap-public" {:url "https://nexus.onap.org/content/repositories/public/"}]
+                 ["onap-staging" {:url "https://nexus.onap.org/content/repositories/staging/"}]
+                 ["onap-snapshot" {:url "https://nexus.onap.org/content/repositories/snapshots/"}]
+                 ]
+  :min-lein-version "2.5.3"
+  :profiles {:dev {:source-paths ["dev"]
+                   :dependencies [[ring/ring-devel "1.6.3"]
+                                  [integrant/repl "0.2.0"]]}
+             :uberjar {:source-paths ["prod"]
+                       :main chameleon.server
+                       :aot [chameleon.server]
+                       :uberjar-name "chameleon.jar"}}
+  :prep-tasks [ "compile"])
diff --git a/src/chameleon/aai_processor.clj b/src/chameleon/aai_processor.clj
new file mode 100644
index 0000000..c709ed1
--- /dev/null
+++ b/src/chameleon/aai_processor.clj
@@ -0,0 +1,26 @@
+(ns chameleon.aai-processor
+  (:require [chameleon.txform :refer :all]
+            [chameleon.route :refer :all]
+            [cheshire.core :refer :all]))
+
+(defn from-gallifrey
+  "Transforms Gallifrey response payloads into a format consumable by AAI-centric clients"
+  [body]
+  (->> body
+       (map (fn [[k v]] [(clojure.string/split k #"\.") v]))
+       ((fn [x] (reduce #(assoc-in %1 (first %2) (second %2) ) {} x)))))
+
+(defn from-spike
+  "Transforms Spike-based event payloads to a format accepted by Gallifrey for vertices and relationships"
+  [gallifrey-host payload]
+  (let [txpayload (map-keywords (parse-string payload))
+        operation (:operation txpayload)
+        entity-type (if (contains? txpayload :vertex)
+                      :vertex
+                      :relationship)
+        entity (map-keywords (entity-type txpayload))
+        key (:key entity)
+        properties (assoc (:properties entity) :type (:type entity))]
+    (assert-gallifrey gallifrey-host "aai" (if (= entity-type :vertex)
+                                             {:meta {:key key :operation operation} :body (generate-string properties)}
+                                             {:meta {:key key :operation operation} :body (generate-string (conj properties (flatten-entry entity :source) (flatten-entry entity :target)))}))))
diff --git a/src/chameleon/config.clj b/src/chameleon/config.clj
new file mode 100644
index 0000000..4982f4a
--- /dev/null
+++ b/src/chameleon/config.clj
@@ -0,0 +1,18 @@
+(ns chameleon.config
+  (:require [integrant.core :as ig]
+            [chameleon.aai-processor :refer :all]))
+
+(defn config
+  [app-config]
+  (let [conf {
+              :chameleon/event
+              {:event-config (assoc-in (:event-config app-config)
+                                       [:aai :processor] from-spike)
+               :gallifrey-host (:gallifrey-host app-config)}
+              :chameleon/handler
+              {:gallifrey-host (:gallifrey-host app-config)}
+              :chameleon/http-server
+              {:port (:http-port app-config)
+               :handler (ig/ref :chameleon/handler)}}]
+    (ig/load-namespaces conf)
+    conf))
diff --git a/src/chameleon/event.clj b/src/chameleon/event.clj
new file mode 100644
index 0000000..8201acb
--- /dev/null
+++ b/src/chameleon/event.clj
@@ -0,0 +1,18 @@
+(ns chameleon.event
+  (:require [chameleon.txform]
+            [chameleon.route]
+            [integrant.core :as ig])
+  (:import [org.onap.aai.event.client DMaaPEventConsumer]))
+
+(defmethod ig/init-key :chameleon/event
+  [_ {:keys [event-config gallifrey-host]}]
+  (let [{:keys [host topic motsid pass consumer-group consumer-id timeout batch-size type processor]} (:aai event-config)
+        event-processor (DMaaPEventConsumer. host topic motsid pass consumer-group consumer-id timeout batch-size type)]
+    (println "Event processor for AAI  created. Starting event polling on " host  topic)
+    (.start (Thread. (fn [] (while true
+                             (let  [it (.iterator (.consume event-processor))]
+                               (println "Polling...")
+                               (while (.hasNext it)
+                                 (let [event (.next it)]
+                                   (processor gallifrey-host event))))))))
+    ))
diff --git a/src/chameleon/handler.clj b/src/chameleon/handler.clj
new file mode 100644
index 0000000..7a4a5fe
--- /dev/null
+++ b/src/chameleon/handler.clj
@@ -0,0 +1,61 @@
+(ns chameleon.handler
+  (:require [chameleon.route :as c-route]
+            [chameleon.aai-processor]
+            [utilis.map :refer [map-vals compact]]
+            [liberator.core :refer [defresource]]
+            [compojure.core :refer [GET PUT PATCH ANY defroutes]]
+            [compojure.route :refer [resources]]
+            [ring.util.response :refer [resource-response content-type]]
+            [ring.middleware.defaults :refer [wrap-defaults api-defaults]]
+            [ring.middleware.anti-forgery :refer [wrap-anti-forgery]]
+            [ring.middleware.session :refer [wrap-session]]
+            [cheshire.core :as json]
+            [clj-time.format :as tf]
+            [integrant.core :as ig]))
+
+(declare handler)
+
+(defonce ^:private g-host (atom nil))
+
+(defmethod ig/init-key :chameleon/handler  [_ {:keys [gallifrey-host]}]
+  (reset! g-host gallifrey-host)
+  handler)
+
+(defmethod ig/halt-key! :chameleon/handler  [_ _]
+  (reset! g-host nil))
+
+(declare serialize de-serialize)
+
+(defresource entity-endpoint [id]
+  :allowed-methods [:get]
+  :available-media-types ["application/json"]
+  :exists? (fn [ctx]
+             (let [resource (-> (c-route/query @g-host id (-> ctx :request :params :t-k)))]
+               (when (= (:status resource) 200)
+                 {::resource (-> resource :body json/parse-string (dissoc "_meta") (chameleon.aai-processor/from-gallifrey))})))
+  :existed? (fn [ctx]
+              (when-let [status (-> (c-route/query @g-host id (-> ctx :request :params :t-k)) :status)]
+                (= status 410)))
+  :handle-ok ::resource)
+
+(defroutes app-routes
+  (GET "/entity/:id" [id] (entity-endpoint id))
+  (resources "/"))
+
+(def handler
+  (-> app-routes
+      (wrap-defaults api-defaults)))
+
+
+;;; Implementation
+
+(defn- serialize
+  [e]
+  (compact
+   (update e :_meta #(map-vals
+                      (fn [m]
+                        (map-vals str m)) %))))
+
+(defn- de-serialize
+  [e]
+  e)
diff --git a/src/chameleon/http_server.clj b/src/chameleon/http_server.clj
new file mode 100644
index 0000000..cf8ae07
--- /dev/null
+++ b/src/chameleon/http_server.clj
@@ -0,0 +1,9 @@
+(ns chameleon.http-server
+  (:require [org.httpkit.server :refer [run-server]]
+            [integrant.core :as ig]))
+
+(defmethod ig/init-key :chameleon/http-server  [_ {:keys [port handler]}]
+  (run-server handler {:port port}))
+
+(defmethod ig/halt-key! :chameleon/http-server  [_ server]
+  (server :timeout 100))
diff --git a/src/chameleon/route.clj b/src/chameleon/route.clj
new file mode 100644
index 0000000..2b30f42
--- /dev/null
+++ b/src/chameleon/route.clj
@@ -0,0 +1,71 @@
+(ns chameleon.route
+  (:require [org.httpkit.client :as kitclient]))
+
+(defn- interpret-response
+  "Print out the response from the Gallifrey server"
+  [key response]
+  (let [{:keys [status body]}@response]
+    (println "Response for request with key " key " resulted in status " status
+             " with body " body )))
+
+(defn query
+  "Retrieve an entity referenced by id at the provided host. Optionally provide
+  a time 't-k' that defines a query based on when the system knew about the
+  state of the entity."
+  [host key & [k]]
+  @(kitclient/request {
+                       :url (str "https://" host "/entity/" key)
+                       :method :get
+                       :query-params (if-let [t-k k] {"t-k" t-k})
+                       :insecure? true
+                       :keepalive 300
+                       :timeout 1000}))
+
+(defn assert-create
+  "Creates an entity in Gallifrey with an initial set of assertions coming from the provided payload"
+  [host actor key payload]
+  (print "Final: " payload)
+  (kitclient/request {
+                      :url (str "https://" host "/entity/" key)
+                      :method :put
+                      :query-params {"actor" actor "create" "true"}
+                      :body payload
+                      :insecure? true
+                      :keepalive 300
+                      :timeout 1000}))
+
+(defn assert-update
+  "Update an entity in Gallifrey with a set of assertions coming from the provided payload"
+  [host actor key payload]
+  (kitclient/request {
+                      :url (str "https://" host "/entity/" key)
+                      :method :put
+                      :query-params {"actor" actor "changes-only" "true"}
+                      :body payload
+                      :insecure? true
+                      :keepalive 300
+                      :timeout 1000}))
+
+(defn assert-delete
+  "Assert a deletion for an entity in Gallifrey based on the provided key."
+  [host actor key]
+  (kitclient/request {
+                      :url (str "https://" host "/entity/" key)
+                      :method :delete
+                      :query-params {"actor" actor}
+                      :insecure? true
+                      :keepalive 300
+                      :timeout 1000}))
+
+(defn assert-gallifrey [host actor payload]
+  "Propagates an assertion to Gallifrey based off of an event payload coming in from the event service."
+  (let [{:keys [meta body]} payload
+        {:keys [key operation]} meta]
+    (println operation " entity with key " key)
+    (interpret-response key (case operation
+                              "CREATE"
+                              (assert-create host actor key body)
+                              "UPDATE"
+                              (assert-update host actor key body)
+                              "DELETE"
+                              (assert-delete host actor key)))))
diff --git a/src/chameleon/txform.clj b/src/chameleon/txform.clj
new file mode 100644
index 0000000..95f357f
--- /dev/null
+++ b/src/chameleon/txform.clj
@@ -0,0 +1,19 @@
+(ns chameleon.txform
+  (:require [cheshire.core :refer :all]
+            [clojure.string :as str]))
+
+(defn map-keywords
+  "Maps all string based keys to keywords"
+  [kmap]
+  (into {} (for [[key value] kmap] [(keyword key) value])))
+
+(defn flatten-key
+  "Maps a parent-child pair to a period separated keyword"
+  [parent child]
+  (keyword (str (name parent) "." (name child))))
+
+(defn flatten-entry
+  "Flattens a nested map entry to a period separated keyword entry"
+  [map key]
+  (reduce #(assoc %1 (flatten-key key (first %2)) (second %2))
+          {} (seq (key map))))