diff options
-rw-r--r-- | .gitreview | 4 | ||||
-rw-r--r-- | README.md | 3 | ||||
m--------- | chameleon | 5 | ||||
-rw-r--r-- | dev/dev.clj | 28 | ||||
-rw-r--r-- | dev/user.clj | 7 | ||||
-rw-r--r-- | devops/chameleon-config.edn | 10 | ||||
-rw-r--r-- | devops/chameleon.service | 13 | ||||
-rw-r--r-- | devops/chameleon/Dockerfile | 45 | ||||
-rw-r--r-- | devops/chameleon/build-chameleon | 2 | ||||
-rw-r--r-- | devops/chameleon/docker-entrypoint.sh | 5 | ||||
-rw-r--r-- | devops/docker-compose.yml | 16 | ||||
-rw-r--r-- | devops/install-service | 6 | ||||
-rw-r--r-- | devops/nginx/Dockerfile | 9 | ||||
-rw-r--r-- | devops/nginx/default.conf | 23 | ||||
-rw-r--r-- | devops/nginx/nginx.conf | 33 | ||||
-rw-r--r-- | prod/chameleon/server.clj | 18 | ||||
-rw-r--r-- | project.clj | 31 | ||||
-rw-r--r-- | src/chameleon/aai_processor.clj | 26 | ||||
-rw-r--r-- | src/chameleon/config.clj | 18 | ||||
-rw-r--r-- | src/chameleon/event.clj | 18 | ||||
-rw-r--r-- | src/chameleon/handler.clj | 61 | ||||
-rw-r--r-- | src/chameleon/http_server.clj | 9 | ||||
-rw-r--r-- | src/chameleon/route.clj | 71 | ||||
-rw-r--r-- | src/chameleon/txform.clj | 19 |
24 files changed, 480 insertions, 0 deletions
diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..8fd4a05 --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.openecomp.org +port=29418 +project=chameleon.git diff --git a/README.md b/README.md new file mode 100644 index 0000000..0895c05 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# chameleon + +Feeds the Gallifrey time database with entity events diff --git a/chameleon b/chameleon new file mode 160000 +Subproject 2610df2fc29a9d16bb869f652d4d21caeae4c15 diff --git a/dev/dev.clj b/dev/dev.clj new file mode 100644 index 0000000..880f540 --- /dev/null +++ b/dev/dev.clj @@ -0,0 +1,28 @@ +(ns dev + "Tools for interactive development with the REPL. This file should + not be included in a production build of the application." + (:require [chameleon.config :refer [config]] + [chameleon.handler :refer [handler]] + [integrant.core :as ig] + [integrant.repl :refer [clear go halt init reset reset-all]] + [integrant.repl.state :refer [system]] + [clojure.tools.namespace.repl :refer [refresh refresh-all disable-reload!]] + [clojure.repl :refer [apropos dir doc find-doc pst source]] + [clojure.test :refer [run-tests run-all-tests]] + [clojure.pprint :refer [pprint]] + [clojure.reflect :refer [reflect]])) + +(disable-reload! (find-ns 'integrant.core)) + +(integrant.repl/set-prep! (constantly (config { + :event-config {:aai {:host "localhost:3904" + :topic "spikeEvents" + :motsid "" + :pass "" + :consumer-group"chameleon1" + :consumer-id"chameleon1" + :timeout 15000 + :batch-size 8 + :type "HTTPAUTH"}} + :gallifrey-host "localhost:443" + :http-port 3449}))) diff --git a/dev/user.clj b/dev/user.clj new file mode 100644 index 0000000..9a2f701 --- /dev/null +++ b/dev/user.clj @@ -0,0 +1,7 @@ +(ns user) + +(defn dev + "Load and switch to the 'dev' namespace." + [] + (require 'dev) + (in-ns 'dev)) diff --git a/devops/chameleon-config.edn b/devops/chameleon-config.edn new file mode 100644 index 0000000..32e830e --- /dev/null +++ b/devops/chameleon-config.edn @@ -0,0 +1,10 @@ +{:event-config {:host "localhost:3904" + :topic "spikeEvents" + :motsid "" + :pass "" + :consumer-group "chameleon" + :consumer-id "chameleon" + :timeout 15000 + :batch-size 8 + :type "HTTPAUTH"} + :gallifrey-host "localhost:443"} diff --git a/devops/chameleon.service b/devops/chameleon.service new file mode 100644 index 0000000..0c4b937 --- /dev/null +++ b/devops/chameleon.service @@ -0,0 +1,13 @@ +[Unit] +Description=Chameleon container +After=docker.socket early-docker.target network.target network-online.target +Wants=network-online.target +BindsTo=docker.service + +[Service] +Restart=always +ExecStart=/usr/bin/docker-compose -f /opt/chameleon/docker-compose.yml -p chameleon up +ExecStop=/usr/bin/docker-compose -f /opt/chameleon/docker-compose.yml -p chameleon down + +[Install] +WantedBy=multi-user.target diff --git a/devops/chameleon/Dockerfile b/devops/chameleon/Dockerfile new file mode 100644 index 0000000..97e9c3b --- /dev/null +++ b/devops/chameleon/Dockerfile @@ -0,0 +1,45 @@ +FROM alpine:3.6 + +# Java Version +ENV JAVA_VERSION_MAJOR 8 +ENV JAVA_VERSION_MINOR 131 +ENV JAVA_VERSION_BUILD 11 +ENV JAVA_PACKAGE jre +ENV GLIBC_VERSION 2.25-r0 + +ENV JAVA_8_BASE_URL http://download.oracle.com/otn-pub/java/jdk/${JAVA_VERSION_MAJOR}u${JAVA_VERSION_MINOR}-b${JAVA_VERSION_BUILD}/d54c1d3a095b4ff2b6607d096fa80163/${JAVA_PACKAGE}-${JAVA_VERSION_MAJOR}u${JAVA_VERSION_MINOR} + +RUN apk update +RUN apk add curl + +# Install glibc (required for java) +RUN curl -jsSL -o /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ + curl -jsSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk && \ + apk add glibc-${GLIBC_VERSION}.apk && \ + rm -f glibc-${GLIBC_VERSION}.apk + +# Install Java +RUN mkdir /opt && \ + curl -jsSL -H "Cookie: oraclelicense=accept-securebackup-cookie" ${JAVA_8_BASE_URL}-linux-x64.tar.gz | tar -xzf - -C /opt && \ + ln -s /opt/${JAVA_PACKAGE}1.${JAVA_VERSION_MAJOR}.0_${JAVA_VERSION_MINOR} /opt/${JAVA_PACKAGE} && \ + cd /opt/${JAVA_PACKAGE}/ && rm -rf *src.zip lib/missioncontrol lib/visualvm lib/plugin.jar plugin lib/*javafx* lib/*jfx* lib/ext/jfxrt.jar bin/javaws lib/javaws.jar lib/desktop lib/deploy* lib/amd64/libdecora_sse.so lib/amd64/libprism_*.so lib/amd64/libfxplugins.so lib/amd64/libglass.so lib/amd64/libgstreamer-lite.so lib/amd64/libjavafx*.so lib/amd64/libjfx*. + +# Install Java Cryptography Extension (JCE) Unlimited Strength +RUN curl -jksSLH "Cookie: oraclelicense=accept-securebackup-cookie" -o /tmp/jce_policy.zip \ + http://download.oracle.com/otn-pub/java/jce/${JAVA_VERSION_MAJOR}/jce_policy-${JAVA_VERSION_MAJOR}.zip && \ + unzip -o -d /opt/${JAVA_PACKAGE}/lib/security /tmp/jce_policy.zip && rm -f /tmp/jce_policy.zip + +RUN apk del curl + +ENV PATH /opt/jre/bin:${PATH} + +RUN apk add zip openssh-keygen openssh-client + +EXPOSE 80 + +RUN mkdir -p /opt/chameleon +COPY chameleon.jar /opt/chameleon + +COPY ./docker-entrypoint.sh / +RUN chmod 700 /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/devops/chameleon/build-chameleon b/devops/chameleon/build-chameleon new file mode 100644 index 0000000..4421407 --- /dev/null +++ b/devops/chameleon/build-chameleon @@ -0,0 +1,2 @@ +lein uberjar +cp -f ../../target/chameleon.jar . diff --git a/devops/chameleon/docker-entrypoint.sh b/devops/chameleon/docker-entrypoint.sh new file mode 100644 index 0000000..4a0b529 --- /dev/null +++ b/devops/chameleon/docker-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -e + +java -jar /opt/chameleon/chameleon.jar diff --git a/devops/docker-compose.yml b/devops/docker-compose.yml new file mode 100644 index 0000000..b17a98b --- /dev/null +++ b/devops/docker-compose.yml @@ -0,0 +1,16 @@ +chameleon: + build: ./chameleon + container_name: chameleon + environment: + - CONFIG_LOCATION=/opt/chameleon/chameleon-config.edn + volumes: + - ./chameleon-config.edn:/opt/chameleon/chameleon-config.edn + +nginx: + build: ./nginx + container_name: nginx + links: + - chameleon + ports: + - 80:80 + - 443:443 diff --git a/devops/install-service b/devops/install-service new file mode 100644 index 0000000..a54b884 --- /dev/null +++ b/devops/install-service @@ -0,0 +1,6 @@ +sudo cp chameleon.service /etc/systemd/system +sudo chown root:root /etc/systemd/system/chameleon.service +sudo chmod 644 /etc/systemd/system/chameleon.service + +sudo systemctl enable chameleon +sudo systemctl daemon-reload diff --git a/devops/nginx/Dockerfile b/devops/nginx/Dockerfile new file mode 100644 index 0000000..4f2ba9f --- /dev/null +++ b/devops/nginx/Dockerfile @@ -0,0 +1,9 @@ +FROM nginx:alpine + +COPY ssl-cert-snakeoil.pem /etc/ssl/certs/ +COPY ssl-cert-snakeoil.key /etc/ssl/private/ +RUN chown -R nginx:nginx /etc/ssl +RUN chmod 640 /etc/ssl/private/ssl-cert-snakeoil.key +RUN chmod 750 /etc/ssl/private + +COPY default.conf /etc/nginx/conf.d/ diff --git a/devops/nginx/default.conf b/devops/nginx/default.conf new file mode 100644 index 0000000..541f5db --- /dev/null +++ b/devops/nginx/default.conf @@ -0,0 +1,23 @@ + +server { +# Listen on 80 and 443 +listen 80; +listen 443 ssl; +# Self-signed certificate. +ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem; +ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key; + +# Redirect all non-SSL traffic to SSL. +if ($ssl_protocol = "") { +rewrite ^ https://$host$request_uri? permanent; +} + +# Split off traffic to chameleon, and make sure that websockets +# are managed correctly. +location / { +proxy_pass http://chameleon:8082; +proxy_http_version 1.1; +proxy_set_header Upgrade websocket; +proxy_set_header Connection upgrade; +} +} diff --git a/devops/nginx/nginx.conf b/devops/nginx/nginx.conf new file mode 100644 index 0000000..3ebc618 --- /dev/null +++ b/devops/nginx/nginx.conf @@ -0,0 +1,33 @@ + +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-available/*.conf; +} diff --git a/prod/chameleon/server.clj b/prod/chameleon/server.clj new file mode 100644 index 0000000..b9b38db --- /dev/null +++ b/prod/chameleon/server.clj @@ -0,0 +1,18 @@ +(ns chameleon.server + (:require [chameleon.config :refer [config]] + [chameleon.handler :refer [handler]] + [config.core :refer [env]] + [org.httpkit.server :refer [run-server]] + [integrant.core :as ig]) + (:gen-class)) + +(defn -main [& args] + (let [port (Integer/parseInt (or (env :http-port) "8082")) + system-config (read-string (slurp (System/getenv "CONFIG_LOCATION" ))) + event-config (:event-config system-config) + route-config (:gallifrey-host system-config)] + (println "Listening on port" port) + (ig/init (config { + :event-config event-config + :gallifrey-host route-config + :http-port port})))) diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..aaab854 --- /dev/null +++ b/project.clj @@ -0,0 +1,31 @@ +(defproject chameleon "0.1.0" + :dependencies [[org.clojure/clojure "1.8.0"] + [com.7theta/utilis "1.0.4"] + [http-kit "2.2.0"] + [ring/ring-core "1.6.3"] + [ring/ring-defaults "0.3.1"] + [ring/ring-anti-forgery "1.1.0"] + [compojure "1.6.0"] + [liberator "0.15.1"] + [cheshire "5.7.1"] + [inflections "0.13.0"] + [clj-time "0.14.2"] + [integrant "0.6.2"] + [clojure-future-spec "1.9.0-beta4"] + [yogthos/config "0.9"] + [org.onap.aai.event-client/event-client-dmaap "1.2.0"] + ] + :repositories [["onap-releases" {:url "https://nexus.onap.org/content/repositories/releases/"}] + ["onap-public" {:url "https://nexus.onap.org/content/repositories/public/"}] + ["onap-staging" {:url "https://nexus.onap.org/content/repositories/staging/"}] + ["onap-snapshot" {:url "https://nexus.onap.org/content/repositories/snapshots/"}] + ] + :min-lein-version "2.5.3" + :profiles {:dev {:source-paths ["dev"] + :dependencies [[ring/ring-devel "1.6.3"] + [integrant/repl "0.2.0"]]} + :uberjar {:source-paths ["prod"] + :main chameleon.server + :aot [chameleon.server] + :uberjar-name "chameleon.jar"}} + :prep-tasks [ "compile"]) diff --git a/src/chameleon/aai_processor.clj b/src/chameleon/aai_processor.clj new file mode 100644 index 0000000..c709ed1 --- /dev/null +++ b/src/chameleon/aai_processor.clj @@ -0,0 +1,26 @@ +(ns chameleon.aai-processor + (:require [chameleon.txform :refer :all] + [chameleon.route :refer :all] + [cheshire.core :refer :all])) + +(defn from-gallifrey + "Transforms Gallifrey response payloads into a format consumable by AAI-centric clients" + [body] + (->> body + (map (fn [[k v]] [(clojure.string/split k #"\.") v])) + ((fn [x] (reduce #(assoc-in %1 (first %2) (second %2) ) {} x))))) + +(defn from-spike + "Transforms Spike-based event payloads to a format accepted by Gallifrey for vertices and relationships" + [gallifrey-host payload] + (let [txpayload (map-keywords (parse-string payload)) + operation (:operation txpayload) + entity-type (if (contains? txpayload :vertex) + :vertex + :relationship) + entity (map-keywords (entity-type txpayload)) + key (:key entity) + properties (assoc (:properties entity) :type (:type entity))] + (assert-gallifrey gallifrey-host "aai" (if (= entity-type :vertex) + {:meta {:key key :operation operation} :body (generate-string properties)} + {:meta {:key key :operation operation} :body (generate-string (conj properties (flatten-entry entity :source) (flatten-entry entity :target)))})))) diff --git a/src/chameleon/config.clj b/src/chameleon/config.clj new file mode 100644 index 0000000..4982f4a --- /dev/null +++ b/src/chameleon/config.clj @@ -0,0 +1,18 @@ +(ns chameleon.config + (:require [integrant.core :as ig] + [chameleon.aai-processor :refer :all])) + +(defn config + [app-config] + (let [conf { + :chameleon/event + {:event-config (assoc-in (:event-config app-config) + [:aai :processor] from-spike) + :gallifrey-host (:gallifrey-host app-config)} + :chameleon/handler + {:gallifrey-host (:gallifrey-host app-config)} + :chameleon/http-server + {:port (:http-port app-config) + :handler (ig/ref :chameleon/handler)}}] + (ig/load-namespaces conf) + conf)) diff --git a/src/chameleon/event.clj b/src/chameleon/event.clj new file mode 100644 index 0000000..8201acb --- /dev/null +++ b/src/chameleon/event.clj @@ -0,0 +1,18 @@ +(ns chameleon.event + (:require [chameleon.txform] + [chameleon.route] + [integrant.core :as ig]) + (:import [org.onap.aai.event.client DMaaPEventConsumer])) + +(defmethod ig/init-key :chameleon/event + [_ {:keys [event-config gallifrey-host]}] + (let [{:keys [host topic motsid pass consumer-group consumer-id timeout batch-size type processor]} (:aai event-config) + event-processor (DMaaPEventConsumer. host topic motsid pass consumer-group consumer-id timeout batch-size type)] + (println "Event processor for AAI created. Starting event polling on " host topic) + (.start (Thread. (fn [] (while true + (let [it (.iterator (.consume event-processor))] + (println "Polling...") + (while (.hasNext it) + (let [event (.next it)] + (processor gallifrey-host event)))))))) + )) diff --git a/src/chameleon/handler.clj b/src/chameleon/handler.clj new file mode 100644 index 0000000..7a4a5fe --- /dev/null +++ b/src/chameleon/handler.clj @@ -0,0 +1,61 @@ +(ns chameleon.handler + (:require [chameleon.route :as c-route] + [chameleon.aai-processor] + [utilis.map :refer [map-vals compact]] + [liberator.core :refer [defresource]] + [compojure.core :refer [GET PUT PATCH ANY defroutes]] + [compojure.route :refer [resources]] + [ring.util.response :refer [resource-response content-type]] + [ring.middleware.defaults :refer [wrap-defaults api-defaults]] + [ring.middleware.anti-forgery :refer [wrap-anti-forgery]] + [ring.middleware.session :refer [wrap-session]] + [cheshire.core :as json] + [clj-time.format :as tf] + [integrant.core :as ig])) + +(declare handler) + +(defonce ^:private g-host (atom nil)) + +(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host]}] + (reset! g-host gallifrey-host) + handler) + +(defmethod ig/halt-key! :chameleon/handler [_ _] + (reset! g-host nil)) + +(declare serialize de-serialize) + +(defresource entity-endpoint [id] + :allowed-methods [:get] + :available-media-types ["application/json"] + :exists? (fn [ctx] + (let [resource (-> (c-route/query @g-host id (-> ctx :request :params :t-k)))] + (when (= (:status resource) 200) + {::resource (-> resource :body json/parse-string (dissoc "_meta") (chameleon.aai-processor/from-gallifrey))}))) + :existed? (fn [ctx] + (when-let [status (-> (c-route/query @g-host id (-> ctx :request :params :t-k)) :status)] + (= status 410))) + :handle-ok ::resource) + +(defroutes app-routes + (GET "/entity/:id" [id] (entity-endpoint id)) + (resources "/")) + +(def handler + (-> app-routes + (wrap-defaults api-defaults))) + + +;;; Implementation + +(defn- serialize + [e] + (compact + (update e :_meta #(map-vals + (fn [m] + (map-vals str m)) %)))) + +(defn- de-serialize + [e] + e) diff --git a/src/chameleon/http_server.clj b/src/chameleon/http_server.clj new file mode 100644 index 0000000..cf8ae07 --- /dev/null +++ b/src/chameleon/http_server.clj @@ -0,0 +1,9 @@ +(ns chameleon.http-server + (:require [org.httpkit.server :refer [run-server]] + [integrant.core :as ig])) + +(defmethod ig/init-key :chameleon/http-server [_ {:keys [port handler]}] + (run-server handler {:port port})) + +(defmethod ig/halt-key! :chameleon/http-server [_ server] + (server :timeout 100)) diff --git a/src/chameleon/route.clj b/src/chameleon/route.clj new file mode 100644 index 0000000..2b30f42 --- /dev/null +++ b/src/chameleon/route.clj @@ -0,0 +1,71 @@ +(ns chameleon.route + (:require [org.httpkit.client :as kitclient])) + +(defn- interpret-response + "Print out the response from the Gallifrey server" + [key response] + (let [{:keys [status body]}@response] + (println "Response for request with key " key " resulted in status " status + " with body " body ))) + +(defn query + "Retrieve an entity referenced by id at the provided host. Optionally provide + a time 't-k' that defines a query based on when the system knew about the + state of the entity." + [host key & [k]] + @(kitclient/request { + :url (str "https://" host "/entity/" key) + :method :get + :query-params (if-let [t-k k] {"t-k" t-k}) + :insecure? true + :keepalive 300 + :timeout 1000})) + +(defn assert-create + "Creates an entity in Gallifrey with an initial set of assertions coming from the provided payload" + [host actor key payload] + (print "Final: " payload) + (kitclient/request { + :url (str "https://" host "/entity/" key) + :method :put + :query-params {"actor" actor "create" "true"} + :body payload + :insecure? true + :keepalive 300 + :timeout 1000})) + +(defn assert-update + "Update an entity in Gallifrey with a set of assertions coming from the provided payload" + [host actor key payload] + (kitclient/request { + :url (str "https://" host "/entity/" key) + :method :put + :query-params {"actor" actor "changes-only" "true"} + :body payload + :insecure? true + :keepalive 300 + :timeout 1000})) + +(defn assert-delete + "Assert a deletion for an entity in Gallifrey based on the provided key." + [host actor key] + (kitclient/request { + :url (str "https://" host "/entity/" key) + :method :delete + :query-params {"actor" actor} + :insecure? true + :keepalive 300 + :timeout 1000})) + +(defn assert-gallifrey [host actor payload] + "Propagates an assertion to Gallifrey based off of an event payload coming in from the event service." + (let [{:keys [meta body]} payload + {:keys [key operation]} meta] + (println operation " entity with key " key) + (interpret-response key (case operation + "CREATE" + (assert-create host actor key body) + "UPDATE" + (assert-update host actor key body) + "DELETE" + (assert-delete host actor key))))) diff --git a/src/chameleon/txform.clj b/src/chameleon/txform.clj new file mode 100644 index 0000000..95f357f --- /dev/null +++ b/src/chameleon/txform.clj @@ -0,0 +1,19 @@ +(ns chameleon.txform + (:require [cheshire.core :refer :all] + [clojure.string :as str])) + +(defn map-keywords + "Maps all string based keys to keywords" + [kmap] + (into {} (for [[key value] kmap] [(keyword key) value]))) + +(defn flatten-key + "Maps a parent-child pair to a period separated keyword" + [parent child] + (keyword (str (name parent) "." (name child)))) + +(defn flatten-entry + "Flattens a nested map entry to a period separated keyword entry" + [map key] + (reduce #(assoc %1 (flatten-key key (first %2)) (second %2)) + {} (seq (key map)))) |