diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | .gitreview | 2 | ||||
| -rw-r--r-- | README.md | 76 | ||||
| m--------- | chameleon | 5 | ||||
| -rw-r--r-- | dev/dev.clj | 10 | ||||
| -rw-r--r-- | devops/chameleon/build-chameleon | 15 | ||||
| -rw-r--r-- | pom.xml | 305 | ||||
| -rw-r--r-- | prod/chameleon/server.clj | 12 | ||||
| -rw-r--r-- | project.clj | 42 | ||||
| -rw-r--r-- | resources/chameleon_logback.xml | 109 | ||||
| -rw-r--r-- | resources/log/ChameleonMsgs.properties | 19 | ||||
| -rw-r--r-- | src/chameleon/aai_processor.clj | 57 | ||||
| -rw-r--r-- | src/chameleon/config.clj | 8 | ||||
| -rw-r--r-- | src/chameleon/event.clj | 22 | ||||
| -rw-r--r-- | src/chameleon/handler.clj | 37 | ||||
| -rw-r--r-- | src/chameleon/logging.clj | 123 | ||||
| -rw-r--r-- | src/chameleon/route.clj | 50 | ||||
| -rw-r--r-- | src/chameleon/specs.clj | 71 | ||||
| -rw-r--r-- | test/chameleon/testing.clj | 42 |
19 files changed, 902 insertions, 106 deletions
@@ -1,8 +1,9 @@ # Created by https://www.gitignore.io/api/clojure,emacs,osx,java ### Clojure ### -pom.xml pom.xml.asc +chameleon.iml +.idea *jar lib/ classes/ @@ -1,4 +1,4 @@ [gerrit] -host=gerrit.openecomp.org +host=gerrit.onap.org port=29418 project=chameleon.git @@ -1,3 +1,77 @@ -# chameleon +# CHAMELEON Feeds the Gallifrey time database with entity events + +### Building the project + +1. mvn clean install +2. mvn package + +### Generating a code coverage report + +1. mvn clean install +2. mvn exec:java -Dexec.classpathScope=test -Dexec.mainClass='clojure.main' -Dexec.args="--main cloverage.coverage -n 'chameleon.*' -t 'chameleon.testing'" + + 2.1. Follow the instructions to view the report. + +3. To tweak the report generation, follow the instrustions here: + https://github.com/cloverage/cloverage + +### Developing + +1. I use Leiningen for building my clojure project and for dependency + management. You can read about Leiningen and follow the tutorial at + the link: + https://github.com/technomancy/leiningen + +2. If you make ** changes to the project.clj file, you are required to + re-generate the pom.xml by running `lein pom`** (requires leiningen + to be instaled). + + You can also add your changes to the pom.xml file manually. + +3. I use Emacs as my editor and CIDER for interactive development in + Clojure. + - CIDER: https://github.com/clojure-emacs/cider + - Emacs: https://www.gnu.org/software/emacs/ + + - Here is a tutorial on Emacs: https://www.gnu.org/software/emacs/tour/ + - Here is the emacs configuration I've used, it includes the CIDER + package: + https://github.com/sandhu/emacs.d + +4. Start nrepl `C-c M-j` (you have to be in any *.clj file in the + chameleon project). If the cider-repl buffer doesn't open + automatically, use `C-x b` and use the arrow keys to find the + `cider-repl chameleon` buffer and press enter. + +5. Open the dev.clj file by `C-x C-f`, load it `C-c C-k` and switch + your namespace to dev `C-c M-n`. + +6. In the repl buffer you should see that the namespace went from `user` + to `dev`. + +7. Make sure the configuration is correct in the dev.clj file and once + verified, in the repl buffer type `(go)` and press enter. + +8. You can verify everthing is running by generating/adding an event + to dmaap and seeing it flow through chameleon in the error.log file. + +### Running it locally (Assumming you're not using Emacs and using + `lein repl` from the command line) + +**Make you're in the directory at the root of your project. Open the + file `chameleon/dev.clj` and update the config to the correct + dmaap host, topic, and make sure you're using the correct + consumer group and id.** + +1. On the command line, execute `lein repl` + +2. Once in the clojure repl, load the dev namespace + (load "dev") + +3. Go into the dev namespace + (in-ns 'dev) + +4. Run the following command in the repl + (go) diff --git a/chameleon b/chameleon deleted file mode 160000 -Subproject 2610df2fc29a9d16bb869f652d4d21caeae4c15 diff --git a/dev/dev.clj b/dev/dev.clj index 880f540..51d0fdf 100644 --- a/dev/dev.clj +++ b/dev/dev.clj @@ -4,19 +4,19 @@ (:require [chameleon.config :refer [config]] [chameleon.handler :refer [handler]] [integrant.core :as ig] + [chameleon.logging :as log] + [chameleon.specs :as sp] [integrant.repl :refer [clear go halt init reset reset-all]] [integrant.repl.state :refer [system]] [clojure.tools.namespace.repl :refer [refresh refresh-all disable-reload!]] [clojure.repl :refer [apropos dir doc find-doc pst source]] [clojure.test :refer [run-tests run-all-tests]] - [clojure.pprint :refer [pprint]] - [clojure.reflect :refer [reflect]])) + [clojure.pprint :refer [pprint]])) (disable-reload! (find-ns 'integrant.core)) -(integrant.repl/set-prep! (constantly (config { - :event-config {:aai {:host "localhost:3904" - :topic "spikeEvents" +(integrant.repl/set-prep! (constantly (config {:event-config {:aai {:host "localhost:3904" + :topic "events" :motsid "" :pass "" :consumer-group"chameleon1" diff --git a/devops/chameleon/build-chameleon b/devops/chameleon/build-chameleon index 4421407..c588e95 100644 --- a/devops/chameleon/build-chameleon +++ b/devops/chameleon/build-chameleon @@ -1,2 +1,13 @@ -lein uberjar -cp -f ../../target/chameleon.jar . +!#/bin/bash +ls build-chameleon +if [ $? -eq 0 ]; then + echo "Building Chameleon" +else + echo "FAILED. Make sure you're in the same directory as the \"build-chameleon\" file." + exit 1 +fi +pushd . +cd ../../ +mvn -X clean package +popd +cp -f ../../target/chameleon-0.1.0-jar-with-dependencies.jar ./chameleon.jar @@ -0,0 +1,305 @@ +<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>chameleon</groupId> + <artifactId>chameleon</artifactId> + <packaging>jar</packaging> + <version>0.1.0</version> + <name>chameleon</name> + <description/> + <scm> + <tag>36b5671af2c3eec5ca81663382c4ca2898f79e55 +</tag> + <url/> + </scm> + <build> + <sourceDirectory>src</sourceDirectory> + <testSourceDirectory>test</testSourceDirectory> + <resources> + <resource> + <directory>resources</directory> + </resource> + </resources> + <testResources> + <testResource> + <directory>resources</directory> + </testResource> + </testResources> + <directory>target</directory> + <outputDirectory>target/classes</outputDirectory> + <plugins> + <plugin> + <groupId>com.theoryinpractise</groupId> + <artifactId>clojure-maven-plugin</artifactId> + <version>1.3.13</version> + <extensions>true</extensions> + <configuration> + <sourceDirectories> + <sourceDirectory>src</sourceDirectory> + <sourceDirectory>prod</sourceDirectory> + <sourceDirectory>test</sourceDirectory> + </sourceDirectories> + </configuration> + <executions> + <execution> + <id>compile</id> + <goals> + <goal>compile</goal> + </goals> + <phase>compile</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + <configuration> + <archive> + <manifest> + <addClasspath>true</addClasspath> + <mainClass>chameleon.server</mainClass> + <classpathPrefix>dependency</classpathPrefix> + </manifest> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> + <executions> + <execution> + <id>copy-dependencies</id> + <goals> + <goal>copy-dependencies</goal> + </goals> + <phase>package</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.4.1</version> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <archive> + <manifest> + <mainClass>chameleon.server</mainClass> + </manifest> + </archive> + </configuration> + <executions> + <execution> + <id>assemble</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>prod</source> + <source>test</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <repositories> + <repository> + <id>central</id> + <url>https://repo1.maven.org/maven2/</url> + <snapshots> + <enabled>false</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>clojars</id> + <url>https://clojars.org/repo/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-releases</id> + <url>https://nexus.onap.org/content/repositories/releases/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-public</id> + <url>https://nexus.onap.org/content/repositories/public/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-staging</id> + <url>https://nexus.onap.org/content/repositories/staging/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-snapshot</id> + <url>https://nexus.onap.org/content/repositories/snapshots/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + </repositories> + <dependencyManagement> + <dependencies/> + </dependencyManagement> + <dependencies> + <dependency> + <groupId>org.clojure</groupId> + <artifactId>clojure</artifactId> + <version>1.9.0</version> + </dependency> + <dependency> + <groupId>com.7theta</groupId> + <artifactId>utilis</artifactId> + <version>1.0.4</version> + </dependency> + <dependency> + <groupId>http-kit</groupId> + <artifactId>http-kit</artifactId> + <version>2.2.0</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-core</artifactId> + <version>1.6.3</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-defaults</artifactId> + <version>0.3.1</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-anti-forgery</artifactId> + <version>1.1.0</version> + </dependency> + <dependency> + <groupId>compojure</groupId> + <artifactId>compojure</artifactId> + <version>1.6.0</version> + </dependency> + <dependency> + <groupId>liberator</groupId> + <artifactId>liberator</artifactId> + <version>0.15.1</version> + </dependency> + <dependency> + <groupId>cheshire</groupId> + <artifactId>cheshire</artifactId> + <version>5.7.1</version> + </dependency> + <dependency> + <groupId>inflections</groupId> + <artifactId>inflections</artifactId> + <version>0.13.0</version> + </dependency> + <dependency> + <groupId>clj-time</groupId> + <artifactId>clj-time</artifactId> + <version>0.14.2</version> + </dependency> + <dependency> + <groupId>integrant</groupId> + <artifactId>integrant</artifactId> + <version>0.6.2</version> + </dependency> + <dependency> + <groupId>yogthos</groupId> + <artifactId>config</artifactId> + <version>0.9</version> + </dependency> + <dependency> + <groupId>org.onap.aai.event-client</groupId> + <artifactId>event-client-dmaap</artifactId> + <version>1.2.1</version> + </dependency> + <dependency> + <groupId>org.onap.aai.logging-service</groupId> + <artifactId>common-logging</artifactId> + <version>1.2.2</version> + </dependency> + <dependency> + <groupId>camel-snake-kebab</groupId> + <artifactId>camel-snake-kebab</artifactId> + <version>0.4.0</version> + </dependency> + <dependency> + <groupId>metosin</groupId> + <artifactId>ring-http-response</artifactId> + <version>0.9.0</version> + </dependency> + <dependency> + <groupId>org.clojure</groupId> + <artifactId>test.check</artifactId> + <version>0.9.0</version> + </dependency> + <dependency> + <groupId>cloverage</groupId> + <artifactId>cloverage</artifactId> + <version>1.0.10</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-devel</artifactId> + <version>1.6.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>integrant</groupId> + <artifactId>repl</artifactId> + <version>0.2.0</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> + +<!-- This file was autogenerated by Leiningen. + Please do not edit it directly; instead edit project.clj and regenerate it. + It should not be considered canonical data. For more information see + https://github.com/technomancy/leiningen --> diff --git a/prod/chameleon/server.clj b/prod/chameleon/server.clj index b9b38db..12c2c02 100644 --- a/prod/chameleon/server.clj +++ b/prod/chameleon/server.clj @@ -3,16 +3,18 @@ [chameleon.handler :refer [handler]] [config.core :refer [env]] [org.httpkit.server :refer [run-server]] - [integrant.core :as ig]) + [integrant.core :as ig] + [chameleon.specs :as cs]) (:gen-class)) (defn -main [& args] (let [port (Integer/parseInt (or (env :http-port) "8082")) system-config (read-string (slurp (System/getenv "CONFIG_LOCATION" ))) event-config (:event-config system-config) - route-config (:gallifrey-host system-config)] + route-config (:gallifrey-host system-config) + log-config (:log-config system-config)] (println "Listening on port" port) - (ig/init (config { - :event-config event-config + (ig/init (config {:event-config event-config :gallifrey-host route-config - :http-port port})))) + :http-port port + :log-config log-config})))) diff --git a/project.clj b/project.clj index 044856b..430173c 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,5 @@ (defproject chameleon "0.1.0" - :dependencies [[org.clojure/clojure "1.8.0"] + :dependencies [[org.clojure/clojure "1.9.0"] [com.7theta/utilis "1.0.4"] [http-kit "2.2.0"] [ring/ring-core "1.6.3"] @@ -11,15 +11,18 @@ [inflections "0.13.0"] [clj-time "0.14.2"] [integrant "0.6.2"] - [clojure-future-spec "1.9.0-beta4"] [yogthos/config "0.9"] [org.onap.aai.event-client/event-client-dmaap "1.2.1"] - ] + [org.onap.aai.logging-service/common-logging "1.2.2"] + [camel-snake-kebab "0.4.0"] + [metosin/ring-http-response "0.9.0"] + [org.clojure/test.check "0.9.0"] + [cloverage/cloverage "1.0.10"]] + :plugins [[lein-cloverage "1.0.10"]] :repositories [["onap-releases" {:url "https://nexus.onap.org/content/repositories/releases/"}] ["onap-public" {:url "https://nexus.onap.org/content/repositories/public/"}] ["onap-staging" {:url "https://nexus.onap.org/content/repositories/staging/"}] - ["onap-snapshot" {:url "https://nexus.onap.org/content/repositories/snapshots/"}] - ] + ["onap-snapshot" {:url "https://nexus.onap.org/content/repositories/snapshots/"}]] :min-lein-version "2.5.3" :profiles {:dev {:source-paths ["dev"] :dependencies [[ring/ring-devel "1.6.3"] @@ -28,4 +31,31 @@ :main chameleon.server :aot [chameleon.server] :uberjar-name "chameleon.jar"}} - :prep-tasks [ "compile"]) + :prep-tasks ["compile"] + :source-paths ["src" "prod" "test"] + :resource-paths ["resources"] + :pom-plugins [[com.theoryinpractise/clojure-maven-plugin "1.3.13" + {:extensions "true" + :configuration [:sourceDirectories + [:sourceDirectory "src"] + [:sourceDirectory "prod"] + [:sourceDirectory "test"]] + :executions ([:execution [:id "compile"] + [:goals ([:goal "compile"])] + [:phase "compile"]])}] + [org.apache.maven.plugins/maven-jar-plugin "2.4" + {:configuration [:archive [:manifest + [:addClasspath true] + [:mainClass "chameleon.server"] + [:classpathPrefix "dependency"]]]}] + [org.apache.maven.plugins/maven-dependency-plugin "2.8" + {:executions ([:execution [:id "copy-dependencies"] + [:goals ([:goal "copy-dependencies"])] + [:phase "package"]])}] + [org.apache.maven.plugins/maven-assembly-plugin "2.4.1" + {:configuration ([:descriptorRefs [:descriptorRef "jar-with-dependencies"]] + [:archive [:manifest + [:mainClass "chameleon.server"]]]) + :executions ([:execution [:id "assemble"] + [:phase "package"] + [:goals ([:goal "single"])]])}]]) diff --git a/resources/chameleon_logback.xml b/resources/chameleon_logback.xml new file mode 100644 index 0000000..5b723d7 --- /dev/null +++ b/resources/chameleon_logback.xml @@ -0,0 +1,109 @@ +<configuration scan="true" scanPeriod="3 seconds" debug="false"> + <!--<jmxConfigurator /> --> + <!-- directory path for all other type logs --> + + <property name="logDir" value="/opt/chameleon/logs" /> + + <!-- specify the component name --> + <property name="componentName" value="AAI-CHAMELEON" /> + + <!-- default eelf log file names --> + <property name="generalLogName" value="error" /> + <property name="metricsLogName" value="metrics" /> + <property name="auditLogName" value="audit" /> + <property name="debugLogName" value="debug" /> + + <property name="errorLogPattern" value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%mdc{RequestId}|%thread|DataRouter|%mdc{PartnerName}|%logger||%.-5level|%msg%n" /> + + <property name="auditMetricPattern" value="%msg%n" /> + + <property name="logDirectory" value="${logDir}/${componentName}" /> + + <!-- ============================================================================ --> + <!-- EELF Appenders --> + <!-- ============================================================================ --> + + <!-- The EELFAppender is used to record events to the general application + log --> + + <appender name="EELF" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${generalLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip + </fileNamePattern> + <maxHistory>60</maxHistory> + </rollingPolicy> + <encoder> + <pattern>${errorLogPattern}</pattern> + <outputPatternAsHeader>true</outputPatternAsHeader> + </encoder> + </appender> + + <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender"> + <!-- deny all events with a level below INFO, that is TRACE and DEBUG --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + <queueSize>256</queueSize> + <appender-ref ref="EELF" /> + </appender> + + + <!-- EELF Audit Appender. This appender is used to record audit engine + related logging events. The audit logger and appender are specializations + of the EELF application root logger and appender. This can be used to segregate + Policy engine events from other components, or it can be eliminated to record + these events as part of the application root log. --> + + <appender name="EELFAudit" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${auditLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip + </fileNamePattern> + <maxHistory>60</maxHistory> + </rollingPolicy> + <encoder> + <outputPatternAsHeader>true</outputPatternAsHeader> + <pattern>${auditMetricPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFAudit" /> + </appender> + <!-- ============================================================================ --> + <!-- EELF loggers --> + <!-- ============================================================================ --> + + <!-- CRUD Service loggers --> + <logger name="chameleon" level="INFO"> + <appender-ref ref="asyncEELF" /> + </logger> + + <logger name="com.att.eelf.audit" level="info" additivity="false"> + <appender-ref ref="asyncEELFAudit" /> + </logger> + + <!-- Other Loggers that may help troubleshoot --> + <logger name="net.sf" level="WARN" /> + <logger name="org.apache" level="WARN" /> + <logger name="org.apache.commons.httpclient" level="WARN" /> + <logger name="org.apache.commons" level="WARN" /> + <logger name="org.apache.coyote" level="WARN" /> + <logger name="org.apache.jasper" level="WARN" /> + + <!-- logback internals logging --> + <logger name="ch.qos.logback.classic" level="WARN" /> + <logger name="ch.qos.logback.core" level="WARN" /> + + <root> + <!-- <appender-ref ref="asyncEELF" /> --> + <!-- <appender-ref ref="asyncEELFDebug" /> --> + </root> + +</configuration> diff --git a/resources/log/ChameleonMsgs.properties b/resources/log/ChameleonMsgs.properties new file mode 100644 index 0000000..4e61e2a --- /dev/null +++ b/resources/log/ChameleonMsgs.properties @@ -0,0 +1,19 @@ +EVENT_PROCESSOR=\ +CHM0001I|\ +Event Processor {0}\ + +GALLIFREY_ASSERTION=\ +CHM0002I|\ +Attempt Gallifrey Assertion of operation {0} with type {1} and key {2} \ + +GALLIFREY_ASSERTED=\ +CHM0002I|\ +Asserted type {0} and key {1} in Gallifrey\ + +RESPONSE=\ +CHM0003I|\ +Response for operation "{0}" for endpoint "{1}" resulted in status "{2}" with body "{3}"\ + +CHAMELEON_REQUEST=\ +CHM0004I|\ +Incoming Request of type "{0}" for endpoint "{1}" from address "{2}"\
\ No newline at end of file diff --git a/src/chameleon/aai_processor.clj b/src/chameleon/aai_processor.clj index d1a7d25..4243b39 100644 --- a/src/chameleon/aai_processor.clj +++ b/src/chameleon/aai_processor.clj @@ -1,9 +1,8 @@ (ns chameleon.aai-processor - (:require - [chameleon.route :refer :all] - [cheshire.core :refer :all] - [integrant.core :as ig] - [clojure.set :refer :all])) + (:require [chameleon.route :refer :all] + [cheshire.core :refer :all] + [integrant.core :as ig] + [clojure.set :refer :all])) (defonce ^:private p-attr (atom nil)) (defonce ^:private t-attr (atom nil)) @@ -33,8 +32,7 @@ {"id" id "type" type "source" {"id" src-id "type" src-type} - "target" {"id" target-id "type" target-type}}) - ) + "target" {"id" target-id "type" target-type}})) (defn from-gallifrey "Transforms Gallifrey response payloads into a format consumable by AAI-centric clients" @@ -42,27 +40,21 @@ (let [resource-type (get-in body ["properties" "_type"]) id (body "_id") type (get-in body ["properties" "type"]) - properties (body "properties")] + properties (body "properties") + entity-response {"id" id + "type" type + "properties" (dissoc properties "_type" "type")}] (if (= resource-type "entity") - ; Transform into an entity type + ;; Transform into an entity type (let [relationships (body "relationships")] - { - "id" id - "type" type - "properties" (dissoc properties "_type" "type") - "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships))) - "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships))) - }) - ; Transform into a relationship type - { - "id" id - "type" type - "properties" (dissoc properties "_type" "type") - }))) + (assoc entity-response + "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships))) + "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships))))) + entity-response))) (defn from-spike "Transforms Spike-based event payloads to a format accepted by Gallifrey for vertices and relationships" - [gallifrey-host payload] + [gallifrey-host payload & [error-logger audit-logger]] (let [txpayload (map-keywords (parse-string payload)) operation (:operation txpayload) parse-type (if (contains? txpayload :vertex) @@ -74,13 +66,14 @@ entity (map-keywords (parse-type txpayload)) key (:key entity) properties (assoc (:properties entity) :type (:type entity)) - truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)}) - assertion {:meta {:key key - :operation operation - :time truth-time}} + truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)}) + assertion {:meta {:key key + :operation operation + :time truth-time}} provenance (get properties @p-attr "aai")] - (assert-gallifrey gallifrey-host provenance (name entity-type) (if (= entity-type :entity) - (assoc assertion :body (generate-string {:properties properties})) - (assoc assertion :body (generate-string (conj {:properties properties} - {:source (rename-keys (:source entity) {"key" "id"})} - {:target (rename-keys (:target entity) {"key" "id"})}))))))) + (assert-gallifrey! gallifrey-host provenance (name entity-type) + (if (= entity-type :entity) + (assoc assertion :body (generate-string {:properties properties})) + (assoc assertion :body (generate-string (conj {:properties properties} + {:source (rename-keys (:source entity) {"key" "id"})} + {:target (rename-keys (:target entity) {"key" "id"})})))) error-logger audit-logger))) diff --git a/src/chameleon/config.clj b/src/chameleon/config.clj index bd99096..10324e2 100644 --- a/src/chameleon/config.clj +++ b/src/chameleon/config.clj @@ -4,14 +4,16 @@ (defn config [app-config] - (let [conf { + (let [conf {:chameleon/loggers (:log-config app-config) :chameleon/event {:event-config (assoc-in (:event-config app-config) [:aai :processor] from-spike) - :gallifrey-host (:gallifrey-host app-config)} + :gallifrey-host (:gallifrey-host app-config) + :loggers (ig/ref :chameleon/loggers)} :chameleon/handler {:gallifrey-host (:gallifrey-host app-config) - :gallifrey-transformer from-gallifrey} + :gallifrey-transformer from-gallifrey + :loggers (ig/ref :chameleon/loggers)} :chameleon/aai-processor {:provenance-attr "last-mod-source-of-truth" :truth-attr "truth-time"} diff --git a/src/chameleon/event.clj b/src/chameleon/event.clj index c4bec8e..92f4211 100644 --- a/src/chameleon/event.clj +++ b/src/chameleon/event.clj @@ -1,19 +1,25 @@ (ns chameleon.event (:require [integrant.core :as ig] - [clojure.string :refer [starts-with?]]) + [clojure.string :refer [starts-with?]] + [chameleon.logging :as log]) (:import [org.onap.aai.event.client DMaaPEventConsumer])) (defmethod ig/init-key :chameleon/event - [_ {:keys [event-config gallifrey-host]}] + [_ {:keys [event-config gallifrey-host loggers]}] (let [{:keys [host topic motsid pass consumer-group consumer-id timeout batch-size type processor]} (:aai event-config) + [error-logger audit-logger] loggers event-processor (DMaaPEventConsumer. host topic motsid pass consumer-group consumer-id timeout batch-size type)] - (println "Event processor for AAI created. Starting event polling on " host topic) + (log/info error-logger "EVENT_PROCESSOR" [(format "AAI created. Starting event polling on %s %s" host topic) ]) (.start (Thread. (fn [] (while true (let [it (.iterator (.consume event-processor))] - (println "Polling...") + (log/info error-logger "EVENT_PROCESSOR" ["Polling ..."]) (while (.hasNext it) + (log/mdc-init! "SPIKE-EVENT" "CHAMELEON" "" "" gallifrey-host) (try (let [event (.next it)] - (if (not (starts-with? event "DMAAP")) ;Temporarily added for current version of dmaap client - (processor gallifrey-host event))) - (catch Exception e (println (str "Unexpected exception during processing: " (.getMessage e))))))))))) - )) + ;;Temporarily added for current version of dmaap client + (when-not (starts-with? event "DMAAP") + (log/info error-logger "EVENT_PROCESSOR" [event]) + (processor gallifrey-host event error-logger audit-logger) + (log/mdc-clear!))) + (catch Exception e + (println (str "Unexpected exception during processing: " (.getMessage e))))))))))))) diff --git a/src/chameleon/handler.clj b/src/chameleon/handler.clj index fd97bf1..675a34b 100644 --- a/src/chameleon/handler.clj +++ b/src/chameleon/handler.clj @@ -10,17 +10,18 @@ [ring.middleware.session :refer [wrap-session]] [cheshire.core :as json] [clj-time.format :as tf] - [integrant.core :as ig])) + [integrant.core :as ig] + [chameleon.logging :as log])) (declare handler) (defonce ^:private g-host (atom nil)) (defonce ^:private g-transformer nil) -(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host gallifrey-transformer]}] +(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host loggers gallifrey-transformer]}] (reset! g-host gallifrey-host) (def g-transformer gallifrey-transformer) - handler) + (handler loggers)) (defmethod ig/halt-key! :chameleon/handler [_ _] (reset! g-host nil) @@ -32,10 +33,10 @@ :allowed-methods [:get] :available-media-types ["application/json"] :exists? (fn [ctx] - (let [resource (c-route/query @g-host id type (-> ctx - :request - :params - (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys + (let [resource (c-route/query @g-host id type (-> ctx + :request + :params + (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys (when (= (:status resource) 200) {::resource (-> resource :body @@ -56,9 +57,27 @@ (GET "/relationship/:id" [id] (resource-endpoint "relationship" id)) (resources "/")) -(def handler +(defn log-reqs + [handler loggers] + (let [[error-logger audit-logger] loggers] + (fn [request] + (log/mdc-init! (get-in request [:headers "X-TransactionId"]) "CHAMELEON" + "CHAMELEON_SERVICE" "ONAP" (:remote-addr request)) + (log/info error-logger "CHAMELEON_REQUEST" (mapv str ((juxt (comp name :request-method) :uri :remote-addr) request))) + (let [resp (handler request) + fields (->> ((juxt :status :body) resp) + (into ((juxt (comp name :request-method) :uri) request)) + (mapv str))] + (log/info error-logger "RESPONSE" fields) + (log/info audit-logger "RESPONSE" fields) + (log/mdc-clear!) + resp)))) + +(defn handler + [loggers] (-> app-routes - (wrap-defaults api-defaults))) + (wrap-defaults api-defaults) + (log-reqs loggers))) ;;; Implementation diff --git a/src/chameleon/logging.clj b/src/chameleon/logging.clj new file mode 100644 index 0000000..15e1e7b --- /dev/null +++ b/src/chameleon/logging.clj @@ -0,0 +1,123 @@ +(ns chameleon.logging + (:require [camel-snake-kebab.core :as cs] + [camel-snake-kebab.extras :refer [transform-keys]] + [clojure.java.io :as io] + [integrant.core :as ig] + [clojure.spec.alpha :as s]) + (:import [org.onap.aai.cl.api Logger LogFields LogLine] + [org.onap.aai.cl.eelf LoggerFactory LogMessageEnum AaiLoggerAdapter AuditLogLine] + [org.onap.aai.cl.mdc MdcContext MdcOverride] + [com.att.eelf.i18n EELFResourceManager] + [clojure.lang.IFn] + [org.slf4j MDC])) + +(declare LOGLINE_DEFINED_FIELDS error-logger audit-logger ->java string->enum logfields) + +(defmethod ig/init-key :chameleon/loggers + [_ {:keys [logback logmsgs] :or {logback (.getPath (io/file "chameleon_logback.xml")) + logmsgs "log/ChameleonMsgs"}}] + (System/setProperty "logback.configurationFile" logback) + (EELFResourceManager/loadMessageBundle logmsgs) + [(error-logger "chameleon.loggging") (audit-logger "chameleon.loggging")]) + +(defn conform-multiple + [& spec-form-pair] + (if (s/valid? :chameleon.specs/spec-form-pair spec-form-pair) + (->> spec-form-pair + (partition 2) + (map (fn [[sp form]] + (when (s/invalid? (s/conform sp form)) + (s/explain-data sp form)))) + (remove nil?)) + (s/explain-data :chameleon.specs/spec-form-pair spec-form-pair))) + +(defn mdc-set! + "Sets the global MDC context for the current thread." + [m] + (doseq [[k v] m] (MDC/put k v))) + +(defn mdc-init! + "Sets the global MDC context as required by the EELF logging library" + [transaction-id service instance partner client-address] + (MdcContext/initialize transaction-id service instance partner client-address)) + +(defn mdc-clear! [] (MDC/clear)) + +(defmacro with-mdc + "Will set the global MDC context with the options (will convert from + keywords to PascalCase), execute the log, and clear the MDC + context." + [opts & body] + `(do + (mdc-set! (transform-keys cs/->PascalCaseString ~opts)) + ~@body + (mdc-clear))) + +(defn mdc-override + [m] + (->java (fn [j k v] (.addAttribute j k v)) (new MdcOverride) + (transform-keys cs/->PascalCaseString m))) + +(defn error-logger + [name] + (.getLogger (LoggerFactory/getInstance) name)) + +(defn audit-logger + [name] + (.getAuditLogger (LoggerFactory/getInstance) name)) + +(defn logger? + [logger] + (instance? AaiLoggerAdapter logger)) + +(defn info + [^AaiLoggerAdapter logger ^String enum msgs & {:keys [fields] :or {fields {}}}] + (let [confirmed-specs (conform-multiple :logging/valid-fields fields :logging/msgs msgs + :chameleon.specs/logger logger)] + (if (empty? confirmed-specs) + (.info logger (string->enum enum) (logfields fields) (into-array java.lang.String msgs)) + confirmed-specs))) + +(defn debug + [^AaiLoggerAdapter logger ^String enum msgs] + (.debug logger (string->enum enum) (into-array java.lang.String msgs))) + +(defn valid-logfields? + [m] + (->> (keys m) + (map cs/->SCREAMING_SNAKE_CASE_STRING) + set + (clojure.set/superset? (-> LOGLINE_DEFINED_FIELDS keys set)))) + +(def ^{:private true + :doc "Adding these fields from \"org.onap.aai.cl.api.LogLine\" + class. Right now there isn't a known way to use the ENUMs of an + abstract JAVA class if they are not STATIC. The field order is very + specific and it should be maintained for the common logging library + version \"1.2.2\". For a better understanding, please look at the + \"DefinedFields\" ENUMs in the org.onap.aai.cl.api.LogLine + class"} + + LOGLINE_DEFINED_FIELDS + (zipmap ["STATUS_CODE" "RESPONSE_CODE" "RESPONSE_DESCRIPTION" "INSTANCE_UUID" + "SEVERITY" "SERVER_IP" "CLIENT_IP" "CLASS_NAME" "PROCESS_KEY" + "TARGET_SVC_NAME" "TARGET_ENTITY" "ERROR_CODE" "ERROR_DESCRIPTION" + "CUSTOM_1" "CUSTOM_2""CUSTOM_3" "CUSTOM_4"] + (range))) + +(defn- string->enum + ([^String enum] (proxy [Enum LogMessageEnum] [enum (hash enum)])) + ([^String enum ordinal] (proxy [Enum LogMessageEnum] [enum ordinal]))) + +(defn- ->java + [f jc m] + (reduce (fn [a [k v]] (f a k v) a) + jc m)) + +(defn- logfields + "Generate a \"LogFields\" object with all the fields set." + [m] + (->> m + (transform-keys cs/->SCREAMING_SNAKE_CASE_STRING) + (transform-keys #(string->enum % (LOGLINE_DEFINED_FIELDS %))) + (->java (fn [j field v] (.setField j field v)) (new LogFields)))) diff --git a/src/chameleon/route.clj b/src/chameleon/route.clj index e8f3a3c..0918da4 100644 --- a/src/chameleon/route.clj +++ b/src/chameleon/route.clj @@ -1,12 +1,7 @@ (ns chameleon.route - (:require [org.httpkit.client :as kitclient])) - -(defn- interpret-response - "Print out the response from the Gallifrey server" - [key response] - (let [{:keys [status body]}@response] - (println "Response for request with key " key " resulted in status " status - " with body " body ))) + (:require [org.httpkit.client :as kitclient] + [chameleon.logging :as log] + [ring.util.http-status :as hs])) (defn query "Retrieve an entity referenced by id at the provided host. Optionally provide @@ -21,11 +16,10 @@ :keepalive 300 :timeout 20000})) -(defn assert-create +(defn assert-create! "Creates an entity in Gallifrey with an initial set of assertions coming from the provided payload" [host actor type key payload & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :put :query-params (into {"actor" actor "create" "true"} time-dimensions) :body payload @@ -33,11 +27,10 @@ :keepalive 300 :timeout 1000})) -(defn assert-update +(defn assert-update! "Update an entity in Gallifrey with a set of assertions coming from the provided payload" [host actor type key payload & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :put :query-params (into {"actor" actor "changes-only" "true"} time-dimensions) :body payload @@ -45,26 +38,27 @@ :keepalive 300 :timeout 1000})) -(defn assert-delete +(defn assert-delete! "Assert a deletion for an entity in Gallifrey based on the provided key." [host actor type key & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :delete - :query-params (into {"actor" actor} time-dimensions) + :query-params (into {"actor" actor} time-dimensions) :insecure? true :keepalive 300 :timeout 1000})) -(defn assert-gallifrey [host actor type payload] +(defn assert-gallifrey! + [host actor type payload & [e-logger a-logger]] "Propagates an assertion to Gallifrey based off of an event payload coming in from the event service." (let [{:keys [meta body]} payload - {:keys [key operation time]} meta] - (println operation " " type " with key " key) - (interpret-response key (case operation - "CREATE" - (assert-create host actor type key body time) - "UPDATE" - (assert-update host actor type key body time) - "DELETE" - (assert-delete host actor type key time))))) + {:keys [key operation time]} meta + _ (log/info e-logger "GALLIFREY_ASSERTION" (mapv str [operation type key])) + g-assert (case operation + "CREATE" (assert-create! host actor type key body time) + "UPDATE" (assert-update! host actor type key body time) + "DELETE" (assert-delete! host actor type key time)) + {:keys [status body]} @g-assert] + (log/info e-logger "GALLIFREY_ASSERTED" [(str type) (str key)]) + (log/info a-logger "RESPONSE" (mapv str [operation key status body]) + :fields {:response-code status :response-description (hs/get-name status)}))) diff --git a/src/chameleon/specs.clj b/src/chameleon/specs.clj new file mode 100644 index 0000000..40d5768 --- /dev/null +++ b/src/chameleon/specs.clj @@ -0,0 +1,71 @@ +(ns chameleon.specs + (:require [clojure.spec.alpha :as s] + [chameleon.logging :as log] + [clojure.spec.gen.alpha :as gen] + [cheshire.core :as c] + [clojure.string :as str])) + +(s/def ::host string?) +(s/def ::provenance string?) +(s/def ::payload map?) +(s/def ::string (s/spec (s/and string? (complement str/blank?)) :gen gen/string-alphanumeric)) +(s/def ::type (s/spec ::string :gen #(gen/elements ["vserver" "pserver" "generic-vnf"]))) +(s/def ::id uuid?) +(s/def ::source (s/keys :req-un [::type ::id])) +(s/def ::target (s/keys :req-un [::type ::id])) +(s/def ::_id ::id) +(s/def ::spec-form-pair (s/* (s/cat :spec #(% (s/registry)) :form any?))) + +;; spike event +(s/def :spike/transaction-id uuid?) +(s/def :spike/schema-version (s/spec #(re-matches #"v\d+" %) :gen #(->> (s/int-in 7 15) + s/gen + (gen/fmap (partial str "v"))))) +(s/def :spike/key uuid?) +(s/def :spike/name ::string) +(s/def :spike/last-mod-source-of-truth ::string) +(s/def :spike/source-of-truth ::string) +(s/def :spike/vserver-selflink ::string) +(s/def :spike/is-closed-loop-disabled boolean?) +(s/def :spike/in-maint boolean?) +(s/def :spike/timestamp inst?) +(s/def :spike/operation (s/spec ::string :gen #(gen/elements ["UPDATE" "CREATE" "DELETE"]))) +(s/def :spike/properties (s/keys :req-un [:spike/name ::id :spike/last-mod-source-of-truth + :spike/source-of-truth :spike/vserver-selflink + :spike/is-closed-loop-disabled :spike/in-maint])) +(s/def :spike/vertex (s/keys :req-un [:spike/schema-version ::type :spike/key :spike/properties])) +(s/def :spike/event (s/keys :req-un [:spike/transaction-id :spike/vertex + :spike/operation :spike/timestamp])) +(s/def :spike/payload (s/spec string? :gen #(gen/fmap (partial c/generate-string) + (s/gen :spike/event)))) +(s/def :gallifrey/k-end-actor ::string) + +;; gallifrey response +(s/def :gallifrey/k-start-actor ::string) +(s/def :gallifrey/k-end inst?) +(s/def :gallifrey/k-start inst?) +(s/def :gallifrey/history (s/keys :req-un [:gallifrey/k-end-actor :gallifrey/k-end + :gallifrey/k-start-actor :gallifrey/k-start])) +(s/def :relationship/_meta (s/map-of ::string :gallifrey/history)) +(s/def :relationship/_type (s/spec string? :gen #(gen/return "relationship"))) +(s/def :gallifrey/_type (s/spec ::string :gen #(gen/return "entity"))) +(s/def :gallifrey/properties (s/keys :req-un [:gallifrey/_type ::type])) +(s/def :relationship/type (s/spec string? :gen #(->> (gen/string-alphanumeric) + (gen/such-that (complement str/blank?)) + (gen/fmap (partial str "tosca.relationship."))))) +(s/def :relationship/properties (s/keys :req-un [:relationship/_type :relationship/type])) +(s/def ::relationship (s/keys :req-un [:relationship/properties ::source + ::target :relationship/_meta ::_id])) +(s/def ::relationships (s/coll-of ::relationship :gen-max 8)) +(s/def :gallifrey/payload (s/spec map? + :gen #(->> [::_id :gallifrey/properties + :gallifrey/properties ::relationships] + (s/keys :req-un) + s/gen + (gen/fmap (partial clojure.walk/stringify-keys))))) + +;; Logger specs +(s/def ::logger (s/spec log/logger? :gen #(gen/return (log/error-logger "chameleon.specs")))) +(s/def ::loggers (s/cat :e :chameleon.specs/logger :a :chameleon.specs/logger)) +(s/def :logging/msgs (s/* ::string)) +(s/def :logging/valid-fields log/valid-logfields?) diff --git a/test/chameleon/testing.clj b/test/chameleon/testing.clj new file mode 100644 index 0000000..13acb1a --- /dev/null +++ b/test/chameleon/testing.clj @@ -0,0 +1,42 @@ +(ns chameleon.testing + (:require [clojure.test :as t] + [clojure.spec.alpha :as s] + [clojure.spec.gen.alpha :as gen] + [clojure.spec.test.alpha :as st] + [chameleon.logging :as log] + [chameleon.specs :as cs] + [chameleon.route :as cr] + [chameleon.aai-processor :as cai])) + +(s/fdef chameleon.route/assert-gallifrey! + :args (s/cat :host :chameleon.specs/host + :provenance :chameleon.specs/provenance + :type :chameleon.specs/type + :payload :chameleon.specs/payload + :loggers :chameleon.specs/loggers) + :ret nil?) + +(s/fdef chameleon.aai-processor/from-spike + :args (s/cat :gallifrey-host :chameleon.specs/host + :payload :spike/payload + :loggers :chameleon.specs/loggers) + :ret nil?) + +(s/fdef chameleon.aai-processor/from-gallifrey + :args (s/cat :body :gallifrey/payload) + :ret map?) + +(s/fdef chameleon.aai-processor/gen-trim-relationship + :args (s/cat :relationship :chameleon.specs/relationship) + :ret map?) + +(st/instrument 'chameleon.route/assert-gallifrey! {:stub '(chameleon.route/assert-gallifrey!)}) + +;; Testing instrumentation +(chameleon.route/assert-gallifrey! "host" "aai" "type" {} (log/error-logger "chameleon.testing") (log/audit-logger "chameleon.testing")) + +(->> '(chameleon.aai-processor/from-spike + chameleon.aai-processor/from-gallifrey + chameleon.aai-processor/gen-trim-relationship) + st/check + st/summarize-results) |