diff options
author | Shwetank Dave <shwetank.dave@amdocs.com> | 2018-06-11 10:15:34 -0400 |
---|---|---|
committer | Shwetank Dave <shwetank.dave@amdocs.com> | 2018-07-26 09:13:18 -0400 |
commit | e16bda37d76e63e0f903bba13ed1dccf3b17f395 (patch) | |
tree | 1b8036103cf598bc645e3d4da1b6340d9c164447 | |
parent | 36b5671af2c3eec5ca81663382c4ca2898f79e55 (diff) |
Add logging and tests and build using mvn
An initial version of adding logs to chameleon
An initial version of adding specs (tests) to chameleon.
Adding pom.xml so the project can be build using maven.
Updating README.md for instructions on running it locally.
Issue-ID: AAI-1220
Change-Id: I85f46fd7f625c83b84f211d6766970431e6d91eb
Signed-off-by: Shwetank Dave <shwetank.dave@amdocs.com>
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | .gitreview | 2 | ||||
-rw-r--r-- | README.md | 76 | ||||
m--------- | chameleon | 5 | ||||
-rw-r--r-- | dev/dev.clj | 10 | ||||
-rw-r--r-- | devops/chameleon/build-chameleon | 15 | ||||
-rw-r--r-- | pom.xml | 305 | ||||
-rw-r--r-- | prod/chameleon/server.clj | 12 | ||||
-rw-r--r-- | project.clj | 42 | ||||
-rw-r--r-- | resources/chameleon_logback.xml | 109 | ||||
-rw-r--r-- | resources/log/ChameleonMsgs.properties | 19 | ||||
-rw-r--r-- | src/chameleon/aai_processor.clj | 57 | ||||
-rw-r--r-- | src/chameleon/config.clj | 8 | ||||
-rw-r--r-- | src/chameleon/event.clj | 22 | ||||
-rw-r--r-- | src/chameleon/handler.clj | 37 | ||||
-rw-r--r-- | src/chameleon/logging.clj | 123 | ||||
-rw-r--r-- | src/chameleon/route.clj | 50 | ||||
-rw-r--r-- | src/chameleon/specs.clj | 71 | ||||
-rw-r--r-- | test/chameleon/testing.clj | 42 |
19 files changed, 902 insertions, 106 deletions
@@ -1,8 +1,9 @@ # Created by https://www.gitignore.io/api/clojure,emacs,osx,java ### Clojure ### -pom.xml pom.xml.asc +chameleon.iml +.idea *jar lib/ classes/ @@ -1,4 +1,4 @@ [gerrit] -host=gerrit.openecomp.org +host=gerrit.onap.org port=29418 project=chameleon.git @@ -1,3 +1,77 @@ -# chameleon +# CHAMELEON Feeds the Gallifrey time database with entity events + +### Building the project + +1. mvn clean install +2. mvn package + +### Generating a code coverage report + +1. mvn clean install +2. mvn exec:java -Dexec.classpathScope=test -Dexec.mainClass='clojure.main' -Dexec.args="--main cloverage.coverage -n 'chameleon.*' -t 'chameleon.testing'" + + 2.1. Follow the instructions to view the report. + +3. To tweak the report generation, follow the instrustions here: + https://github.com/cloverage/cloverage + +### Developing + +1. I use Leiningen for building my clojure project and for dependency + management. You can read about Leiningen and follow the tutorial at + the link: + https://github.com/technomancy/leiningen + +2. If you make ** changes to the project.clj file, you are required to + re-generate the pom.xml by running `lein pom`** (requires leiningen + to be instaled). + + You can also add your changes to the pom.xml file manually. + +3. I use Emacs as my editor and CIDER for interactive development in + Clojure. + - CIDER: https://github.com/clojure-emacs/cider + - Emacs: https://www.gnu.org/software/emacs/ + + - Here is a tutorial on Emacs: https://www.gnu.org/software/emacs/tour/ + - Here is the emacs configuration I've used, it includes the CIDER + package: + https://github.com/sandhu/emacs.d + +4. Start nrepl `C-c M-j` (you have to be in any *.clj file in the + chameleon project). If the cider-repl buffer doesn't open + automatically, use `C-x b` and use the arrow keys to find the + `cider-repl chameleon` buffer and press enter. + +5. Open the dev.clj file by `C-x C-f`, load it `C-c C-k` and switch + your namespace to dev `C-c M-n`. + +6. In the repl buffer you should see that the namespace went from `user` + to `dev`. + +7. Make sure the configuration is correct in the dev.clj file and once + verified, in the repl buffer type `(go)` and press enter. + +8. You can verify everthing is running by generating/adding an event + to dmaap and seeing it flow through chameleon in the error.log file. + +### Running it locally (Assumming you're not using Emacs and using + `lein repl` from the command line) + +**Make you're in the directory at the root of your project. Open the + file `chameleon/dev.clj` and update the config to the correct + dmaap host, topic, and make sure you're using the correct + consumer group and id.** + +1. On the command line, execute `lein repl` + +2. Once in the clojure repl, load the dev namespace + (load "dev") + +3. Go into the dev namespace + (in-ns 'dev) + +4. Run the following command in the repl + (go) diff --git a/chameleon b/chameleon deleted file mode 160000 -Subproject 2610df2fc29a9d16bb869f652d4d21caeae4c15 diff --git a/dev/dev.clj b/dev/dev.clj index 880f540..51d0fdf 100644 --- a/dev/dev.clj +++ b/dev/dev.clj @@ -4,19 +4,19 @@ (:require [chameleon.config :refer [config]] [chameleon.handler :refer [handler]] [integrant.core :as ig] + [chameleon.logging :as log] + [chameleon.specs :as sp] [integrant.repl :refer [clear go halt init reset reset-all]] [integrant.repl.state :refer [system]] [clojure.tools.namespace.repl :refer [refresh refresh-all disable-reload!]] [clojure.repl :refer [apropos dir doc find-doc pst source]] [clojure.test :refer [run-tests run-all-tests]] - [clojure.pprint :refer [pprint]] - [clojure.reflect :refer [reflect]])) + [clojure.pprint :refer [pprint]])) (disable-reload! (find-ns 'integrant.core)) -(integrant.repl/set-prep! (constantly (config { - :event-config {:aai {:host "localhost:3904" - :topic "spikeEvents" +(integrant.repl/set-prep! (constantly (config {:event-config {:aai {:host "localhost:3904" + :topic "events" :motsid "" :pass "" :consumer-group"chameleon1" diff --git a/devops/chameleon/build-chameleon b/devops/chameleon/build-chameleon index 4421407..c588e95 100644 --- a/devops/chameleon/build-chameleon +++ b/devops/chameleon/build-chameleon @@ -1,2 +1,13 @@ -lein uberjar -cp -f ../../target/chameleon.jar . +!#/bin/bash +ls build-chameleon +if [ $? -eq 0 ]; then + echo "Building Chameleon" +else + echo "FAILED. Make sure you're in the same directory as the \"build-chameleon\" file." + exit 1 +fi +pushd . +cd ../../ +mvn -X clean package +popd +cp -f ../../target/chameleon-0.1.0-jar-with-dependencies.jar ./chameleon.jar @@ -0,0 +1,305 @@ +<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>chameleon</groupId> + <artifactId>chameleon</artifactId> + <packaging>jar</packaging> + <version>0.1.0</version> + <name>chameleon</name> + <description/> + <scm> + <tag>36b5671af2c3eec5ca81663382c4ca2898f79e55 +</tag> + <url/> + </scm> + <build> + <sourceDirectory>src</sourceDirectory> + <testSourceDirectory>test</testSourceDirectory> + <resources> + <resource> + <directory>resources</directory> + </resource> + </resources> + <testResources> + <testResource> + <directory>resources</directory> + </testResource> + </testResources> + <directory>target</directory> + <outputDirectory>target/classes</outputDirectory> + <plugins> + <plugin> + <groupId>com.theoryinpractise</groupId> + <artifactId>clojure-maven-plugin</artifactId> + <version>1.3.13</version> + <extensions>true</extensions> + <configuration> + <sourceDirectories> + <sourceDirectory>src</sourceDirectory> + <sourceDirectory>prod</sourceDirectory> + <sourceDirectory>test</sourceDirectory> + </sourceDirectories> + </configuration> + <executions> + <execution> + <id>compile</id> + <goals> + <goal>compile</goal> + </goals> + <phase>compile</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + <configuration> + <archive> + <manifest> + <addClasspath>true</addClasspath> + <mainClass>chameleon.server</mainClass> + <classpathPrefix>dependency</classpathPrefix> + </manifest> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> + <executions> + <execution> + <id>copy-dependencies</id> + <goals> + <goal>copy-dependencies</goal> + </goals> + <phase>package</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.4.1</version> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <archive> + <manifest> + <mainClass>chameleon.server</mainClass> + </manifest> + </archive> + </configuration> + <executions> + <execution> + <id>assemble</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>prod</source> + <source>test</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <repositories> + <repository> + <id>central</id> + <url>https://repo1.maven.org/maven2/</url> + <snapshots> + <enabled>false</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>clojars</id> + <url>https://clojars.org/repo/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-releases</id> + <url>https://nexus.onap.org/content/repositories/releases/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-public</id> + <url>https://nexus.onap.org/content/repositories/public/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-staging</id> + <url>https://nexus.onap.org/content/repositories/staging/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + <repository> + <id>onap-snapshot</id> + <url>https://nexus.onap.org/content/repositories/snapshots/</url> + <snapshots> + <enabled>true</enabled> + </snapshots> + <releases> + <enabled>true</enabled> + </releases> + </repository> + </repositories> + <dependencyManagement> + <dependencies/> + </dependencyManagement> + <dependencies> + <dependency> + <groupId>org.clojure</groupId> + <artifactId>clojure</artifactId> + <version>1.9.0</version> + </dependency> + <dependency> + <groupId>com.7theta</groupId> + <artifactId>utilis</artifactId> + <version>1.0.4</version> + </dependency> + <dependency> + <groupId>http-kit</groupId> + <artifactId>http-kit</artifactId> + <version>2.2.0</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-core</artifactId> + <version>1.6.3</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-defaults</artifactId> + <version>0.3.1</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-anti-forgery</artifactId> + <version>1.1.0</version> + </dependency> + <dependency> + <groupId>compojure</groupId> + <artifactId>compojure</artifactId> + <version>1.6.0</version> + </dependency> + <dependency> + <groupId>liberator</groupId> + <artifactId>liberator</artifactId> + <version>0.15.1</version> + </dependency> + <dependency> + <groupId>cheshire</groupId> + <artifactId>cheshire</artifactId> + <version>5.7.1</version> + </dependency> + <dependency> + <groupId>inflections</groupId> + <artifactId>inflections</artifactId> + <version>0.13.0</version> + </dependency> + <dependency> + <groupId>clj-time</groupId> + <artifactId>clj-time</artifactId> + <version>0.14.2</version> + </dependency> + <dependency> + <groupId>integrant</groupId> + <artifactId>integrant</artifactId> + <version>0.6.2</version> + </dependency> + <dependency> + <groupId>yogthos</groupId> + <artifactId>config</artifactId> + <version>0.9</version> + </dependency> + <dependency> + <groupId>org.onap.aai.event-client</groupId> + <artifactId>event-client-dmaap</artifactId> + <version>1.2.1</version> + </dependency> + <dependency> + <groupId>org.onap.aai.logging-service</groupId> + <artifactId>common-logging</artifactId> + <version>1.2.2</version> + </dependency> + <dependency> + <groupId>camel-snake-kebab</groupId> + <artifactId>camel-snake-kebab</artifactId> + <version>0.4.0</version> + </dependency> + <dependency> + <groupId>metosin</groupId> + <artifactId>ring-http-response</artifactId> + <version>0.9.0</version> + </dependency> + <dependency> + <groupId>org.clojure</groupId> + <artifactId>test.check</artifactId> + <version>0.9.0</version> + </dependency> + <dependency> + <groupId>cloverage</groupId> + <artifactId>cloverage</artifactId> + <version>1.0.10</version> + </dependency> + <dependency> + <groupId>ring</groupId> + <artifactId>ring-devel</artifactId> + <version>1.6.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>integrant</groupId> + <artifactId>repl</artifactId> + <version>0.2.0</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> + +<!-- This file was autogenerated by Leiningen. + Please do not edit it directly; instead edit project.clj and regenerate it. + It should not be considered canonical data. For more information see + https://github.com/technomancy/leiningen --> diff --git a/prod/chameleon/server.clj b/prod/chameleon/server.clj index b9b38db..12c2c02 100644 --- a/prod/chameleon/server.clj +++ b/prod/chameleon/server.clj @@ -3,16 +3,18 @@ [chameleon.handler :refer [handler]] [config.core :refer [env]] [org.httpkit.server :refer [run-server]] - [integrant.core :as ig]) + [integrant.core :as ig] + [chameleon.specs :as cs]) (:gen-class)) (defn -main [& args] (let [port (Integer/parseInt (or (env :http-port) "8082")) system-config (read-string (slurp (System/getenv "CONFIG_LOCATION" ))) event-config (:event-config system-config) - route-config (:gallifrey-host system-config)] + route-config (:gallifrey-host system-config) + log-config (:log-config system-config)] (println "Listening on port" port) - (ig/init (config { - :event-config event-config + (ig/init (config {:event-config event-config :gallifrey-host route-config - :http-port port})))) + :http-port port + :log-config log-config})))) diff --git a/project.clj b/project.clj index 044856b..430173c 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,5 @@ (defproject chameleon "0.1.0" - :dependencies [[org.clojure/clojure "1.8.0"] + :dependencies [[org.clojure/clojure "1.9.0"] [com.7theta/utilis "1.0.4"] [http-kit "2.2.0"] [ring/ring-core "1.6.3"] @@ -11,15 +11,18 @@ [inflections "0.13.0"] [clj-time "0.14.2"] [integrant "0.6.2"] - [clojure-future-spec "1.9.0-beta4"] [yogthos/config "0.9"] [org.onap.aai.event-client/event-client-dmaap "1.2.1"] - ] + [org.onap.aai.logging-service/common-logging "1.2.2"] + [camel-snake-kebab "0.4.0"] + [metosin/ring-http-response "0.9.0"] + [org.clojure/test.check "0.9.0"] + [cloverage/cloverage "1.0.10"]] + :plugins [[lein-cloverage "1.0.10"]] :repositories [["onap-releases" {:url "https://nexus.onap.org/content/repositories/releases/"}] ["onap-public" {:url "https://nexus.onap.org/content/repositories/public/"}] ["onap-staging" {:url "https://nexus.onap.org/content/repositories/staging/"}] - ["onap-snapshot" {:url "https://nexus.onap.org/content/repositories/snapshots/"}] - ] + ["onap-snapshot" {:url "https://nexus.onap.org/content/repositories/snapshots/"}]] :min-lein-version "2.5.3" :profiles {:dev {:source-paths ["dev"] :dependencies [[ring/ring-devel "1.6.3"] @@ -28,4 +31,31 @@ :main chameleon.server :aot [chameleon.server] :uberjar-name "chameleon.jar"}} - :prep-tasks [ "compile"]) + :prep-tasks ["compile"] + :source-paths ["src" "prod" "test"] + :resource-paths ["resources"] + :pom-plugins [[com.theoryinpractise/clojure-maven-plugin "1.3.13" + {:extensions "true" + :configuration [:sourceDirectories + [:sourceDirectory "src"] + [:sourceDirectory "prod"] + [:sourceDirectory "test"]] + :executions ([:execution [:id "compile"] + [:goals ([:goal "compile"])] + [:phase "compile"]])}] + [org.apache.maven.plugins/maven-jar-plugin "2.4" + {:configuration [:archive [:manifest + [:addClasspath true] + [:mainClass "chameleon.server"] + [:classpathPrefix "dependency"]]]}] + [org.apache.maven.plugins/maven-dependency-plugin "2.8" + {:executions ([:execution [:id "copy-dependencies"] + [:goals ([:goal "copy-dependencies"])] + [:phase "package"]])}] + [org.apache.maven.plugins/maven-assembly-plugin "2.4.1" + {:configuration ([:descriptorRefs [:descriptorRef "jar-with-dependencies"]] + [:archive [:manifest + [:mainClass "chameleon.server"]]]) + :executions ([:execution [:id "assemble"] + [:phase "package"] + [:goals ([:goal "single"])]])}]]) diff --git a/resources/chameleon_logback.xml b/resources/chameleon_logback.xml new file mode 100644 index 0000000..5b723d7 --- /dev/null +++ b/resources/chameleon_logback.xml @@ -0,0 +1,109 @@ +<configuration scan="true" scanPeriod="3 seconds" debug="false"> + <!--<jmxConfigurator /> --> + <!-- directory path for all other type logs --> + + <property name="logDir" value="/opt/chameleon/logs" /> + + <!-- specify the component name --> + <property name="componentName" value="AAI-CHAMELEON" /> + + <!-- default eelf log file names --> + <property name="generalLogName" value="error" /> + <property name="metricsLogName" value="metrics" /> + <property name="auditLogName" value="audit" /> + <property name="debugLogName" value="debug" /> + + <property name="errorLogPattern" value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%mdc{RequestId}|%thread|DataRouter|%mdc{PartnerName}|%logger||%.-5level|%msg%n" /> + + <property name="auditMetricPattern" value="%msg%n" /> + + <property name="logDirectory" value="${logDir}/${componentName}" /> + + <!-- ============================================================================ --> + <!-- EELF Appenders --> + <!-- ============================================================================ --> + + <!-- The EELFAppender is used to record events to the general application + log --> + + <appender name="EELF" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${generalLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip + </fileNamePattern> + <maxHistory>60</maxHistory> + </rollingPolicy> + <encoder> + <pattern>${errorLogPattern}</pattern> + <outputPatternAsHeader>true</outputPatternAsHeader> + </encoder> + </appender> + + <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender"> + <!-- deny all events with a level below INFO, that is TRACE and DEBUG --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + <queueSize>256</queueSize> + <appender-ref ref="EELF" /> + </appender> + + + <!-- EELF Audit Appender. This appender is used to record audit engine + related logging events. The audit logger and appender are specializations + of the EELF application root logger and appender. This can be used to segregate + Policy engine events from other components, or it can be eliminated to record + these events as part of the application root log. --> + + <appender name="EELFAudit" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${auditLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip + </fileNamePattern> + <maxHistory>60</maxHistory> + </rollingPolicy> + <encoder> + <outputPatternAsHeader>true</outputPatternAsHeader> + <pattern>${auditMetricPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFAudit" /> + </appender> + <!-- ============================================================================ --> + <!-- EELF loggers --> + <!-- ============================================================================ --> + + <!-- CRUD Service loggers --> + <logger name="chameleon" level="INFO"> + <appender-ref ref="asyncEELF" /> + </logger> + + <logger name="com.att.eelf.audit" level="info" additivity="false"> + <appender-ref ref="asyncEELFAudit" /> + </logger> + + <!-- Other Loggers that may help troubleshoot --> + <logger name="net.sf" level="WARN" /> + <logger name="org.apache" level="WARN" /> + <logger name="org.apache.commons.httpclient" level="WARN" /> + <logger name="org.apache.commons" level="WARN" /> + <logger name="org.apache.coyote" level="WARN" /> + <logger name="org.apache.jasper" level="WARN" /> + + <!-- logback internals logging --> + <logger name="ch.qos.logback.classic" level="WARN" /> + <logger name="ch.qos.logback.core" level="WARN" /> + + <root> + <!-- <appender-ref ref="asyncEELF" /> --> + <!-- <appender-ref ref="asyncEELFDebug" /> --> + </root> + +</configuration> diff --git a/resources/log/ChameleonMsgs.properties b/resources/log/ChameleonMsgs.properties new file mode 100644 index 0000000..4e61e2a --- /dev/null +++ b/resources/log/ChameleonMsgs.properties @@ -0,0 +1,19 @@ +EVENT_PROCESSOR=\ +CHM0001I|\ +Event Processor {0}\ + +GALLIFREY_ASSERTION=\ +CHM0002I|\ +Attempt Gallifrey Assertion of operation {0} with type {1} and key {2} \ + +GALLIFREY_ASSERTED=\ +CHM0002I|\ +Asserted type {0} and key {1} in Gallifrey\ + +RESPONSE=\ +CHM0003I|\ +Response for operation "{0}" for endpoint "{1}" resulted in status "{2}" with body "{3}"\ + +CHAMELEON_REQUEST=\ +CHM0004I|\ +Incoming Request of type "{0}" for endpoint "{1}" from address "{2}"\
\ No newline at end of file diff --git a/src/chameleon/aai_processor.clj b/src/chameleon/aai_processor.clj index d1a7d25..4243b39 100644 --- a/src/chameleon/aai_processor.clj +++ b/src/chameleon/aai_processor.clj @@ -1,9 +1,8 @@ (ns chameleon.aai-processor - (:require - [chameleon.route :refer :all] - [cheshire.core :refer :all] - [integrant.core :as ig] - [clojure.set :refer :all])) + (:require [chameleon.route :refer :all] + [cheshire.core :refer :all] + [integrant.core :as ig] + [clojure.set :refer :all])) (defonce ^:private p-attr (atom nil)) (defonce ^:private t-attr (atom nil)) @@ -33,8 +32,7 @@ {"id" id "type" type "source" {"id" src-id "type" src-type} - "target" {"id" target-id "type" target-type}}) - ) + "target" {"id" target-id "type" target-type}})) (defn from-gallifrey "Transforms Gallifrey response payloads into a format consumable by AAI-centric clients" @@ -42,27 +40,21 @@ (let [resource-type (get-in body ["properties" "_type"]) id (body "_id") type (get-in body ["properties" "type"]) - properties (body "properties")] + properties (body "properties") + entity-response {"id" id + "type" type + "properties" (dissoc properties "_type" "type")}] (if (= resource-type "entity") - ; Transform into an entity type + ;; Transform into an entity type (let [relationships (body "relationships")] - { - "id" id - "type" type - "properties" (dissoc properties "_type" "type") - "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships))) - "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships))) - }) - ; Transform into a relationship type - { - "id" id - "type" type - "properties" (dissoc properties "_type" "type") - }))) + (assoc entity-response + "in" (into [] (map gen-trim-relationship (filter #(= (get-in % ["target" "id"]) id) relationships))) + "out" (into [] (map gen-trim-relationship (filter #(= (get-in % ["source" "id"]) id) relationships))))) + entity-response))) (defn from-spike "Transforms Spike-based event payloads to a format accepted by Gallifrey for vertices and relationships" - [gallifrey-host payload] + [gallifrey-host payload & [error-logger audit-logger]] (let [txpayload (map-keywords (parse-string payload)) operation (:operation txpayload) parse-type (if (contains? txpayload :vertex) @@ -74,13 +66,14 @@ entity (map-keywords (parse-type txpayload)) key (:key entity) properties (assoc (:properties entity) :type (:type entity)) - truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)}) - assertion {:meta {:key key - :operation operation - :time truth-time}} + truth-time (if (not (nil? (get properties @t-attr))) {:t-t (get properties @t-attr)}) + assertion {:meta {:key key + :operation operation + :time truth-time}} provenance (get properties @p-attr "aai")] - (assert-gallifrey gallifrey-host provenance (name entity-type) (if (= entity-type :entity) - (assoc assertion :body (generate-string {:properties properties})) - (assoc assertion :body (generate-string (conj {:properties properties} - {:source (rename-keys (:source entity) {"key" "id"})} - {:target (rename-keys (:target entity) {"key" "id"})}))))))) + (assert-gallifrey! gallifrey-host provenance (name entity-type) + (if (= entity-type :entity) + (assoc assertion :body (generate-string {:properties properties})) + (assoc assertion :body (generate-string (conj {:properties properties} + {:source (rename-keys (:source entity) {"key" "id"})} + {:target (rename-keys (:target entity) {"key" "id"})})))) error-logger audit-logger))) diff --git a/src/chameleon/config.clj b/src/chameleon/config.clj index bd99096..10324e2 100644 --- a/src/chameleon/config.clj +++ b/src/chameleon/config.clj @@ -4,14 +4,16 @@ (defn config [app-config] - (let [conf { + (let [conf {:chameleon/loggers (:log-config app-config) :chameleon/event {:event-config (assoc-in (:event-config app-config) [:aai :processor] from-spike) - :gallifrey-host (:gallifrey-host app-config)} + :gallifrey-host (:gallifrey-host app-config) + :loggers (ig/ref :chameleon/loggers)} :chameleon/handler {:gallifrey-host (:gallifrey-host app-config) - :gallifrey-transformer from-gallifrey} + :gallifrey-transformer from-gallifrey + :loggers (ig/ref :chameleon/loggers)} :chameleon/aai-processor {:provenance-attr "last-mod-source-of-truth" :truth-attr "truth-time"} diff --git a/src/chameleon/event.clj b/src/chameleon/event.clj index c4bec8e..92f4211 100644 --- a/src/chameleon/event.clj +++ b/src/chameleon/event.clj @@ -1,19 +1,25 @@ (ns chameleon.event (:require [integrant.core :as ig] - [clojure.string :refer [starts-with?]]) + [clojure.string :refer [starts-with?]] + [chameleon.logging :as log]) (:import [org.onap.aai.event.client DMaaPEventConsumer])) (defmethod ig/init-key :chameleon/event - [_ {:keys [event-config gallifrey-host]}] + [_ {:keys [event-config gallifrey-host loggers]}] (let [{:keys [host topic motsid pass consumer-group consumer-id timeout batch-size type processor]} (:aai event-config) + [error-logger audit-logger] loggers event-processor (DMaaPEventConsumer. host topic motsid pass consumer-group consumer-id timeout batch-size type)] - (println "Event processor for AAI created. Starting event polling on " host topic) + (log/info error-logger "EVENT_PROCESSOR" [(format "AAI created. Starting event polling on %s %s" host topic) ]) (.start (Thread. (fn [] (while true (let [it (.iterator (.consume event-processor))] - (println "Polling...") + (log/info error-logger "EVENT_PROCESSOR" ["Polling ..."]) (while (.hasNext it) + (log/mdc-init! "SPIKE-EVENT" "CHAMELEON" "" "" gallifrey-host) (try (let [event (.next it)] - (if (not (starts-with? event "DMAAP")) ;Temporarily added for current version of dmaap client - (processor gallifrey-host event))) - (catch Exception e (println (str "Unexpected exception during processing: " (.getMessage e))))))))))) - )) + ;;Temporarily added for current version of dmaap client + (when-not (starts-with? event "DMAAP") + (log/info error-logger "EVENT_PROCESSOR" [event]) + (processor gallifrey-host event error-logger audit-logger) + (log/mdc-clear!))) + (catch Exception e + (println (str "Unexpected exception during processing: " (.getMessage e))))))))))))) diff --git a/src/chameleon/handler.clj b/src/chameleon/handler.clj index fd97bf1..675a34b 100644 --- a/src/chameleon/handler.clj +++ b/src/chameleon/handler.clj @@ -10,17 +10,18 @@ [ring.middleware.session :refer [wrap-session]] [cheshire.core :as json] [clj-time.format :as tf] - [integrant.core :as ig])) + [integrant.core :as ig] + [chameleon.logging :as log])) (declare handler) (defonce ^:private g-host (atom nil)) (defonce ^:private g-transformer nil) -(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host gallifrey-transformer]}] +(defmethod ig/init-key :chameleon/handler [_ {:keys [gallifrey-host loggers gallifrey-transformer]}] (reset! g-host gallifrey-host) (def g-transformer gallifrey-transformer) - handler) + (handler loggers)) (defmethod ig/halt-key! :chameleon/handler [_ _] (reset! g-host nil) @@ -32,10 +33,10 @@ :allowed-methods [:get] :available-media-types ["application/json"] :exists? (fn [ctx] - (let [resource (c-route/query @g-host id type (-> ctx - :request - :params - (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys + (let [resource (c-route/query @g-host id type (-> ctx + :request + :params + (select-keys [:t-t :t-k])))] ; Only pass through the allowable set of keys (when (= (:status resource) 200) {::resource (-> resource :body @@ -56,9 +57,27 @@ (GET "/relationship/:id" [id] (resource-endpoint "relationship" id)) (resources "/")) -(def handler +(defn log-reqs + [handler loggers] + (let [[error-logger audit-logger] loggers] + (fn [request] + (log/mdc-init! (get-in request [:headers "X-TransactionId"]) "CHAMELEON" + "CHAMELEON_SERVICE" "ONAP" (:remote-addr request)) + (log/info error-logger "CHAMELEON_REQUEST" (mapv str ((juxt (comp name :request-method) :uri :remote-addr) request))) + (let [resp (handler request) + fields (->> ((juxt :status :body) resp) + (into ((juxt (comp name :request-method) :uri) request)) + (mapv str))] + (log/info error-logger "RESPONSE" fields) + (log/info audit-logger "RESPONSE" fields) + (log/mdc-clear!) + resp)))) + +(defn handler + [loggers] (-> app-routes - (wrap-defaults api-defaults))) + (wrap-defaults api-defaults) + (log-reqs loggers))) ;;; Implementation diff --git a/src/chameleon/logging.clj b/src/chameleon/logging.clj new file mode 100644 index 0000000..15e1e7b --- /dev/null +++ b/src/chameleon/logging.clj @@ -0,0 +1,123 @@ +(ns chameleon.logging + (:require [camel-snake-kebab.core :as cs] + [camel-snake-kebab.extras :refer [transform-keys]] + [clojure.java.io :as io] + [integrant.core :as ig] + [clojure.spec.alpha :as s]) + (:import [org.onap.aai.cl.api Logger LogFields LogLine] + [org.onap.aai.cl.eelf LoggerFactory LogMessageEnum AaiLoggerAdapter AuditLogLine] + [org.onap.aai.cl.mdc MdcContext MdcOverride] + [com.att.eelf.i18n EELFResourceManager] + [clojure.lang.IFn] + [org.slf4j MDC])) + +(declare LOGLINE_DEFINED_FIELDS error-logger audit-logger ->java string->enum logfields) + +(defmethod ig/init-key :chameleon/loggers + [_ {:keys [logback logmsgs] :or {logback (.getPath (io/file "chameleon_logback.xml")) + logmsgs "log/ChameleonMsgs"}}] + (System/setProperty "logback.configurationFile" logback) + (EELFResourceManager/loadMessageBundle logmsgs) + [(error-logger "chameleon.loggging") (audit-logger "chameleon.loggging")]) + +(defn conform-multiple + [& spec-form-pair] + (if (s/valid? :chameleon.specs/spec-form-pair spec-form-pair) + (->> spec-form-pair + (partition 2) + (map (fn [[sp form]] + (when (s/invalid? (s/conform sp form)) + (s/explain-data sp form)))) + (remove nil?)) + (s/explain-data :chameleon.specs/spec-form-pair spec-form-pair))) + +(defn mdc-set! + "Sets the global MDC context for the current thread." + [m] + (doseq [[k v] m] (MDC/put k v))) + +(defn mdc-init! + "Sets the global MDC context as required by the EELF logging library" + [transaction-id service instance partner client-address] + (MdcContext/initialize transaction-id service instance partner client-address)) + +(defn mdc-clear! [] (MDC/clear)) + +(defmacro with-mdc + "Will set the global MDC context with the options (will convert from + keywords to PascalCase), execute the log, and clear the MDC + context." + [opts & body] + `(do + (mdc-set! (transform-keys cs/->PascalCaseString ~opts)) + ~@body + (mdc-clear))) + +(defn mdc-override + [m] + (->java (fn [j k v] (.addAttribute j k v)) (new MdcOverride) + (transform-keys cs/->PascalCaseString m))) + +(defn error-logger + [name] + (.getLogger (LoggerFactory/getInstance) name)) + +(defn audit-logger + [name] + (.getAuditLogger (LoggerFactory/getInstance) name)) + +(defn logger? + [logger] + (instance? AaiLoggerAdapter logger)) + +(defn info + [^AaiLoggerAdapter logger ^String enum msgs & {:keys [fields] :or {fields {}}}] + (let [confirmed-specs (conform-multiple :logging/valid-fields fields :logging/msgs msgs + :chameleon.specs/logger logger)] + (if (empty? confirmed-specs) + (.info logger (string->enum enum) (logfields fields) (into-array java.lang.String msgs)) + confirmed-specs))) + +(defn debug + [^AaiLoggerAdapter logger ^String enum msgs] + (.debug logger (string->enum enum) (into-array java.lang.String msgs))) + +(defn valid-logfields? + [m] + (->> (keys m) + (map cs/->SCREAMING_SNAKE_CASE_STRING) + set + (clojure.set/superset? (-> LOGLINE_DEFINED_FIELDS keys set)))) + +(def ^{:private true + :doc "Adding these fields from \"org.onap.aai.cl.api.LogLine\" + class. Right now there isn't a known way to use the ENUMs of an + abstract JAVA class if they are not STATIC. The field order is very + specific and it should be maintained for the common logging library + version \"1.2.2\". For a better understanding, please look at the + \"DefinedFields\" ENUMs in the org.onap.aai.cl.api.LogLine + class"} + + LOGLINE_DEFINED_FIELDS + (zipmap ["STATUS_CODE" "RESPONSE_CODE" "RESPONSE_DESCRIPTION" "INSTANCE_UUID" + "SEVERITY" "SERVER_IP" "CLIENT_IP" "CLASS_NAME" "PROCESS_KEY" + "TARGET_SVC_NAME" "TARGET_ENTITY" "ERROR_CODE" "ERROR_DESCRIPTION" + "CUSTOM_1" "CUSTOM_2""CUSTOM_3" "CUSTOM_4"] + (range))) + +(defn- string->enum + ([^String enum] (proxy [Enum LogMessageEnum] [enum (hash enum)])) + ([^String enum ordinal] (proxy [Enum LogMessageEnum] [enum ordinal]))) + +(defn- ->java + [f jc m] + (reduce (fn [a [k v]] (f a k v) a) + jc m)) + +(defn- logfields + "Generate a \"LogFields\" object with all the fields set." + [m] + (->> m + (transform-keys cs/->SCREAMING_SNAKE_CASE_STRING) + (transform-keys #(string->enum % (LOGLINE_DEFINED_FIELDS %))) + (->java (fn [j field v] (.setField j field v)) (new LogFields)))) diff --git a/src/chameleon/route.clj b/src/chameleon/route.clj index e8f3a3c..0918da4 100644 --- a/src/chameleon/route.clj +++ b/src/chameleon/route.clj @@ -1,12 +1,7 @@ (ns chameleon.route - (:require [org.httpkit.client :as kitclient])) - -(defn- interpret-response - "Print out the response from the Gallifrey server" - [key response] - (let [{:keys [status body]}@response] - (println "Response for request with key " key " resulted in status " status - " with body " body ))) + (:require [org.httpkit.client :as kitclient] + [chameleon.logging :as log] + [ring.util.http-status :as hs])) (defn query "Retrieve an entity referenced by id at the provided host. Optionally provide @@ -21,11 +16,10 @@ :keepalive 300 :timeout 20000})) -(defn assert-create +(defn assert-create! "Creates an entity in Gallifrey with an initial set of assertions coming from the provided payload" [host actor type key payload & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :put :query-params (into {"actor" actor "create" "true"} time-dimensions) :body payload @@ -33,11 +27,10 @@ :keepalive 300 :timeout 1000})) -(defn assert-update +(defn assert-update! "Update an entity in Gallifrey with a set of assertions coming from the provided payload" [host actor type key payload & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :put :query-params (into {"actor" actor "changes-only" "true"} time-dimensions) :body payload @@ -45,26 +38,27 @@ :keepalive 300 :timeout 1000})) -(defn assert-delete +(defn assert-delete! "Assert a deletion for an entity in Gallifrey based on the provided key." [host actor type key & [time-dimensions]] - (kitclient/request { - :url (str "https://" host "/" type "/" key) + (kitclient/request {:url (str "https://" host "/" type "/" key) :method :delete - :query-params (into {"actor" actor} time-dimensions) + :query-params (into {"actor" actor} time-dimensions) :insecure? true :keepalive 300 :timeout 1000})) -(defn assert-gallifrey [host actor type payload] +(defn assert-gallifrey! + [host actor type payload & [e-logger a-logger]] "Propagates an assertion to Gallifrey based off of an event payload coming in from the event service." (let [{:keys [meta body]} payload - {:keys [key operation time]} meta] - (println operation " " type " with key " key) - (interpret-response key (case operation - "CREATE" - (assert-create host actor type key body time) - "UPDATE" - (assert-update host actor type key body time) - "DELETE" - (assert-delete host actor type key time))))) + {:keys [key operation time]} meta + _ (log/info e-logger "GALLIFREY_ASSERTION" (mapv str [operation type key])) + g-assert (case operation + "CREATE" (assert-create! host actor type key body time) + "UPDATE" (assert-update! host actor type key body time) + "DELETE" (assert-delete! host actor type key time)) + {:keys [status body]} @g-assert] + (log/info e-logger "GALLIFREY_ASSERTED" [(str type) (str key)]) + (log/info a-logger "RESPONSE" (mapv str [operation key status body]) + :fields {:response-code status :response-description (hs/get-name status)}))) diff --git a/src/chameleon/specs.clj b/src/chameleon/specs.clj new file mode 100644 index 0000000..40d5768 --- /dev/null +++ b/src/chameleon/specs.clj @@ -0,0 +1,71 @@ +(ns chameleon.specs + (:require [clojure.spec.alpha :as s] + [chameleon.logging :as log] + [clojure.spec.gen.alpha :as gen] + [cheshire.core :as c] + [clojure.string :as str])) + +(s/def ::host string?) +(s/def ::provenance string?) +(s/def ::payload map?) +(s/def ::string (s/spec (s/and string? (complement str/blank?)) :gen gen/string-alphanumeric)) +(s/def ::type (s/spec ::string :gen #(gen/elements ["vserver" "pserver" "generic-vnf"]))) +(s/def ::id uuid?) +(s/def ::source (s/keys :req-un [::type ::id])) +(s/def ::target (s/keys :req-un [::type ::id])) +(s/def ::_id ::id) +(s/def ::spec-form-pair (s/* (s/cat :spec #(% (s/registry)) :form any?))) + +;; spike event +(s/def :spike/transaction-id uuid?) +(s/def :spike/schema-version (s/spec #(re-matches #"v\d+" %) :gen #(->> (s/int-in 7 15) + s/gen + (gen/fmap (partial str "v"))))) +(s/def :spike/key uuid?) +(s/def :spike/name ::string) +(s/def :spike/last-mod-source-of-truth ::string) +(s/def :spike/source-of-truth ::string) +(s/def :spike/vserver-selflink ::string) +(s/def :spike/is-closed-loop-disabled boolean?) +(s/def :spike/in-maint boolean?) +(s/def :spike/timestamp inst?) +(s/def :spike/operation (s/spec ::string :gen #(gen/elements ["UPDATE" "CREATE" "DELETE"]))) +(s/def :spike/properties (s/keys :req-un [:spike/name ::id :spike/last-mod-source-of-truth + :spike/source-of-truth :spike/vserver-selflink + :spike/is-closed-loop-disabled :spike/in-maint])) +(s/def :spike/vertex (s/keys :req-un [:spike/schema-version ::type :spike/key :spike/properties])) +(s/def :spike/event (s/keys :req-un [:spike/transaction-id :spike/vertex + :spike/operation :spike/timestamp])) +(s/def :spike/payload (s/spec string? :gen #(gen/fmap (partial c/generate-string) + (s/gen :spike/event)))) +(s/def :gallifrey/k-end-actor ::string) + +;; gallifrey response +(s/def :gallifrey/k-start-actor ::string) +(s/def :gallifrey/k-end inst?) +(s/def :gallifrey/k-start inst?) +(s/def :gallifrey/history (s/keys :req-un [:gallifrey/k-end-actor :gallifrey/k-end + :gallifrey/k-start-actor :gallifrey/k-start])) +(s/def :relationship/_meta (s/map-of ::string :gallifrey/history)) +(s/def :relationship/_type (s/spec string? :gen #(gen/return "relationship"))) +(s/def :gallifrey/_type (s/spec ::string :gen #(gen/return "entity"))) +(s/def :gallifrey/properties (s/keys :req-un [:gallifrey/_type ::type])) +(s/def :relationship/type (s/spec string? :gen #(->> (gen/string-alphanumeric) + (gen/such-that (complement str/blank?)) + (gen/fmap (partial str "tosca.relationship."))))) +(s/def :relationship/properties (s/keys :req-un [:relationship/_type :relationship/type])) +(s/def ::relationship (s/keys :req-un [:relationship/properties ::source + ::target :relationship/_meta ::_id])) +(s/def ::relationships (s/coll-of ::relationship :gen-max 8)) +(s/def :gallifrey/payload (s/spec map? + :gen #(->> [::_id :gallifrey/properties + :gallifrey/properties ::relationships] + (s/keys :req-un) + s/gen + (gen/fmap (partial clojure.walk/stringify-keys))))) + +;; Logger specs +(s/def ::logger (s/spec log/logger? :gen #(gen/return (log/error-logger "chameleon.specs")))) +(s/def ::loggers (s/cat :e :chameleon.specs/logger :a :chameleon.specs/logger)) +(s/def :logging/msgs (s/* ::string)) +(s/def :logging/valid-fields log/valid-logfields?) diff --git a/test/chameleon/testing.clj b/test/chameleon/testing.clj new file mode 100644 index 0000000..13acb1a --- /dev/null +++ b/test/chameleon/testing.clj @@ -0,0 +1,42 @@ +(ns chameleon.testing + (:require [clojure.test :as t] + [clojure.spec.alpha :as s] + [clojure.spec.gen.alpha :as gen] + [clojure.spec.test.alpha :as st] + [chameleon.logging :as log] + [chameleon.specs :as cs] + [chameleon.route :as cr] + [chameleon.aai-processor :as cai])) + +(s/fdef chameleon.route/assert-gallifrey! + :args (s/cat :host :chameleon.specs/host + :provenance :chameleon.specs/provenance + :type :chameleon.specs/type + :payload :chameleon.specs/payload + :loggers :chameleon.specs/loggers) + :ret nil?) + +(s/fdef chameleon.aai-processor/from-spike + :args (s/cat :gallifrey-host :chameleon.specs/host + :payload :spike/payload + :loggers :chameleon.specs/loggers) + :ret nil?) + +(s/fdef chameleon.aai-processor/from-gallifrey + :args (s/cat :body :gallifrey/payload) + :ret map?) + +(s/fdef chameleon.aai-processor/gen-trim-relationship + :args (s/cat :relationship :chameleon.specs/relationship) + :ret map?) + +(st/instrument 'chameleon.route/assert-gallifrey! {:stub '(chameleon.route/assert-gallifrey!)}) + +;; Testing instrumentation +(chameleon.route/assert-gallifrey! "host" "aai" "type" {} (log/error-logger "chameleon.testing") (log/audit-logger "chameleon.testing")) + +(->> '(chameleon.aai-processor/from-spike + chameleon.aai-processor/from-gallifrey + chameleon.aai-processor/gen-trim-relationship) + st/check + st/summarize-results) |