diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | .gitreview | 4 | ||||
| -rw-r--r-- | ChangeLog.md | 13 | ||||
| -rw-r--r-- | Dockerfile | 10 | ||||
| -rw-r--r-- | LICENSE.txt | 32 | ||||
| -rw-r--r-- | README.md | 45 | ||||
| -rw-r--r-- | fixtures/4_insert.json | 59 | ||||
| -rw-r--r-- | fixtures/5_delete.json | 32 | ||||
| -rw-r--r-- | fixtures/location-sample.json | 1 | ||||
| -rw-r--r-- | pom.xml | 186 | ||||
| -rw-r--r-- | project.clj | 38 | ||||
| -rw-r--r-- | src/sch/asdc_client.clj | 74 | ||||
| -rw-r--r-- | src/sch/core.clj | 201 | ||||
| -rw-r--r-- | src/sch/handle.clj | 170 | ||||
| -rw-r--r-- | src/sch/inventory_client.clj | 82 | ||||
| -rw-r--r-- | src/sch/parse.clj | 133 | ||||
| -rw-r--r-- | src/sch/util.clj | 48 | ||||
| -rw-r--r-- | test/sch/handle_test.clj | 38 | ||||
| -rw-r--r-- | test/sch/parse_test.clj | 63 |
19 files changed, 1230 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..fb3d264 --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.onap.org +port=29418 +project=dcaegen2/platform/servicechange-handler diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 0000000..7ed2195 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,13 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [1.1.0] + +This body of work is aimed at getting SCH to be run on the new DCAE platform. + +* Add the ability to remotely fetch configuration in json form +* Add Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1245a2d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM maven:3-jdk-8 + +WORKDIR /opt/sch +ADD . /opt/sch +RUN mvn clean package + +# TODO: This is bogus. This is simply to be used for Registrator registration. +EXPOSE 65000 + +CMD ["java", "-jar", "/opt/sch/target/dcae-service-change-handler.jar", "prod", "http://consul:8500/v1/kv/service-change-handler?raw=true"] diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..cb8008a --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,32 @@ +============LICENSE_START======================================================= +org.onap.dcae +================================================================================ +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +================================================================================ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +============LICENSE_END========================================================= + +ECOMP is a trademark and service mark of AT&T Intellectual Property. + + +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +=================================================================== +Licensed under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +you may not use this documentation except in compliance with the License. +You may obtain a copy of the License at + https://creativecommons.org/licenses/by/4.0/ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c793d3d --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# DCAE service change handler + +Application that is responsible for polling for ASDC distribution notification events and handling those events. Handling means: + +* Parsing the event for DCAE artifacts +* Identifying whether its complementary DCAE service type resource in DCAE inventory has changed +* Taking action + - Insert a new DCAE service type + - Update an exisintg DCAE service type + - Deactivate an existing DCAE service type +* Send appropriate acknowledgements back + +## Dependencies + +Uses the SDC distribution client to interface with the SDC API. + +## To run + +Two modes of operation: development and production. + +### Development + +The application in development mode does not actually pull from ASDC but rather takes in a file that contains a single ASDC notfication event as a third argument and processes it. + +Usage of development mode: + +``` +java -jar dcae-service-change-handler-0.1.0.jar dev <config file path> <event file path> +``` + +### Production + +The application in production mode continuously pulls events from ASDC and processes them. + +Usage of production mode when config is a file on the filesystem: + +``` +java -jar dcae-service-change-handler-0.1.0.jar prod <config file path> +``` + +Usage of production mode when config is remote stored in Consul: + +``` +java -jar dcae-service-change-handler-0.1.0.jar prod http://consul:8500/v1/kv/service-change-handler?raw=true +``` diff --git a/fixtures/4_insert.json b/fixtures/4_insert.json new file mode 100644 index 0000000..88918e4 --- /dev/null +++ b/fixtures/4_insert.json @@ -0,0 +1,59 @@ +{ + "uuid": "3ff44fa9-9372-45dc-a432-46f7ff8c5103", + "invariantUUID": "9eaf59ee-2fe0-48a9-8d20-6f9b09ba807b", + "name": "DcaeTestService", + "version": "2.0", + "toscaModelURL": "https://mtanjv9sdcb21.cip.com:8443/sdc/v1/catalog/services/3ff44fa9-9372-45dc-a432-46f7ff8c5103/toscaModel", + "category": "Mobility", + "lifecycleState": "CERTIFIED", + "distributionStatus": "DISTRIBUTED", + "lastUpdaterFullName": "MICHAEL SHITRIT", + "resources": [ + { + "resourceInstanceName": "DcaeTestVF 2", + "resourceName": "DcaeTestVF", + "resourceInvariantUUID": "3d5927fc-a28e-41e9-9e79-57289aa7f754", + "resourceVersion": "2.0", + "resoucreType": "VF", + "resourceUUID": "d8d91013-5bc5-4600-9079-fae3cefd4ca7", + "artifacts": [ + { + "artifactName": "dcaetestvf2_modules.json", + "artifactType": "VF_MODULES_METADATA", + "artifactURL": "/sdc/v1/catalog/services/3ff44fa9-9372-45dc-a432-46f7ff8c5103/resourceInstances/dcaetestvf2/artifacts/a5998f90-7335-4b30-a988-dbfd39147a9c", + "artifactDescription": "Auto-generated VF Modules information artifact", + "artifactChecksum": "ZDc1MTcxMzk4ODk4N2U5MzMxOTgwMzYzZTI0MTg5Y2U=", + "artifactUUID": "a5998f90-7335-4b30-a988-dbfd39147a9c", + "artifactVersion": "1" + }, + { + "artifactName": "sample-blueprint", + "artifactType": "DCAE_INVENTORY_BLUEPRINT", + "artifactURL": "/sdc/v1/catalog/services/3ff44fa9-9372-45dc-a432-46f7ff8c5103/resourceInstances/dcaetestvf2/artifacts/6737fcea-b0ac-4e2f-b7d4-5d224961510e", + "artifactDescription": "sdfsd", + "artifactChecksum": "ZjhmYjVjZDI4MWM5MzA4OTA0MWFjNTYwOWU3NjhhYjY=", + "artifactUUID": "6737fcea-b0ac-4e2f-b7d4-5d224961510e", + "artifactVersion": "2" + }, + { + "artifactName": "fake-out", + "artifactType": "DCAE_DOC", + "artifactURL": "/sdc/v1/catalog/services/3ff44fa9-9372-45dc-a432-46f7ff8c5103/resourceInstances/dcaetestvf2/artifacts/6737fcea-b0ac-4e2f-b7d4-5d224961510e", + "artifactDescription": "sdfsd", + "artifactChecksum": "ZjhmYjVjZDI4MWM5MzA4OTA0MWFjNTYwOWU3NjhhYjY=", + "artifactUUID": "6737fcea-b0ac-4e2f-b7d4-5d224961510f", + "artifactVersion": "2" + }, + { + "artifactName": "sample-blueprint-location", + "artifactType": "DCAE_INVENTORY_JSON", + "artifactURL": "fixtures/location-sample.json", + "artifactDescription": "sdfsd", + "artifactChecksum": "ZjhmYjVjZDI4MWM5MzA4OTA0MWFjNTYwOWU3NjhhYjY=", + "artifactUUID": "6737fcea-b0ac-4e2f-b7d4-5d224961510e", + "artifactVersion": "2" + } + ] + } + ] +} diff --git a/fixtures/5_delete.json b/fixtures/5_delete.json new file mode 100644 index 0000000..4eeeb44 --- /dev/null +++ b/fixtures/5_delete.json @@ -0,0 +1,32 @@ +{ + "uuid": "3ff44fa9-9372-45dc-a432-46f7ff8c5103", + "invariantUUID": "9eaf59ee-2fe0-48a9-8d20-6f9b09ba807b", + "name": "DcaeTestService", + "version": "2.0", + "toscaModelURL": "https://mtanjv9sdcb21.cip.com:8443/sdc/v1/catalog/services/3ff44fa9-9372-45dc-a432-46f7ff8c5103/toscaModel", + "category": "Mobility", + "lifecycleState": "CERTIFIED", + "distributionStatus": "DISTRIBUTED", + "lastUpdaterFullName": "MICHAEL SHITRIT", + "resources": [ + { + "resourceInstanceName": "DcaeTestVF 2", + "resourceName": "DcaeTestVF", + "resourceInvariantUUID": "3d5927fc-a28e-41e9-9e79-57289aa7f754", + "resourceVersion": "2.0", + "resoucreType": "VF", + "resourceUUID": "d8d91013-5bc5-4600-9079-fae3cefd4ca7", + "artifacts": [ + { + "artifactName": "dcaetestvf2_modules.json", + "artifactType": "VF_MODULES_METADATA", + "artifactURL": "/sdc/v1/catalog/services/3ff44fa9-9372-45dc-a432-46f7ff8c5103/resourceInstances/dcaetestvf2/artifacts/a5998f90-7335-4b30-a988-dbfd39147a9c", + "artifactDescription": "Auto-generated VF Modules information artifact", + "artifactChecksum": "ZDc1MTcxMzk4ODk4N2U5MzMxOTgwMzYzZTI0MTg5Y2U=", + "artifactUUID": "a5998f90-7335-4b30-a988-dbfd39147a9c", + "artifactVersion": "1" + } + ] + } + ] +} diff --git a/fixtures/location-sample.json b/fixtures/location-sample.json new file mode 100644 index 0000000..cac67bb --- /dev/null +++ b/fixtures/location-sample.json @@ -0,0 +1 @@ +{"artifactName" : "sample-blueprint", "locations" : ["CLLI1", "CLL2"]} @@ -0,0 +1,186 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.onap.dcae</groupId> + <artifactId>dcae-service-change-handler</artifactId> + <version>1.1.0</version> + <!-- Not sure why clojure-maven-plugin says packaging should be "clojure" --> + <packaging>jar</packaging> + + <!-- This maybe an issue --> + <repositories> + <repository> + <id>clojars.org</id> + <url>http://clojars.org/repo</url> + </repository> + <!-- TODO: Fill in the onap maven repository info --> + </repositories> + + <dependencies> + <dependency> + <groupId>org.clojure</groupId> + <artifactId>clojure</artifactId> + <version>1.8.0</version> + </dependency> + <dependency> + <groupId>cheshire</groupId> + <artifactId>cheshire</artifactId> + <version>5.6.3</version> + </dependency> + <dependency> + <groupId>org.openecomp.sdc</groupId> + <artifactId>sdc-distribution-client</artifactId> + <version>1.1.4</version> + </dependency> + <dependency> + <groupId>com.taoensso</groupId> + <artifactId>timbre</artifactId> + <version>4.7.4</version> + </dependency> + <!-- Does magic so traditional Java loggers show up --> + <dependency> + <groupId>com.fzakaria</groupId> + <artifactId>slf4j-timbre</artifactId> + <version>0.3.2</version> + </dependency> + <dependency> + <groupId>clj-http</groupId> + <artifactId>clj-http</artifactId> + <version>3.3.0</version> + </dependency> + <dependency> + <groupId>org.bovinegenius</groupId> + <artifactId>exploding-fish</artifactId> + <version>0.3.4</version> + </dependency> + <dependency> + <groupId>clj-yaml</groupId> + <artifactId>clj-yaml</artifactId> + <version>0.4.0</version> + </dependency> + </dependencies> + + <build> + <plugins> + <!-- This seems to be the only place to specify the application entrypoint --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.0.2</version> + <configuration> + <archive> + <manifest> + <addClasspath>true</addClasspath> + <mainClass>sch.core</mainClass> + <classpathPrefix>dependency</classpathPrefix> + </manifest> + </archive> + <finalName>${project.artifactId}</finalName> + </configuration> + </plugin> + <!-- Used to copy in the dependencies when packaging JAR --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.10</version> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + </execution> + </executions> + </plugin> + <!-- Explicitly use a certain version of Java. May not be needed --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.5</version> + <configuration> + <source>1.8</source> + <target>1.8</target> + </configuration> + </plugin> + <plugin> + <groupId>com.theoryinpractise</groupId> + <artifactId>clojure-maven-plugin</artifactId> + <version>1.8.1</version> + <extensions>true</extensions> + <configuration> + <sourceDirectories> + <sourceDirectory>src</sourceDirectory> + </sourceDirectories> + <mainClass>sch.core</mainClass> + </configuration> + <executions> + <execution> + <id>compile</id> + <phase>compile</phase> + <goals> + <goal>compile</goal> + </goals> + </execution> + </executions> + </plugin> + + <!-- Package an Uber jar --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.4.3</version> + <executions> + <!-- Run shade goal on package phase --> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <transformers> + <!-- NOTE: Need the following transformer else gets "Could not resolve type id 'https' into a subtype" error + Solution found from here: + http://stackoverflow.com/questions/27543060/why-does-dropwizard-configuration-is-not-working + Some more context here: + https://github.com/dropwizard/dropwizard/issues/455 --> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> + <!-- add Main-Class to manifest file --> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>sch.core</mainClass> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.spotify</groupId> + <artifactId>docker-maven-plugin</artifactId> + <version>0.4.13</version> + <configuration> + <maintainer>Michael Hwang</maintainer> + <imageName>dcae-service-change-handler</imageName> + <imageTags> + <imageTag>${project.version}</imageTag> + </imageTags> + <baseImage>java:8-jre</baseImage> + <entryPoint>["java", "-jar", "/opt/${project.build.finalName}.jar", "prod", "/opt/config.yml"]</entryPoint> + <!-- copy the service's jar file from target into the root directory of the image --> + <resources> + <resource> + <targetPath>/opt</targetPath> + <directory>${project.build.directory}</directory> + <include>${project.build.finalName}.jar</include> + </resource> + </resources> + </configuration> + </plugin> + + </plugins> + </build> +</project> diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..3ccbe64 --- /dev/null +++ b/project.clj @@ -0,0 +1,38 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +; Using lein for REPL and testing because couldn't get Maven clojure plugin to work +; for these functional areas. + +(defproject service-change-handler "0.1.0" + :description "Service change handler" + :dependencies [[org.clojure/clojure "1.8.0"] + [cheshire/cheshire "5.6.3"] + [com.taoensso/timbre "4.7.4"] + [com.fzakaria/slf4j-timbre "0.3.2"] + [clj-http/clj-http "3.3.0"] + [org.bovinegenius/exploding-fish "0.3.4"] + [clj-yaml/clj-yaml "0.4.0"] + [org.openecomp.sdc/sdc-distribution-client "1.1.4"]] + + ; TODO: Fill in the onap maven repository info + :repositories [] + + ) diff --git a/src/sch/asdc_client.clj b/src/sch/asdc_client.clj new file mode 100644 index 0000000..702c571 --- /dev/null +++ b/src/sch/asdc_client.clj @@ -0,0 +1,74 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.asdc-client + (:require [clj-http.client :as client] + [taoensso.timbre :as timbre :refer [error]] + [cheshire.core :refer [parse-string]] + [org.bovinegenius.exploding-fish :refer [uri param]]) + (:gen-class)) + + +(defn create-asdc-conn + + ([asdc-uri user password consumer-id] + [(uri asdc-uri) user password consumer-id]) + + ([config] + (let [config-asdc (:asdcDistributionClient config)] + (create-asdc-conn (:asdcUri config-asdc) (:user config-asdc) + (:password config-asdc) (:consumerId config-asdc)))) + ) + + +(defn get-consumer-id + [asdc-conn] + (get asdc-conn 3)) + +(defn construct-service-path + [service-uuid] + (str "/asdc/v1/catalog/services/" service-uuid "/metadata")) + + +(defn get-artifact! + [connection artifact-path] + (let [[asdc-uri user password instance-id] connection + target-uri (assoc asdc-uri :path artifact-path) + resp (client/get (str target-uri) { :basic-auth [user password] + :headers { "X-ECOMP-InstanceID" instance-id } })] + (if (= (:status resp) 200) + ; Response media type is application/octet-stream + ; TODO: Use X-ECOMP-RequestID? + (:body resp) + (error (str "GET asdc artifact failed: " (:status resp) ", " (:body resp)))) + )) + +(defn get-service-metadata! + [connection service-uuid] + (let [[asdc-uri user password instance-id] connection + target-uri (assoc asdc-uri :path (construct-service-path service-uuid)) + resp (client/get (str target-uri) { :basic-auth [user password] + :headers { "X-ECOMP-InstanceID" instance-id } })] + (if (= (:status resp) 200) + ; Response media type is application/octet-stream + ; TODO: Use X-ECOMP-RequestID? + (parse-string (:body resp) true) + (error (str "GET asdc service metadata failed: " (:status resp) ", " (:body resp)))) + )) diff --git a/src/sch/core.clj b/src/sch/core.clj new file mode 100644 index 0000000..8684576 --- /dev/null +++ b/src/sch/core.clj @@ -0,0 +1,201 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.core + (:require [clojure.java.io :refer :all] + [cheshire.core :refer [parse-stream parse-string]] + [taoensso.timbre :as timbre :refer [info error]] + [taoensso.timbre.appenders.3rd-party.rolling :refer [rolling-appender]] + [sch.handle :refer [handle-change-event! download-artifacts! deploy-artifacts! + deployed-ok deployed-error deployed-already]] + [sch.asdc-client :refer [get-service-metadata! create-asdc-conn get-consumer-id]] + [sch.inventory-client :refer [create-inventory-conn]] + [sch.parse :refer [get-dcae-artifact-types pick-out-artifact]] + [sch.util :refer [read-config]] + ) + (:import (org.openecomp.sdc.impl DistributionClientFactory) + (org.openecomp.sdc.api.consumer IConfiguration INotificationCallback + IDistributionStatusMessage) + (org.openecomp.sdc.utils DistributionActionResultEnum DistributionStatusEnum) + (com.google.gson Gson) + ) + (:gen-class)) + + +(defn process-event-from-local-file! + [event-file-path] + (with-open [rdr (reader event-file-path)] + (parse-stream rdr true)) + ) + +; Distribution client code +; TODO: Should be moved to separate namespace + +(defn send-distribution-status! + "Convenience function used to send distribution status messages" + [dist-client distribution-id consumer-id artifact status] + (let [dist-message (proxy [IDistributionStatusMessage] [] + (getDistributionID [] distribution-id) + (getConsumerID [] consumer-id) + (getTimestamp [] + (. java.lang.System currentTimeMillis)) + (getArtifactURL [] (:artifactURL artifact)) + (getStatus [] status)) + resp (.sendDeploymentStatus dist-client dist-message) + ] + (if (not= (.getDistributionActionResult resp) (. DistributionActionResultEnum SUCCESS)) + (error (str "Problem sending status: " (:artifactName artifact) ", " + (str (.getDistributionMessageResult resp))))) + )) + +(defn deploy-artifacts-ex! + "Enhanced deploy artifacts function + + After calling deploy-artifacts!, this method takes the results and sends out + appropriate distribution status messages per artifact processed" + [inventory-uri service-metadata requests send-dist-status] + (let [[to-post posted to-delete deleted] (deploy-artifacts! inventory-uri service-metadata + requests) + pick-out-artifact (partial pick-out-artifact service-metadata)] + + (dorun (map #(send-dist-status (pick-out-artifact %) + (. DistributionStatusEnum DEPLOY_OK)) + (deployed-ok to-post posted))) + (dorun (map #(send-dist-status (pick-out-artifact %) + (. DistributionStatusEnum DEPLOY_ERROR)) + (deployed-error to-post posted))) + (dorun (map #(send-dist-status (pick-out-artifact %) + (. DistributionStatusEnum ALREADY_DEPLOYED)) + (deployed-already requests to-post))) + ; REVIEW: How about the deleted service types? + )) + + +(defn create-distribution-client-config + [config] + (let [config-asdc (:asdcDistributionClient config)] + (proxy [IConfiguration] [] + (getAsdcAddress [] (:asdcAddress config-asdc)) + (getUser [] (:user config-asdc)) + (getPassword [] (:password config-asdc)) + (getPollingInterval [] (:pollingInterval config-asdc)) + (getPollingTimeout [] (:pollingTimeout config-asdc)) + ; Note: The following didn't work + ; (. Arrays asList (. ArtifactTypeEnum values)) + ; Also, cannot just use a narrow list of artifact types in order + ; to handle the deletion scenario. + (getRelevantArtifactTypes [] (java.util.ArrayList. + (get-dcae-artifact-types))) + (getConsumerGroup [] (:consumerGroup config-asdc)) + (getConsumerID [] (:consumerId config-asdc)) + (getEnvironmentName [] (:environmentName config-asdc)) + (getKeyStorePath [] (:keyStorePath config-asdc)) + (getKeyStorePassword [] (:keyStorePassword config-asdc)) + (activateServerTLSAuth [] (:activateServerTLSAuth config-asdc)) + (isFilterInEmptyResources [] (:isFilterInEmptyResources config-asdc)) + ))) + +(defn run-distribution-client! + "Entry point to the core production functionality + + Uses the asdc distribution client and to poll for notification events and makes calls + to handle those events" + [dist-client-config inventory-uri asdc-conn] + (let [dist-client (. DistributionClientFactory createDistributionClient) + dist-client-callback (proxy [INotificationCallback] [] + (activateCallback [data] + "Callback executed upon notification events + + The input parameter is of type `NotificationDataImpl` which fails + to translate via the clojure `bean` call because class is not + public. So mirroring what's done in the distribution client - + use Gson. + + Discovered that the notification event and the service metadata + data models are different. Use service metadata because its + richer." + (let [change-event (parse-string (.toJson (Gson.) data) true) + service-id (:serviceUUID change-event) + distribution-id (:distributionID change-event) + service-metadata (get-service-metadata! asdc-conn + service-id) + send-dist-status (partial send-distribution-status! + dist-client distribution-id + (get-consumer-id asdc-conn)) + ] + + (info (str "Handle change event: " (:serviceName change-event) + ", " distribution-id)) + + (let [requests (download-artifacts! inventory-uri asdc-conn + service-metadata) + artifacts (map #(pick-out-artifact service-metadata %) + requests) + ] + + (dorun (map #(send-dist-status + % (. DistributionStatusEnum DOWNLOAD_OK)) + artifacts)) + (deploy-artifacts-ex! inventory-uri service-metadata + requests send-dist-status) + ) + + ))) + ] + (let [dist-client-init-result (.init dist-client dist-client-config dist-client-callback)] + (if (= (.getDistributionActionResult dist-client-init-result) + (. DistributionActionResultEnum SUCCESS)) + (.start dist-client) + (error dist-client-init-result)) + ))) + +(defn- setup-logging-rolling + "Setup logging with the rolling appender" + [{ {:keys [currentLogFilename rotationFrequency]} :logging }] + (let [rolling-params (when currentLogFilename { :path currentLogFilename }) + rolling-params (when rotationFrequency + (assoc rolling-params :pattern (keyword rotationFrequency)))] + (timbre/merge-config! { :level :debug :appenders { :rolling (rolling-appender rolling-params) } }) + (info "Setup logging: Rolling appender" (if rolling-params rolling-params "DEFAULT")) + )) + + +(defn -main [& args] + (let [[mode config-path event-path] args + config (read-config config-path) + inventory-uri (create-inventory-conn config) + asdc-conn (create-asdc-conn config) + ] + + (setup-logging-rolling config) + + (if (= "DEV" (clojure.string/upper-case mode)) + (do + (info "Development mode") + (handle-change-event! inventory-uri asdc-conn + (process-event-from-local-file! event-path)) + ) + (run-distribution-client! + (create-distribution-client-config config) + inventory-uri asdc-conn) + ) + + (info "Done")) + ) diff --git a/src/sch/handle.clj b/src/sch/handle.clj new file mode 100644 index 0000000..13201c4 --- /dev/null +++ b/src/sch/handle.clj @@ -0,0 +1,170 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.handle + (:require [clojure.java.io :refer :all] + [taoensso.timbre :as timbre :refer [info error]] + [sch.inventory-client :refer [get-service-types! post-service-type! + delete-service-type!]] + [sch.asdc-client :refer [get-artifact!]] + [sch.parse :refer [generate-dcae-service-type-requests + get-service-locations]] + ) + (:import (org.openecomp.sdc.utils DistributionStatusEnum)) + (:gen-class)) + + +; Handle the ASDC distribtuion notification of change events and take action + +(defn should-update? + [service-type-request service-type-prev] + (if (< (:typeVersion service-type-prev) (:typeVersion service-type-request)) true false)) + +(defn should-insert? + [service-type-request service-type-prev] + (if (empty? service-type-prev) true false)) + +(defn- find-service-types-to-post! + [inventory-uri service-type-requests] + (letfn [(post? [service-type-request] + (let [type-name (:typeName service-type-request) + asdc-service-id (:asdcServiceId service-type-request) + asdc-resource-id (:asdcResourceId service-type-request) + query-result (get-service-types! inventory-uri + ["typeName" type-name + "asdcServiceId" asdc-service-id + "asdcResourceId" asdc-resource-id + "onlyActive" false]) + service-type-prev (first query-result)] + + (cond + ; Unexpected error from the GET call to inventory + (nil? query-result) (error "Unexpected error querying inventory") + + ; Insert and update actions are the same + (or (should-insert? service-type-request service-type-prev) + (should-update? service-type-request service-type-prev)) true + + :else (info "Insert/update not needed: " (:typeName service-type-request)) + )))] + + (filter #(post? %1) service-type-requests))) + +(defn- post-service-types! + [inventory-uri service-type-requests] + (letfn [(post [service-type-request] + (let [service-type (post-service-type! inventory-uri service-type-request) + type-id (:typeId service-type)] + (if service-type + (do + (info (str "Inserted/updated new dcae service type: " type-id)) + service-type) + (error (str "Error inserting/updated new dcae service type: " type-id)))))] + (remove nil? (map post service-type-requests)) + )) + + +(defn- find-service-types-to-delete! + [inventory-uri service-id service-type-requests] + (let [query-result (get-service-types! inventory-uri ["asdcServiceId" service-id])] + + (letfn [(gone? [resource-id type-name] + (nil? (some #(and (= resource-id (:asdcResourceId %1)) + (= type-name (:typeName %1))) + service-type-requests)))] + + (filter #(gone? (:asdcResourceId %1) (:typeName %1)) query-result) + ))) + +(defn- delete-service-types! + [inventory-uri service-type-requests] + (letfn [(delete [service-type-request] + (let [type-id (delete-service-type! inventory-uri (:typeId service-type-request))] + (if type-id + (do + (info (str "Deleted dcae service type: " type-id)) + type-id) + (error (str "Error deleting dcae service type: " type-id)))))] + (remove nil? (map delete service-type-requests)) + )) + + +(defn download-artifacts! + "Generates dcae service type requests from the service metadata" + [inventory-uri asdc-conn service-metadata] + (let [get-artifact-func (partial get-artifact! asdc-conn) + get-locations-func (partial get-service-locations get-artifact-func) + requests (generate-dcae-service-type-requests get-artifact-func + get-locations-func + service-metadata)] + (info (str "Done downloading artifacts: " (count requests))) + requests + )) + +(defn deploy-artifacts! + "Takes action on dcae service types produced from the service metadata" + [inventory-uri service-metadata requests] + (let [service-id (:invariantUUID service-metadata) + + to-post (find-service-types-to-post! inventory-uri requests) + to-delete (find-service-types-to-delete! inventory-uri service-id requests) + + posted (if (not-empty to-post) (post-service-types! inventory-uri to-post)) + deleted (if (not-empty to-delete) (delete-service-types! inventory-uri to-delete)) + + stats (zipmap [ :num-requests :num-to-post :num-posted :num-to-delete :num-deleted] + (map count [requests to-post posted to-delete deleted])) + ] + + (info (str "Done deploying artifacts: " stats)) + [to-post posted to-delete deleted] + )) + + +(defn handle-change-event! + "Convenience method that calls download-artifacts then deploy-artifacts + + This function takes a service metadata to: + + 1. Generate dcae service type requests + 2. Posts dcae service type requests that are *new* or *updated* + 3. Deletes dcae service types that are no longer part of a (service, resource)" + [inventory-uri asdc-conn service-metadata] + (let [requests (download-artifacts! inventory-uri asdc-conn service-metadata)] + (deploy-artifacts! inventory-uri service-metadata requests))) + +; Classify the outputs from the deploy and download calls + +(defn- filtering-lists + [filter-func requests results] + (letfn [(success? [request] + (true? (some #(and (= (:asdcResourceId request) (:asdcResourceId %)) + (= (:typeName request) (:typeName %))) + results)))] + (filter-func success? requests))) + +; attempted-requests service-types +(def deployed-ok (partial filtering-lists filter)) + +; attempted-requests service-types +(def deployed-error (partial filtering-lists remove)) + +; requests attempted-requests +(def deployed-already (partial filtering-lists remove)) diff --git a/src/sch/inventory_client.clj b/src/sch/inventory_client.clj new file mode 100644 index 0000000..a38b693 --- /dev/null +++ b/src/sch/inventory_client.clj @@ -0,0 +1,82 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.inventory-client + (:require [clj-http.client :as client] + [taoensso.timbre :as timbre :refer [error]] + [cheshire.core :refer [parse-string]] + [org.bovinegenius.exploding-fish :refer [uri param]]) + (:gen-class)) + + +(defn create-inventory-conn + [config] + (uri (get-in config [:dcaeInventoryClient :uri]))) + +; HTTP client to DCAE inventory + +(defn- append-params + "Appends arbitrary list of parameter pairs to a URI + + `params` must be in the form [*field name* *value* ...] + + Returns the updated URI" + [uri params] + (let [[field value & more-params] params + uri-updated (param uri field value)] + (if more-params + (append-params uri-updated more-params) + uri-updated))) + + +(defn get-service-types! + "GET DCAE service types from inventory + + TODO: Now its generic, how to put checks?" + [inventory-uri query-params] + (let [path "/dcae-service-types" + inventory-uri (append-params (assoc inventory-uri :path path) + query-params) + resp (client/get (str inventory-uri) { :content-type :json })] + (if (= (:status resp) 200) + (:items (parse-string (:body resp) true)) + (error (str "GET dcae-service-types failed: " (:status resp) ", " (:body resp))) + ))) + + +(defn post-service-type! + [inventory-uri dcae-service-type] + (let [resp (client/post (str (assoc inventory-uri :path "/dcae-service-types")) + { :content-type :json + :form-params dcae-service-type })] + (if (= (:status resp) 200) + (parse-string (:body resp) true) + (error (str "POST dcae-service-types failed: " (:status resp) ", " (:body resp))) + ))) + + +(defn delete-service-type! + [inventory-uri service-type-id] + (let [path (str "/dcae-service-types/" service-type-id) + resp (client/delete (str (assoc inventory-uri :path path)))] + (if (or (= (:status resp) 200) (= (:status resp) 410)) + service-type-id + (error (str "DELETE dcae-service-types failed: " (:status resp) ", " (:body resp))) + ))) diff --git a/src/sch/parse.clj b/src/sch/parse.clj new file mode 100644 index 0000000..7ce48e6 --- /dev/null +++ b/src/sch/parse.clj @@ -0,0 +1,133 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.parse + (:require [clojure.java.io :refer :all] + [taoensso.timbre :as timbre :refer [info error]] + [sch.asdc-client :refer [construct-service-path]] + [cheshire.core :refer [parse-string]] + ) + (:import (org.openecomp.sdc.utils ArtifactTypeEnum)) + (:gen-class)) + + +; Abstraction to parse the ASDC distribution notification of change events and +; transforms them into DCAE service type requests + +(defn get-dcae-artifact-types + "Returns lazy-seq of string representations of ArtifactTypeEnums that DCAE-related" + [] + (letfn [(dcae-artifact-type? [artifact-type] + (boolean (re-find #"DCAE_" artifact-type)))] + (filter dcae-artifact-type? + (map #(.name %) (seq (. ArtifactTypeEnum values)))) + )) + +(defn dcae-artifact? + "Checks to see if the artifact is a DCAE artifact" + [artifact] + (let [supported-dcae-artifact-types (get-dcae-artifact-types)] + (true? (some #(= (:artifactType artifact) %) supported-dcae-artifact-types)) + )) + + +(defn dcae-artifact-inventory-blueprint? + "Check to see if the artifact is an inventory blueprint" + [artifact] + (= (:artifactType artifact) "DCAE_INVENTORY_BLUEPRINT")) + + +(defn get-service-locations + "Gets service locations for a given blueprint + + The service location information is attached as a separate artifact. This function + is responsible for finding the matching locations JSON artifact that is of the form: + + { \"artifactName\": <artifact name of the blueprint artifact>, + \"locations\": <list of location strings> }" + [get-artifact-func resource-metadata artifact-name] + (let [target-artifacts (filter #(= (:artifactType %) "DCAE_INVENTORY_JSON") + (:artifacts resource-metadata)) + inventory-jsons (map #(parse-string (get-artifact-func (:artifactURL %)) true) + target-artifacts) + location-jsons (filter #(and + (= (:artifactName %) artifact-name) + (contains? % :locations)) inventory-jsons)] + (flatten (map :locations location-jsons)) + )) + +(defn generate-dcae-service-type-requests + "Generates DCAE service type requests from ASDC change event + + The ASDC change event is a nested structure. The single arity of this method + handles at the service level of the event. The two arity of this method handles + at the resource level of the event. + + `get-blueprint-func` is function that takes the `artifactURL` and retrieves + the DCAE blueprint artifact. + + Returns a list of DCAE service type requests" + ([get-blueprint-func get-locations-func service-change-event] + (let [; TODO: Where do I get this from? + service-location nil + service-id (:invariantUUID service-change-event) + service-part { :asdcServiceId service-id + :asdcServiceURL (construct-service-path service-id) + :owner (:lastUpdaterFullName service-change-event) }] + + ; Given the resource part, create dcae service type requests + (letfn [(generate-for-resource + [resource-change-event] + (let [dcae-artifacts (filter dcae-artifact-inventory-blueprint? + (:artifacts resource-change-event)) + resource-part { :asdcResourceId + (:resourceInvariantUUID resource-change-event) }] + + (map #(-> service-part + (merge resource-part) + ; WATCH! Using artifactName over artifactUUID because artifactUUID + ; is variant between versions. ASDC folks should be adding invariant + ; UUID. + (assoc :typeName (:artifactName %) + :typeVersion (Integer. (:artifactVersion %)) + :blueprintTemplate (get-blueprint-func (:artifactURL %)) + :serviceLocations (get-locations-func resource-change-event + (:artifactName %)) + + ) + ) + dcae-artifacts) + ))] + + (flatten (map #(generate-for-resource %) (:resources service-change-event))) + )))) + + +(defn pick-out-artifact + "Given dcae service type, fetch complementary asdc artifact" + [service-metadata request] + (let [target-resource (:asdcResourceId request) + resource-metadata (first (filter #(= (:resourceInvariantUUID %) target-resource) + (:resources service-metadata))) + target-artifact (:typeName request) + artifact-metadata (filter #(= (:artifactName %) target-artifact) + (:artifacts resource-metadata)) + ] + (first artifact-metadata))) diff --git a/src/sch/util.clj b/src/sch/util.clj new file mode 100644 index 0000000..f5b0bc2 --- /dev/null +++ b/src/sch/util.clj @@ -0,0 +1,48 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.util + (:require [clj-http.client :as client] + [clj-yaml.core :as yaml] + [cheshire.core :refer [parse-string]] + ) + (:gen-class)) + + +(defn- read-config-http-json + [config-url] + (let [resp (client/get config-url)] + (if (= (:status resp) 200) + (parse-string (:body resp) true) + ))) + +(defn read-config + "Read configuration from file or from an http server + + Returns a native map representation of the configuration" + [config-path] + (letfn [(is-http? [config-path] + (not (nil? (re-find #"(?:https|http)://.*" config-path))))] + (if (is-http? config-path) + (read-config-http-json config-path) + (yaml/parse-string (slurp config-path)) + ) + )) + diff --git a/test/sch/handle_test.clj b/test/sch/handle_test.clj new file mode 100644 index 0000000..9cbf9f7 --- /dev/null +++ b/test/sch/handle_test.clj @@ -0,0 +1,38 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.handle-test + (:use (clojure test)) + (:require [cheshire.core :refer [parse-stream]] + [sch.handle :refer :all]) + ) + +(deftest deployed-funcs-test + (let [requests [{:asdcResourceId "123" :typeName "pizza"} + {:asdcResourceId "456" :typeName "hamburger"} + {:asdcResourceId "789" :typeName "hotdog"}] + attempted [{:asdcResourceId "456" :typeName "hamburger"} + {:asdcResourceId "789" :typeName "hotdog"}] + completed [{:asdcResourceId "789" :typeName "hotdog"}] + ] + (is (= (deployed-ok attempted completed) completed)) + (is (= (deployed-error attempted completed) [(first attempted)])) + (is (= (deployed-already requests attempted) [(first requests)])) + )) diff --git a/test/sch/parse_test.clj b/test/sch/parse_test.clj new file mode 100644 index 0000000..06b5734 --- /dev/null +++ b/test/sch/parse_test.clj @@ -0,0 +1,63 @@ +; ============LICENSE_START======================================================= +; org.onap.dcae +; ================================================================================ +; Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +; ================================================================================ +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. +; ============LICENSE_END========================================================= +; +; ECOMP is a trademark and service mark of AT&T Intellectual Property. + +(ns sch.parse-test + (:use (clojure test)) + (:require [cheshire.core :refer [parse-stream]] + [sch.parse :refer :all]) + ) + + +; TODO: May want to use fixtures.. +(defn read-change-event! + [event-file-path] + (with-open [rdr (clojure.java.io/reader event-file-path)] + (parse-stream rdr true)) + ) + +(deftest generate-test + (letfn [(get-blueprint [artifact-url] + "Fake blueprint")] + (let [change-event (read-change-event! "fixtures/4_insert.json") + get-locations (partial get-service-locations slurp) + service-types (generate-dcae-service-type-requests get-blueprint + get-locations + change-event) + expected-st { :asdcResourceId "3d5927fc-a28e-41e9-9e79-57289aa7f754", + :asdcServiceId "9eaf59ee-2fe0-48a9-8d20-6f9b09ba807b", + :owner "MICHAEL SHITRIT", + :typeName "sample-blueprint", + :typeVersion 2, + :blueprintTemplate "Fake blueprint", + :asdcServiceURL "/asdc/v1/catalog/services/9eaf59ee-2fe0-48a9-8d20-6f9b09ba807b/metadata" + :serviceLocations ["CLLI1" "CLL2"] } + ] + + (is (= (count service-types) 1)) + (is (= expected-st (first service-types))) + ))) + +(deftest pick-out-artifact-test + (let [service-metadata (read-change-event! "fixtures/4_insert.json") + stub-request { :asdcResourceId "3d5927fc-a28e-41e9-9e79-57289aa7f754" + :typeName "sample-blueprint" } + actual-artifact (pick-out-artifact service-metadata stub-request)] + (is (= (:artifactName actual-artifact) (:typeName stub-request))) + )) |