From 70253f7088be04125d9fac8f9bddfaa63778608e Mon Sep 17 00:00:00 2001 From: Alex Shatov Date: Tue, 16 Jan 2018 11:14:52 -0500 Subject: variable collection of policies per component * new feature variable collection of policies per component in DCAE Change-Id: Iafe366edd749220b347d1ba6f530d54c95e2f1a7 Issue-ID: DCAEGEN2-249 Signed-off-by: Alex Shatov --- deployment-handler-API.yaml | 19 +- deployment-handler.js | 2 +- lib/policy.js | 512 ++++++++++++++++++++++++++++---------------- package.json | 2 +- pom.xml | 4 +- tests/test_policy.js | 2 +- version.properties | 4 +- 7 files changed, 355 insertions(+), 190 deletions(-) diff --git a/deployment-handler-API.yaml b/deployment-handler-API.yaml index 31395a5..eb1aed6 100644 --- a/deployment-handler-API.yaml +++ b/deployment-handler-API.yaml @@ -3,7 +3,7 @@ swagger: '2.0' info: - version: "4.1.0" + version: "4.2.0" title: "deployment-handler API" license: name: "Apache 2.0" @@ -469,11 +469,26 @@ definitions: description: request to update policies on DCAE components. type: object required: + - catch_up - latest_policies + - removed_policies properties: + catch_up: + description: "flag to indicate whether the request contains all the policies in PDP or not" + type: boolean + default: false + latest_policies: - description: "dictionary of (policy_id -> Policy object). In example: replace additionalProp1,2,3 with policy_id1,2,3 values" + description: "dictionary of (policy_id -> DCAEPolicy object). In example: replace additionalProp1,2,3 with policy_id1,2,3 values" + type: object + default: {} + additionalProperties: + $ref: "#/definitions/DCAEPolicy" + + removed_policies: + description: "dictionary of (policy_id -> DCAEPolicy object). In example: replace additionalProp1,2,3 with policy_id1,2,3 values" type: object + default: {} additionalProperties: $ref: "#/definitions/DCAEPolicy" diff --git a/deployment-handler.js b/deployment-handler.js index 1d59733..15b2807 100644 --- a/deployment-handler.js +++ b/deployment-handler.js @@ -18,7 +18,7 @@ See the License for the specific language governing permissions and limitations "use strict"; -const API_VERSION = "4.1.0"; +const API_VERSION = "4.2.0"; const fs = require('fs'); const util = require('util'); diff --git a/lib/policy.js b/lib/policy.js index 620870c..482650a 100644 --- a/lib/policy.js +++ b/lib/policy.js @@ -1,181 +1,331 @@ -/* -Copyright(c) 2017 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. - -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. -*/ - -/** - * handling policy updates - */ - -"use strict"; - -const POLICY_UPDATE_OPERATION = "dcae.interfaces.policy.policy_update"; - -const config = process.mainModule.exports.config; -const createError = require('./dispatcher-error').createDispatcherError; -const logger = require('./logging').getLogger(); - -var cloudify = require("./cloudify.js"); - -// Set config for cloudify interface library -cloudify.setAPIAddress(config.cloudify.url); -cloudify.setCredentials(config.cloudify.user, config.cloudify.password); -cloudify.setLogger(logger); - -/** - * receive the policy-updated message from the policy-handler - */ -function policyUpdate(req, res, next) { - var latest_policies = JSON.stringify((req.body && req.body.latest_policies) || {}); - logger.debug(req.dcaeReqId, "policyUpdate " + req.originalUrl + " " + latest_policies); - /** - * reply to and free up the policy_handler - */ - res.json({}); - - latest_policies = JSON.parse(latest_policies); - /** - * filter out the policies to what is deployed in components and needs updating (new policyVersion) - */ - var policy_deployments = {}; - var policy_ids = {}; - - cloudify.getNodeInstances(req, function(node_instances) { - node_instances.forEach(node_instance => { - if (!node_instance.runtime_properties || !node_instance.runtime_properties.policies) { - return; - } - var deployment = policy_deployments[node_instance.deployment_id] || { - "deployment_id": node_instance.deployment_id, "policies": {}, "component_ids": [] - }; - - logger.debug(req.dcaeReqId, "have policy on node_instance: " + JSON.stringify(node_instance)); - var have_policies = false; - Object.keys(node_instance.runtime_properties.policies).forEach(policy_id => { - var deployed_policy = node_instance.runtime_properties.policies[policy_id]; - var latest_policy = latest_policies[policy_id]; - if (!latest_policy || !latest_policy.policy_body - || isNaN(latest_policy.policy_body.policyVersion) - || latest_policy.policy_body.policyVersion - === (deployed_policy.policy_body && deployed_policy.policy_body.policyVersion)) { - return; - } - have_policies = true; - deployment.policies[policy_id] = latest_policy; - policy_ids[policy_id] = true; - }); - if (have_policies) { - deployment.component_ids.push(node_instance.id); - policy_deployments[deployment.deployment_id] = deployment; - } - }); - - logger.debug(req.dcaeReqId, "collected policy_deployments to update " + JSON.stringify(policy_deployments)); - }) - .then(function(result) { - logger.debug(req.dcaeReqId, "finished loading policy_deployments" + JSON.stringify(result)); - if (result.status !== 200) { - const error_msg = "failed to retrieve component policies from cloudify " + result.message; - logger.error(createError(error_msg, result.status, "api", 502, 'cloudify-manager'), req); - logger.audit(req, result.status, error_msg); - return; - } - - var deployment_ids = Object.keys(policy_deployments); - var policy_id_count = Object.keys(policy_ids).length; - if (!deployment_ids.length) { - const msg = "no updated policies to apply to deployments"; - logger.debug(req.dcaeReqId, msg); - logger.audit(req, result.status, msg); - return; - } - const msg = "going to apply updated policies[" + policy_id_count + "] to deployments " + deployment_ids.length; - logger.debug(req.dcaeReqId, msg + ": " + JSON.stringify(deployment_ids)); - logger.audit(req, result.status, msg); - deployment_ids.forEach(deployment_id => { - var deployment = policy_deployments[deployment_id]; - deployment.policies = Object.keys(deployment.policies).map(policy_id => { - return deployment.policies[policy_id]; - }); - - logger.debug(req.dcaeReqId, "ready to execute-operation policy-update on deployment " + JSON.stringify(deployment)); - cloudify.executeOperation(req, deployment.deployment_id, POLICY_UPDATE_OPERATION, - {'updated_policies': deployment.policies}, deployment.component_ids); - }); - }); -} - -/** - * retrieve all component-policies from cloudify - */ -function getComponentPoliciesFromCloudify(req, res, next) { - logger.debug(req.dcaeReqId, "getComponentPoliciesFromCloudify " + req.originalUrl); - var response = {"requestId": req.dcaeReqId}; - response.started = new Date(); - response.component_policies = []; - response.component_ids = []; - response.node_instances = []; - - cloudify.getNodeInstances(req, function(node_instances) { - Array.prototype.push.apply(response.node_instances, node_instances); - node_instances.forEach(node_instance => { - if (!node_instance.runtime_properties || !node_instance.runtime_properties.policies) { - return; - } - - var policies_count = 0; - Object.keys(node_instance.runtime_properties.policies).forEach(policy_id => { - ++policies_count; - var policy = node_instance.runtime_properties.policies[policy_id]; - policy.component_id = node_instance.id; - policy.deployment_id = node_instance.deployment_id; - response.component_policies.push(policy); - }); - if (policies_count) { - response.component_ids.push({ - "component_id" : node_instance.id, - "policies_count" : policies_count - }); - } - }); - - logger.debug(req.dcaeReqId, "collected " + response.component_ids.length - + " component_ids: " + JSON.stringify(response.component_ids) - + " component_policies: " + JSON.stringify(response.component_policies)); - }) - .then(function(result) { - response.ended = new Date(); - response.status = result.status; - response.message = result.message; - logger.debug(req.dcaeReqId, result.message); - if (result.status !== 200) { - logger.error(createError(result.message, result.status, "api", 502, 'cloudify-manager'), req); - } - res.status(result.status).json(response); - logger.audit(req, result.status, result.message); - }); -} - -// ======================================================== - -const app = require('express')(); -app.set('x-powered-by', false); -app.set('etag', false); -app.use(require('./middleware').checkType('application/json')); -app.use(require('body-parser').json({strict: true})); - -app.post('/', policyUpdate); -app.get('/components', getComponentPoliciesFromCloudify); - -module.exports = app; +/* +Copyright(c) 2017 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. +*/ + +/** + * handling policy updates + */ + +"use strict"; + +const POLICY_UPDATE_OPERATION = "dcae.interfaces.policy.policy_update"; + +const config = process.mainModule.exports.config; +const createError = require('./dispatcher-error').createDispatcherError; +const logger = require('./logging').getLogger(); + +var cloudify = require("./cloudify.js"); + +// Set config for cloudify interface library +cloudify.setAPIAddress(config.cloudify.url); +cloudify.setCredentials(config.cloudify.user, config.cloudify.password); +cloudify.setLogger(logger); + +/** + * receive the policy-updated message from the policy-handler + */ +function policyUpdate(req, res, next) { + + const policy_update = { + catch_up : req.body && req.body.catch_up, + latest_policies : JSON.stringify((req.body && req.body.latest_policies) || {}), + removed_policies : JSON.stringify((req.body && req.body.removed_policies) || {}), + errored_policies : JSON.stringify((req.body && req.body.errored_policies) || {}), + policy_deployments : {}, + updated_policy_ids : {}, + added_policy_ids : {}, + removed_policy_ids : {} + }; + + logger.debug(req.dcaeReqId, "policyUpdate " + + req.method + ' ' + req.protocol + '://' + req.get('host') + req.originalUrl + + " catch_up: " + policy_update.catch_up + + " latest_policies: " + policy_update.latest_policies + + " removed_policies: " + policy_update.removed_policies + + " errored_policies: " + policy_update.errored_policies + ); + /** + * reply to and free up the policy_handler + */ + res.json({}); + + policy_update.latest_policies = JSON.parse(policy_update.latest_policies); + policy_update.removed_policies = JSON.parse(policy_update.removed_policies); + policy_update.errored_policies = JSON.parse(policy_update.errored_policies); + + /** + * filter out the policies to what is deployed in components and needs updating (new policyVersion) + */ + + const collect_policy_deployments = function(node_instances) { + node_instances.forEach(node_instance => { + if (!node_instance.runtime_properties + || (!node_instance.runtime_properties.policies + && !node_instance.runtime_properties.policy_filters)) { + return; + } + logger.debug(req.dcaeReqId, "checking policies on node_instance: " + JSON.stringify(node_instance)); + + const deployment = policy_update.policy_deployments[node_instance.deployment_id] || { + "deployment_id": node_instance.deployment_id, + "updated_policies": {}, + "added_policies": {}, + "removed_policy_ids": {}, + "node_instance_ids": [] + }; + + var have_policies = false; + const deployed_policies = node_instance.runtime_properties.policies || {}; + + Object.keys(deployed_policies).forEach(policy_id => { + const latest_policy = policy_update.latest_policies[policy_id]; + if (policy_update.removed_policies[policy_id] + || (policy_update.catch_up && !latest_policy + && !policy_update.errored_policies[policy_id])) { + have_policies = true; + deployment.removed_policy_ids[policy_id] = true; + policy_update.removed_policy_ids[policy_id] = true; + logger.debug(req.dcaeReqId, "going to remove policy " + policy_id + " from node_instance: " + JSON.stringify(node_instance)); + return; + } + + const deployed_policy = deployed_policies[policy_id]; + if (!latest_policy || !latest_policy.policy_body + || isNaN(latest_policy.policy_body.policyVersion) + || latest_policy.policy_body.policyVersion + === (deployed_policy.policy_body && deployed_policy.policy_body.policyVersion)) { + return; + } + have_policies = true; + deployment.updated_policies[policy_id] = latest_policy; + policy_update.updated_policy_ids[policy_id] = true; + logger.debug(req.dcaeReqId, "going to update policy " + policy_id + " on node_instance: " + JSON.stringify(node_instance)); + }); + + const policy_filters = node_instance.runtime_properties.policy_filters || {}; + const policy_filter_ids = Object.keys(policy_filters); + if (policy_filter_ids.length) { + logger.debug(req.dcaeReqId, "matching latest policies to policy_filters[" + policy_filter_ids.length + "] on node_instance: " + JSON.stringify(node_instance)); + try { + Object.keys(policy_update.latest_policies).forEach(policy_id => { + if (deployment.updated_policies[policy_id] || deployed_policies[policy_id]) {return;} + + const latest_policy = policy_update.latest_policies[policy_id]; + const policy_body = latest_policy && latest_policy.policy_body; + if (!policy_body || isNaN(policy_body.policyVersion)) {return;} + const policy_name = policy_body.policyName; + if (!policy_name) {return;} + const matching_conditions = policy_body.matchingConditions || {}; + + logger.debug(req.dcaeReqId, "matching policy " + JSON.stringify(latest_policy)); + policy_filter_ids.some(policy_filter_id => { + const policy_filter = policy_filters[policy_filter_id].policy_filter; + if (!policy_filter || !policy_filter.policyName) {return false;} + + logger.debug(req.dcaeReqId, "matching to policy_filter " + JSON.stringify(policy_filter)); + + if (!!policy_filter.onapName + && policy_filter.onapName !== matching_conditions.ONAPName) { + logger.debug(req.dcaeReqId, "not match policy_filter_id " + policy_filter_id + + " by ONAPName: " + + policy_filter.onapName + " !== " + matching_conditions.ONAPName); + return false; + } + if (!!policy_filter.configName + && policy_filter.configName !== matching_conditions.ConfigName) { + logger.debug(req.dcaeReqId, "not match policy_filter_id " + policy_filter_id + + " by configName: " + + policy_filter.configName + " !== " + matching_conditions.ConfigName); + return false; + } + + if (policy_filter.configAttributes + && !Object.keys(policy_filter.configAttributes).every(filter_key => { + return (matching_conditions.hasOwnProperty(filter_key) + && policy_filter.configAttributes[filter_key] + === matching_conditions[filter_key]); + })) { + logger.debug(req.dcaeReqId, "not match policy_filter_id " + policy_filter_id + + " by configAttributes: " + + JSON.stringify(policy_filter.configAttributes) + " !== " + JSON.stringify(matching_conditions)); + return false; + } + + if (policy_filter.policyName !== policy_id && policy_filter.policyName !== policy_name) { + const match_policy_name = new RegExp(policy_filter.policyName); + if (!match_policy_name.test(policy_name)) { + logger.debug(req.dcaeReqId, "not match policy_filter_id " + policy_filter_id + + " by policyName: " + + policy_filter.policyName + " versus " + policy_name); + return false; + } + } + + have_policies = true; + if (!deployment.added_policies[policy_filter_id]) { + deployment.added_policies[policy_filter_id] = { + "policy_filter_id" : policy_filter_id, + "policies" : {} + }; + } + deployment.added_policies[policy_filter_id].policies[policy_id] = latest_policy; + policy_update.added_policy_ids[policy_id] = true; + logger.debug(req.dcaeReqId, "going to add policy " + JSON.stringify(latest_policy) + + " per policy_filter_id " + policy_filter_id + + " on node_instance: " + JSON.stringify(node_instance)); + return true; + }); + }); + } catch (e) { + const error_msg = "error on matching policy to filter " + (e.message || "") + + " " + (e.stack || "").replace(/\n/g, " ") + logger.error(createError(error_msg, 500, "api", 553, 'deployment-handler'), req); + } + } + + if (have_policies) { + deployment.node_instance_ids.push(node_instance.id); + policy_update.policy_deployments[deployment.deployment_id] = deployment; + } + }); + + logger.debug(req.dcaeReqId, "collected policy_deployments to update " + JSON.stringify(policy_update.policy_deployments)); + }; + + const update_policies_on_deployments = function(result) { + logger.debug(req.dcaeReqId, "finished loading policy_deployments" + JSON.stringify(result)); + if (result.status !== 200) { + const error_msg = "failed to retrieve component policies from cloudify " + result.message; + logger.error(createError(error_msg, result.status, "api", 502, 'cloudify-manager'), req); + logger.audit(req, result.status, error_msg); + return; + } + + const deployment_ids = Object.keys(policy_update.policy_deployments); + if (!deployment_ids.length) { + const audit_msg = "no updated policies to apply to deployments"; + logger.debug(req.dcaeReqId, audit_msg); + logger.audit(req, result.status, audit_msg); + return; + } + const audit_msg = "going to apply updated policies[" + Object.keys(policy_update.updated_policy_ids).length + + "] and added policies[" + Object.keys(policy_update.added_policy_ids).length + + "] and removed policies[" + Object.keys(policy_update.removed_policy_ids).length + + "] to deployments[" + deployment_ids.length + "]"; + logger.debug(req.dcaeReqId, audit_msg + ": " + JSON.stringify(deployment_ids)); + logger.audit(req, result.status, audit_msg); + deployment_ids.forEach(deployment_id => { + const deployment = policy_update.policy_deployments[deployment_id]; + deployment.updated_policies = Object.keys(deployment.updated_policies).map(policy_id => { + return deployment.updated_policies[policy_id]; + }); + deployment.removed_policy_ids = Object.keys(deployment.removed_policy_ids); + + logger.debug(req.dcaeReqId, "ready to execute-operation policy-update on deployment " + JSON.stringify(deployment)); + cloudify.executeOperation(req, deployment.deployment_id, POLICY_UPDATE_OPERATION, + { + 'updated_policies': deployment.updated_policies, + 'added_policies': deployment.added_policies, + 'removed_policies': deployment.removed_policy_ids + }, + deployment.node_instance_ids + ); + }); + }; + + cloudify.getNodeInstances(req, collect_policy_deployments).then(update_policies_on_deployments); +} + +/** + * retrieve all component-policies from cloudify + */ +function getComponentPoliciesFromCloudify(req, res, next) { + logger.debug(req.dcaeReqId, "getComponentPoliciesFromCloudify " + req.originalUrl); + var response = {"requestId": req.dcaeReqId}; + response.started = new Date(); + response.node_instance_ids = []; + response.component_policies = []; + response.component_policy_filters = []; + response.node_instances = []; + + cloudify.getNodeInstances(req, function(node_instances) { + Array.prototype.push.apply(response.node_instances, node_instances); + node_instances.forEach(node_instance => { + if (!node_instance.runtime_properties + || (!node_instance.runtime_properties.policies + && !node_instance.runtime_properties.policy_filters)) { + return; + } + + var policies_count = 0; + var policy_filters_count = 0; + if (node_instance.runtime_properties.policies) { + Object.keys(node_instance.runtime_properties.policies).forEach(policy_id => { + ++policies_count; + var policy = node_instance.runtime_properties.policies[policy_id]; + policy.component_id = node_instance.id; + policy.deployment_id = node_instance.deployment_id; + response.component_policies.push(policy); + }); + } + if (node_instance.runtime_properties.policy_filters) { + Object.keys(node_instance.runtime_properties.policy_filters).forEach(policy_filter => { + ++policy_filters_count; + policy_filter = node_instance.runtime_properties.policy_filters[policy_filter]; + policy.component_id = node_instance.id; + policy.deployment_id = node_instance.deployment_id; + response.component_policy_filters.push(policy_filter); + }); + } + if (policies_count + policy_filters_count) { + response.node_instance_ids.push({ + "node_instance_id" : node_instance.id, + "policies_count" : policies_count, + "policy_filters_count" : policy_filters_count + }); + } + }); + + logger.debug(req.dcaeReqId, "collected " + response.node_instance_ids.length + + " node_instance_ids: " + JSON.stringify(response.node_instance_ids) + + " component_policies: " + JSON.stringify(response.component_policies) + + " component_policy_filters: " + JSON.stringify(response.component_policy_filters) + ); + }) + .then(function(result) { + response.ended = new Date(); + response.status = result.status; + response.message = result.message; + logger.debug(req.dcaeReqId, result.message); + if (result.status !== 200) { + logger.error(createError(result.message, result.status, "api", 502, 'cloudify-manager'), req); + } + res.status(result.status).json(response); + logger.audit(req, result.status, result.message); + }); +} + +// ======================================================== + +const app = require('express')(); +app.set('x-powered-by', false); +app.set('etag', false); +app.use(require('./middleware').checkType('application/json')); +app.use(require('body-parser').json({strict: true})); + +app.post('/', policyUpdate); +app.get('/components', getComponentPoliciesFromCloudify); + +module.exports = app; diff --git a/package.json b/package.json index 2a4f7e5..5308cf5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "onap-dcae-deployment-handler", - "version": "4.2.2", + "version": "4.3.0", "description": "ONAP DCAE Deployment Handler", "main": "deployment-handler.js", "dependencies": { diff --git a/pom.xml b/pom.xml index 3eb1200..945f4c5 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property. org.onap.dcaegen2.platform deployment-handler dcaegen2-platform-deployment-handler - 1.2.0-SNAPSHOT + 2.0.0-SNAPSHOT http://maven.apache.org UTF-8 @@ -111,7 +111,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property. - org.apache.maven.plugins diff --git a/tests/test_policy.js b/tests/test_policy.js index c0ad243..8161032 100644 --- a/tests/test_policy.js +++ b/tests/test_policy.js @@ -65,7 +65,7 @@ function create_policy_body(policy_id, policy_version=1) { POLICY_VERSION: this_ver, POLICY_CONFIG: config, "matchingConditions": { - "ECOMPName": "DCAE", + "ONAPName": "DCAE", "ConfigName": "alex_config_name" }, "responseAttributes": {}, diff --git a/version.properties b/version.properties index 07578e5..fbd1b27 100644 --- a/version.properties +++ b/version.properties @@ -1,5 +1,5 @@ -major=1 -minor=2 +major=2 +minor=0 patch=0 base_version=${major}.${minor}.${patch} release_version=${base_version} -- cgit 1.2.3-korg