Initial commit for OpenECOMP SDN-C OA&M

Change-Id: I7ab579fd0d206bf356f36d52dcdf4f71f1fa2680
Signed-off-by: Timoney, Daniel (dt5972) <dtimoney@att.com>

Former-commit-id: 2a9f0edd09581f907e62ec4689b5ac94dd5382ba

5 files changed, 1402 insertions, 0 deletions

diff --git a/dgbuilder/red/nodes/Node.js b/dgbuilder/red/nodes/Node.js
new file mode 100644
index 00000000..0e6fc525
--- /dev/null
+++ b/dgbuilder/red/nodes/Node.js
@@ -0,0 +1,147 @@
+/**
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var util = require("util");
+var EventEmitter = require("events").EventEmitter;
+var clone = require("clone");
+var when = require("when");
+
+var flows = require("./flows");
+var comms = require("../comms");
+
+function Node(n) {
+    this.id = n.id;
+    flows.add(this);
+    this.type = n.type;
+    if (n.name) {
+        this.name = n.name;
+    }
+    this.wires = n.wires||[];
+}
+
+util.inherits(Node,EventEmitter);
+
+Node.prototype._on = Node.prototype.on;
+
+Node.prototype.on = function(event,callback) {
+    var node = this;
+    if (event == "close") {
+        if (callback.length == 1) {
+            this.close = function() {
+                return when.promise(function(resolve) {
+                    callback.call(node,function() {
+                        resolve();
+                    });
+                });
+            }
+        } else {
+            this.close = callback;
+        }
+    } else {
+        this._on(event,callback);
+    }
+}
+
+Node.prototype.close = function() {
+}
+
+Node.prototype.send = function(msg) {
+    // instanceof doesn't work for some reason here
+    if (msg == null) {
+        return;
+    } else if (!util.isArray(msg)) {
+        msg = [msg];
+    }
+    for (var i=0;i<this.wires.length;i++) {
+        var wires = this.wires[i];
+        if (i < msg.length) {
+            if (msg[i] != null) {
+                var msgs = msg[i];
+                if (!util.isArray(msg[i])) {
+                    msgs = [msg[i]];
+                }
+                //if (wires.length == 1) {
+                //    // Single recipient, don't need to clone the message
+                //    var node = flows.get(wires[0]);
+                //    if (node) {
+                //        for (var k in msgs) {
+                //            var mm = msgs[k];
+                //            node.receive(mm);
+                //        }
+                //    }
+                //} else {
+                    // Multiple recipients, must send message copies
+                    for (var j=0;j<wires.length;j++) {
+                        var node = flows.get(wires[j]);
+                        if (node) {
+                            for (var k=0;k<msgs.length;k++) {
+                                var mm = msgs[k];
+                                // Temporary fix for #97
+                                // TODO: remove this http-node-specific fix somehow
+                                var req = mm.req;
+                                var res = mm.res;
+                                delete mm.req;
+                                delete mm.res;
+                                var m = clone(mm);
+                                if (req) {
+                                    m.req = req;
+                                    mm.req = req;
+                                }
+                                if (res) {
+                                    m.res = res;
+                                    mm.res = res;
+                                }
+                                node.receive(m);
+                            }
+                        }
+                    }
+                //}
+            }
+        }
+    }
+}
+
+Node.prototype.receive = function(msg) {
+    this.emit("input",msg);
+}
+
+function log_helper(self, level, msg) {
+    var o = {level:level, id:self.id, type:self.type, msg:msg};
+    if (self.name) {
+        o.name = self.name;
+    }
+    self.emit("log",o);
+}
+
+Node.prototype.log = function(msg) {
+    log_helper(this, 'log', msg);
+}
+
+Node.prototype.warn = function(msg) {
+    log_helper(this, 'warn', msg);
+}
+
+Node.prototype.error = function(msg) {
+    log_helper(this, 'error', msg);
+}
+
+/**
+ * status: { fill:"red|green", shape:"dot|ring", text:"blah" }
+ */
+Node.prototype.status = function(status) {
+    comms.publish("status/"+this.id,status,true);
+}
+module.exports = Node;
diff --git a/dgbuilder/red/nodes/credentials.js b/dgbuilder/red/nodes/credentials.js
new file mode 100644
index 00000000..22e78d81
--- /dev/null
+++ b/dgbuilder/red/nodes/credentials.js
@@ -0,0 +1,208 @@
+/**
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var util = require("util");
+var when = require("when");
+
+var credentialCache = {};
+var storage = null;
+var credentialsDef = {};
+var redApp = null;
+
+/**
+ * Adds an HTTP endpoint to allow look up of credentials for a given node id.
+ */
+function registerEndpoint(type) {
+    redApp.get('/credentials/' + type + '/:id', function (req, res) {
+        // TODO: This could be a generic endpoint with the type value
+        //       parameterised.
+        //
+        // TODO: It should verify the given node id is of the type specified -
+        //       but that would add a dependency from this module to the
+        //       registry module that knows about node types.
+        var nodeType = type;
+        var nodeID = req.params.id;
+
+        var credentials = credentialCache[nodeID];
+        if (credentials === undefined) {
+            res.json({});
+            return;
+        }
+        var definition = credentialsDef[nodeType];
+
+        var sendCredentials = {};
+        for (var cred in definition) {
+            if (definition.hasOwnProperty(cred)) {
+                if (definition[cred].type == "password") {
+                    var key = 'has_' + cred;
+                    sendCredentials[key] = credentials[cred] != null && credentials[cred] !== '';
+                    continue;
+                }
+                sendCredentials[cred] = credentials[cred] || '';
+            }
+        }
+        res.json(sendCredentials);
+
+    });
+}
+
+
+module.exports = {
+    init: function (_storage) {
+        storage = _storage;
+        // TODO: this should get passed in init function call rather than
+        //       required directly.
+        redApp = require("../server").app;
+    },
+    
+    /**
+     * Loads the credentials from storage.
+     */
+    load: function () {
+        return storage.getCredentials().then(function (creds) {
+            credentialCache = creds;
+        }).otherwise(function (err) {
+            util.log("[red] Error loading credentials : " + err);
+        });
+    },
+    
+    /**
+     * Adds a set of credentials for the given node id.
+     * @param id the node id for the credentials
+     * @param creds an object of credential key/value pairs
+     * @return a promise for the saving of credentials to storage
+     */
+    add: function (id, creds) {
+        credentialCache[id] = creds;
+        return storage.saveCredentials(credentialCache);
+    },
+
+    /**
+     * Gets the credentials for the given node id.
+     * @param id the node id for the credentials
+     * @return the credentials
+     */
+    get: function (id) {
+        return credentialCache[id];
+    },
+
+    /**
+     * Deletes the credentials for the given node id.
+     * @param id the node id for the credentials
+     * @return a promise for the saving of credentials to storage
+     */
+    delete: function (id) {
+        delete credentialCache[id];
+        storage.saveCredentials(credentialCache);
+    },
+
+    /**
+     * Deletes any credentials for nodes that no longer exist
+     * @param getNode a function that can return a node for a given id
+     * @return a promise for the saving of credentials to storage
+     */
+    clean: function (getNode) {
+        var deletedCredentials = false;
+        for (var c in credentialCache) {
+            if (credentialCache.hasOwnProperty(c)) {
+                var n = getNode(c);
+                if (!n) {
+                    deletedCredentials = true;
+                    delete credentialCache[c];
+                }
+            }
+        }
+        if (deletedCredentials) {
+            return storage.saveCredentials(credentialCache);
+        } else {
+            return when.resolve();
+        }
+    },
+    
+    /**
+     * Registers a node credential definition.
+     * @param type the node type
+     * @param definition the credential definition
+     */
+    register: function (type, definition) {
+        var dashedType = type.replace(/\s+/g, '-');
+        credentialsDef[dashedType] = definition;
+        registerEndpoint(dashedType);
+    },
+    
+    /**
+     * Extracts and stores any credential updates in the provided node.
+     * The provided node may have a .credentials property that contains
+     * new credentials for the node.
+     * This function loops through the credentials in the definition for
+     * the node-type and applies any of the updates provided in the node.
+     * 
+     * This function does not save the credentials to disk as it is expected
+     * to be called multiple times when a new flow is deployed.
+     *
+     * @param node the node to extract credentials from
+     */
+    extract: function(node) {
+        var nodeID = node.id;
+        var nodeType = node.type;
+        var newCreds = node.credentials;
+        if (newCreds) {
+            var savedCredentials = credentialCache[nodeID] || {};
+            
+            var dashedType = nodeType.replace(/\s+/g, '-');
+            var definition = credentialsDef[dashedType];
+            
+            if (!definition) {
+                util.log('Credential Type ' + nodeType + ' is not registered.');
+                return;
+            }
+            
+            for (var cred in definition) {
+                if (definition.hasOwnProperty(cred)) {
+                    if (newCreds[cred] === undefined) {
+                        continue;
+                    }
+                    if (definition[cred].type == "password" && newCreds[cred] == '__PWRD__') {
+                        continue;
+                    }
+                    if (0 === newCreds[cred].length || /^\s*$/.test(newCreds[cred])) {
+                        delete savedCredentials[cred];
+                        continue;
+                    }
+                    savedCredentials[cred] = newCreds[cred];
+                }
+            }
+            credentialCache[nodeID] = savedCredentials;
+        }
+    },
+    
+    /**
+     * Saves the credentials to storage
+     * @return a promise for the saving of credentials to storage
+     */
+    save: function () {
+        return storage.saveCredentials(credentialCache);
+    },
+    
+    /**
+     * Gets the credential definition for the given node type
+     * @param type the node type
+     * @return the credential definition
+     */
+    getDefinition: function (type) {
+        return credentialsDef[type];
+    }
+}
diff --git a/dgbuilder/red/nodes/flows.js b/dgbuilder/red/nodes/flows.js
new file mode 100644
index 00000000..b0b5d514
--- /dev/null
+++ b/dgbuilder/red/nodes/flows.js
@@ -0,0 +1,220 @@
+/**
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+var util = require("util");
+var when = require("when");
+
+var typeRegistry = require("./registry");
+var credentials = require("./credentials");
+var log = require("../log");
+var events = require("../events");
+
+var storage = null;
+
+var nodes = {};
+var activeConfig = [];
+var missingTypes = [];
+
+events.on('type-registered',function(type) {
+        if (missingTypes.length > 0) {
+            var i = missingTypes.indexOf(type);
+            if (i != -1) {
+                missingTypes.splice(i,1);
+                util.log("[red] Missing type registered: "+type);
+                if (missingTypes.length === 0) {
+                    parseConfig();
+                }
+            }
+        }
+});
+
+/**
+ * Parses the current activeConfig and creates the required node instances
+ */ 
+function parseConfig() {
+    var i;
+    var nt;
+    missingTypes = [];
+    
+    // Scan the configuration for any unknown node types
+    for (i=0;i<activeConfig.length;i++) {
+        var type = activeConfig[i].type;
+        // TODO: remove workspace in next release+1
+        if (type != "workspace" && type != "tab") {
+            nt = typeRegistry.get(type);
+            if (!nt && missingTypes.indexOf(type) == -1) {
+                missingTypes.push(type);
+            }
+        }
+    }
+    // Abort if there are any missing types
+    if (missingTypes.length > 0) {
+        util.log("[red] Waiting for missing types to be registered:");
+        for (i=0;i<missingTypes.length;i++) {
+            util.log("[red]  - "+missingTypes[i]);
+        }
+        return;
+    }
+
+    util.log("[red] Starting flows");
+    events.emit("nodes-starting");
+    
+    // Instantiate each node in the flow
+    for (i=0;i<activeConfig.length;i++) {
+        var nn = null;
+        // TODO: remove workspace in next release+1
+        if (activeConfig[i].type != "workspace" && activeConfig[i].type != "tab") {
+            nt = typeRegistry.get(activeConfig[i].type);
+            if (nt) {
+                try {
+                    nn = new nt(activeConfig[i]);
+                }
+                catch (err) {
+                    util.log("[red] "+activeConfig[i].type+" : "+err);
+                }
+            }
+            // console.log(nn);
+            if (nn === null) {
+                util.log("[red] unknown type: "+activeConfig[i].type);
+            }
+        }
+    }
+    // Clean up any orphaned credentials
+    credentials.clean(flowNodes.get);
+    events.emit("nodes-started");
+}
+
+/**
+ * Stops the current activeConfig
+ */
+function stopFlows() {
+    if (activeConfig&&activeConfig.length > 0) {
+        util.log("[red] Stopping flows");
+    }
+    return flowNodes.clear();
+}
+
+var flowNodes = module.exports = {
+    init: function(_storage) {
+        storage = _storage;
+    },
+    
+    /**
+     * Load the current activeConfig from storage and start it running
+     * @return a promise for the loading of the config
+     */
+    load: function() {
+        return storage.getFlows().then(function(flows) {
+            return credentials.load().then(function() {
+                activeConfig = flows;
+                if (activeConfig && activeConfig.length > 0) {
+                    parseConfig();
+                }
+            });
+        }).otherwise(function(err) {
+            util.log("[red] Error loading flows : "+err);
+        });
+    },
+    
+    /**
+     * Add a node to the current active set
+     * @param n the node to add
+     */
+    add: function(n) {
+        nodes[n.id] = n;
+        n.on("log",log.log);
+    },
+    
+    /**
+     * Get a node
+     * @param i the node id
+     * @return the node
+     */
+    get: function(i) {
+        return nodes[i];
+    },
+    
+    /**
+     * Stops all active nodes and clears the active set
+     * @return a promise for the stopping of all active nodes
+     */
+    clear: function() {
+        return when.promise(function(resolve) {
+            events.emit("nodes-stopping");
+            var promises = [];
+            for (var n in nodes) {
+                if (nodes.hasOwnProperty(n)) {
+                    try {
+                        var p = nodes[n].close();
+                        if (p) {
+                            promises.push(p);
+                        }
+                    } catch(err) {
+                        nodes[n].error(err);
+                    }
+                }
+            }
+            when.settle(promises).then(function() {
+                events.emit("nodes-stopped");
+                nodes = {};
+                resolve();
+            });
+        });
+    },
+    
+    /**
+     * Provides an iterator over the active set of nodes
+     * @param cb a function to be called for each node in the active set
+     */
+    each: function(cb) {
+        for (var n in nodes) {
+            if (nodes.hasOwnProperty(n)) {
+                cb(nodes[n]);
+            }
+        }
+    },
+
+    /**
+     * @return the active configuration
+     */
+    getFlows: function() {
+        return activeConfig;
+    },
+    
+    /**
+     * Sets the current active config.
+     * @param config the configuration to enable
+     * @return a promise for the starting of the new flow
+     */
+    setFlows: function (config) {
+        // Extract any credential updates
+        for (var i=0; i<config.length; i++) {
+            var node = config[i];
+            if (node.credentials) {
+                credentials.extract(node);
+                delete node.credentials;
+            }
+        }
+        return credentials.save()
+            .then(function() { return storage.saveFlows(config);})
+            .then(function() { return stopFlows();})
+            .then(function () {
+                activeConfig = config;
+                parseConfig();
+            });
+    },
+    stopFlows: stopFlows
+};
diff --git a/dgbuilder/red/nodes/index.js b/dgbuilder/red/nodes/index.js
new file mode 100644
index 00000000..3d5ad719
--- /dev/null
+++ b/dgbuilder/red/nodes/index.js
@@ -0,0 +1,134 @@
+/**
+ * Copyright 2013, 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+var registry = require("./registry");
+var credentials = require("./credentials");
+var flows = require("./flows");
+var Node = require("./Node");
+
+/**
+ * Registers a node constructor
+ * @param type - the string type name
+ * @param constructor - the constructor function for this node type
+ * @param opts - optional additional options for the node
+ */
+function registerType(type,constructor,opts) {
+    if (opts && opts.credentials) {
+        credentials.register(type,opts.credentials);
+    }
+    registry.registerType(type,constructor);    
+}
+
+/**
+ * Called from a Node's constructor function, invokes the super-class
+ * constructor and attaches any credentials to the node.
+ * @param node the node object being created
+ * @param def the instance definition for the node 
+ */
+function createNode(node,def) {
+    Node.call(node,def);
+    var creds = credentials.get(node.id);
+    if (creds) {
+        node.credentials = creds;
+    }
+}
+
+function init(_settings,storage) {
+    credentials.init(storage);
+    flows.init(storage);
+    registry.init(_settings);
+}
+
+function checkTypeInUse(id) {
+    var nodeInfo = registry.getNodeInfo(id);
+    if (!nodeInfo) {
+        throw new Error("Unrecognised id: "+info);
+    }
+    var inUse = {};
+    flows.each(function(n) {
+        inUse[n.type] = (inUse[n.type]||0)+1;
+    });
+    var nodesInUse = [];
+    nodeInfo.types.forEach(function(t) {
+        if (inUse[t]) {
+            nodesInUse.push(t);
+        }
+    });
+    if (nodesInUse.length > 0) {
+        var msg = nodesInUse.join(", ");
+        throw new Error("Type in use: "+msg);
+    }
+}
+
+function removeNode(id) {
+    checkTypeInUse(id);
+    return registry.removeNode(id);
+}
+
+function removeModule(module) {
+    var info = registry.getNodeModuleInfo(module);
+    for (var i=0;i<info.nodes.length;i++) {
+        checkTypeInUse(info.nodes[i]);
+    }
+    return registry.removeModule(module);
+}
+
+
+function disableNode(id) {
+    checkTypeInUse(id);
+    return registry.disableNode(id);
+}
+
+module.exports = {
+    // Lifecycle
+    init: init,
+    load: registry.load,
+    
+    // Node registry
+    createNode: createNode,
+    getNode: flows.get,
+    
+    addNode: registry.addNode,
+    removeNode: removeNode,
+    
+    addModule: registry.addModule,
+    removeModule: removeModule,
+    
+    enableNode: registry.enableNode,
+    disableNode: disableNode,
+    
+    // Node type registry
+    registerType: registerType,
+    getType: registry.get,
+    getNodeInfo: registry.getNodeInfo,
+    getNodeModuleInfo: registry.getNodeModuleInfo,
+    getNodeList: registry.getNodeList,
+    getNodeConfigs: registry.getNodeConfigs,
+    getNodeConfig: registry.getNodeConfig,
+    clearRegistry: registry.clear,
+    cleanNodeList: registry.cleanNodeList,
+    
+    // Flow handling
+    loadFlows: flows.load,
+    stopFlows: flows.stopFlows,
+    setFlows: flows.setFlows,
+    getFlows: flows.getFlows,
+    
+    // Credentials
+    addCredentials: credentials.add,
+    getCredentials: credentials.get,
+    deleteCredentials: credentials.delete
+}
+
diff --git a/dgbuilder/red/nodes/registry.js b/dgbuilder/red/nodes/registry.js
new file mode 100644
index 00000000..f2073aff
--- /dev/null
+++ b/dgbuilder/red/nodes/registry.js
@@ -0,0 +1,693 @@
+/**
+ * Copyright 2014 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+ 
+var util = require("util");
+var when = require("when");
+var whenNode = require('when/node');
+var fs = require("fs");
+var path = require("path");
+var crypto = require("crypto"); 
+var UglifyJS = require("uglify-js");
+
+var events = require("../events");
+
+var Node;
+var settings;
+
+function filterNodeInfo(n) {
+    var r = {
+        id: n.id,
+        name: n.name,
+        types: n.types,
+        enabled: n.enabled
+    }
+    if (n.hasOwnProperty("loaded")) {
+        r.loaded = n.loaded;
+    }
+    if (n.hasOwnProperty("module")) {
+        r.module = n.module;
+    }
+    if (n.hasOwnProperty("err")) {
+        r.err = n.err.toString();
+    }
+    return r;
+}
+
+var registry = (function() {
+    var nodeConfigCache = null;
+    var nodeConfigs = {};
+    var nodeList = [];
+    var nodeConstructors = {};
+    var nodeTypeToId = {};
+    var nodeModules = {};
+    
+    function saveNodeList() {
+        var nodeList = {};
+        
+        for (var i in nodeConfigs) {
+            if (nodeConfigs.hasOwnProperty(i)) {
+                var nodeConfig = nodeConfigs[i];
+                var n = filterNodeInfo(nodeConfig);
+                n.file = nodeConfig.file;
+                delete n.loaded;
+                delete n.err;
+                delete n.file;
+                delete n.id;
+                nodeList[i] = n;
+            }
+        }
+        if (settings.available()) {
+            return settings.set("nodes",nodeList);
+        } else {
+            return when.reject("Settings unavailable");
+        }
+    }
+    
+    return {
+        init: function() {
+            if (settings.available()) {
+                nodeConfigs = settings.get("nodes")||{};
+                // Restore the node id property to individual entries
+                for (var id in nodeConfigs) {
+                    if (nodeConfigs.hasOwnProperty(id)) {
+                        nodeConfigs[id].id = id;
+                    }
+                }
+            } else {
+                nodeConfigs = {};
+            }
+            nodeModules = {};
+            nodeTypeToId = {};
+            nodeConstructors = {};
+            nodeList = [];
+            nodeConfigCache = null;
+        },
+        
+        addNodeSet: function(id,set) {
+            if (!set.err) {
+                set.types.forEach(function(t) {
+                    nodeTypeToId[t] = id;
+                });
+            }
+            
+            if (set.module) {
+                nodeModules[set.module] = nodeModules[set.module]||{nodes:[]};
+                nodeModules[set.module].nodes.push(id);
+            }
+            
+            nodeConfigs[id] = set;
+            nodeList.push(id);
+            nodeConfigCache = null;
+        },
+        removeNode: function(id) {
+            var config = nodeConfigs[id];
+            if (!config) {
+                throw new Error("Unrecognised id: "+id);
+            }
+            delete nodeConfigs[id];
+            var i = nodeList.indexOf(id);
+            if (i > -1) {
+                nodeList.splice(i,1);
+            }
+            config.types.forEach(function(t) {
+                delete nodeConstructors[t];
+                delete nodeTypeToId[t];
+            });
+            config.enabled = false;
+            config.loaded = false;
+            nodeConfigCache = null;
+            return filterNodeInfo(config);
+        },
+        removeModule: function(module) {
+            if (!settings.available()) {
+                throw new Error("Settings unavailable");
+            }
+            var nodes = nodeModules[module];
+            if (!nodes) {
+                throw new Error("Unrecognised module: "+module);
+            }
+            var infoList = [];
+            for (var i=0;i<nodes.nodes.length;i++) {
+                infoList.push(registry.removeNode(nodes.nodes[i]));
+            }
+            delete nodeModules[module];
+            saveNodeList();
+            return infoList;
+        },
+        getNodeInfo: function(typeOrId) {
+            if (nodeTypeToId[typeOrId]) {
+                return filterNodeInfo(nodeConfigs[nodeTypeToId[typeOrId]]);
+            } else if (nodeConfigs[typeOrId]) {
+                return filterNodeInfo(nodeConfigs[typeOrId]);
+            }
+            return null;
+        },
+        getNodeList: function() {
+            var list = [];
+            for (var id in nodeConfigs) {
+                if (nodeConfigs.hasOwnProperty(id)) {
+                    list.push(filterNodeInfo(nodeConfigs[id]))
+                }
+            }
+            return list;
+        },
+        registerNodeConstructor: function(type,constructor) {
+            if (nodeConstructors[type]) {
+                throw new Error(type+" already registered");
+            }
+            //TODO: Ensure type is known - but doing so will break some tests
+            //      that don't have a way to register a node template ahead
+            //      of registering the constructor
+            util.inherits(constructor,Node);
+            nodeConstructors[type] = constructor;
+            events.emit("type-registered",type);
+        },
+        
+        
+        /**
+         * Gets all of the node template configs
+         * @return all of the node templates in a single string
+         */
+        getAllNodeConfigs: function() {
+            if (!nodeConfigCache) {
+                var result = "";
+                var script = "";
+                for (var i=0;i<nodeList.length;i++) {
+                    var config = nodeConfigs[nodeList[i]];
+                    if (config.enabled && !config.err) {
+                        result += config.config;
+                        script += config.script;
+                    }
+                }
+                if (script.length > 0) {
+                    result += '<script type="text/javascript">';
+                    result += UglifyJS.minify(script, {fromString: true}).code;
+                    result += '</script>';
+                }
+                nodeConfigCache = result;
+            }
+            return nodeConfigCache;
+        },
+        
+        getNodeConfig: function(id) {
+            var config = nodeConfigs[id];
+            if (config) {
+                var result = config.config;
+                if (config.script) {
+                    result += '<script type="text/javascript">'+config.script+'</script>';
+                }
+                return result;
+            } else {
+                return null;
+            }
+        },
+        
+        getNodeConstructor: function(type) {
+            var config = nodeConfigs[nodeTypeToId[type]];
+            if (!config || (config.enabled && !config.err)) {
+                return nodeConstructors[type];
+            }
+            return null;
+        },
+        
+        clear: function() {
+            nodeConfigCache = null;
+            nodeConfigs = {};
+            nodeList = [];
+            nodeConstructors = {};
+            nodeTypeToId = {};
+        },
+        
+        getTypeId: function(type) {
+            return nodeTypeToId[type];
+        },
+        
+        getModuleInfo: function(type) {
+            return nodeModules[type];
+        },
+        
+        enableNodeSet: function(id) {
+            if (!settings.available()) {
+                throw new Error("Settings unavailable");
+            }
+            var config = nodeConfigs[id];
+            if (config) {
+                delete config.err;
+                config.enabled = true;
+                if (!config.loaded) {
+                    // TODO: honour the promise this returns
+                    loadNodeModule(config);
+                }
+                nodeConfigCache = null;
+                saveNodeList();
+            } else {
+                throw new Error("Unrecognised id: "+id);
+            }
+            return filterNodeInfo(config);
+        },
+        
+        disableNodeSet: function(id) {
+            if (!settings.available()) {
+                throw new Error("Settings unavailable");
+            }
+            var config = nodeConfigs[id];
+            if (config) {
+                // TODO: persist setting
+                config.enabled = false;
+                nodeConfigCache = null;
+                saveNodeList();
+            } else {
+                throw new Error("Unrecognised id: "+id);
+            }
+            return filterNodeInfo(config);
+        },
+        
+        saveNodeList: saveNodeList,
+        
+        cleanNodeList: function() {
+            var removed = false;
+            for (var id in nodeConfigs) {
+                if (nodeConfigs.hasOwnProperty(id)) {
+                    if (nodeConfigs[id].module && !nodeModules[nodeConfigs[id].module]) {
+                        registry.removeNode(id);
+                        removed = true;
+                    }
+                }
+            }
+            if (removed) {
+                saveNodeList();
+            }
+        }
+    }
+})();
+
+
+
+function init(_settings) {
+    Node = require("./Node");
+    settings = _settings;
+    registry.init();
+}
+
+/**
+ * Synchronously walks the directory looking for node files.
+ * Emits 'node-icon-dir' events for an icon dirs found
+ * @param dir the directory to search
+ * @return an array of fully-qualified paths to .js files
+ */
+function getNodeFiles(dir) {
+    var result = [];
+    var files = [];
+    try {
+        files = fs.readdirSync(dir);
+    } catch(err) {
+        return result;
+    }
+    files.sort();
+    files.forEach(function(fn) {
+        var stats = fs.statSync(path.join(dir,fn));
+        if (stats.isFile()) {
+            if (/\.js$/.test(fn)) {
+                var valid = true;
+                if (settings.nodesExcludes) {
+                    for (var i=0;i<settings.nodesExcludes.length;i++) {
+                        if (settings.nodesExcludes[i] == fn) {
+                            valid = false;
+                            break;
+                        }
+                    }
+                }
+                valid = valid && fs.existsSync(path.join(dir,fn.replace(/\.js$/,".html")))
+                
+                if (valid) {
+                    result.push(path.join(dir,fn));
+                }
+            }
+        } else if (stats.isDirectory()) {
+            // Ignore /.dirs/, /lib/ /node_modules/ 
+            if (!/^(\..*|lib|icons|node_modules|test)$/.test(fn)) {
+                result = result.concat(getNodeFiles(path.join(dir,fn)));
+            } else if (fn === "icons") {
+                events.emit("node-icon-dir",path.join(dir,fn));
+            }
+        }
+    });
+    return result;
+}
+
+/**
+ * Scans the node_modules path for nodes
+ * @param moduleName the name of the module to be found
+ * @return a list of node modules: {dir,package}
+ */
+function scanTreeForNodesModules(moduleName) {
+    var dir = __dirname+"/../../nodes";
+    var results = [];
+    var up = path.resolve(path.join(dir,".."));
+    while (up !== dir) {
+        var pm = path.join(dir,"node_modules");
+        try {
+            var files = fs.readdirSync(pm);
+            for (var i=0;i<files.length;i++) {
+                var fn = files[i];
+                if (!registry.getModuleInfo(fn)) {
+                    if (!moduleName || fn == moduleName) {
+                        var pkgfn = path.join(pm,fn,"package.json");
+                        try {
+                            var pkg = require(pkgfn);
+                            if (pkg['node-red']) {
+                                var moduleDir = path.join(pm,fn);
+                                results.push({dir:moduleDir,package:pkg});
+                            }
+                        } catch(err) {
+                            if (err.code != "MODULE_NOT_FOUND") {
+                                // TODO: handle unexpected error
+                            }
+                        }
+                        if (fn == moduleName) {
+                            break;
+                        }
+                    }
+                }
+            }
+        } catch(err) {
+        }
+        
+        dir = up;
+        up = path.resolve(path.join(dir,".."));
+    }
+    return results;
+}
+
+/**
+ * Loads the nodes provided in an npm package.
+ * @param moduleDir the root directory of the package
+ * @param pkg the module's package.json object
+ */
+function loadNodesFromModule(moduleDir,pkg) {
+    var nodes = pkg['node-red'].nodes||{};
+    var results = [];
+    var iconDirs = [];
+    for (var n in nodes) {
+        if (nodes.hasOwnProperty(n)) {
+            var file = path.join(moduleDir,nodes[n]);
+            try {
+                results.push(loadNodeConfig(file,pkg.name,n));
+            } catch(err) {
+            }
+            var iconDir = path.join(moduleDir,path.dirname(nodes[n]),"icons");
+            if (iconDirs.indexOf(iconDir) == -1) {
+                if (fs.existsSync(iconDir)) {
+                    events.emit("node-icon-dir",iconDir);
+                    iconDirs.push(iconDir);
+                }
+            }
+        }
+    }
+    return results;
+}
+
+
+/**
+ * Loads a node's configuration
+ * @param file the fully qualified path of the node's .js file
+ * @param name the name of the node
+ * @return the node object
+ *         {
+ *            id: a unqiue id for the node file
+ *            name: the name of the node file, or label from the npm module
+ *            file: the fully qualified path to the node's .js file
+ *            template: the fully qualified path to the node's .html file
+ *            config: the non-script parts of the node's .html file
+ *            script: the script part of the node's .html file
+ *            types: an array of node type names in this file
+ *         }
+ */
+function loadNodeConfig(file,module,name) {
+    var id = crypto.createHash('sha1').update(file).digest("hex");
+    if (module && name) {
+        var newid = crypto.createHash('sha1').update(module+":"+name).digest("hex");
+        var existingInfo = registry.getNodeInfo(id);
+        if (existingInfo) {
+            // For a brief period, id for modules were calculated incorrectly.
+            // To prevent false-duplicates, this removes the old id entry
+            registry.removeNode(id);
+            registry.saveNodeList();
+        }
+        id = newid;
+
+    }
+    var info = registry.getNodeInfo(id);
+    
+    var isEnabled = true;
+
+    if (info) {
+        if (info.hasOwnProperty("loaded")) {
+            throw new Error(file+" already loaded");
+        }
+        isEnabled = info.enabled;
+    }
+    
+    var node = {
+        id: id,
+        file: file,
+        template: file.replace(/\.js$/,".html"),
+        enabled: isEnabled,
+        loaded:false
+    }
+    
+    if (module) {
+        node.name = module+":"+name;
+        node.module = module;
+    } else {
+        node.name = path.basename(file)
+    }
+    try {
+        var content = fs.readFileSync(node.template,'utf8');
+        
+        var types = [];
+        
+        var regExp = /<script ([^>]*)data-template-name=['"]([^'"]*)['"]/gi;
+        var match = null;
+        
+        while((match = regExp.exec(content)) !== null) {
+            types.push(match[2]);
+        }
+        node.types = types;
+        node.config = content;
+        
+        // TODO: parse out the javascript portion of the template
+        node.script = "";
+        
+        for (var i=0;i<node.types.length;i++) {
+            if (registry.getTypeId(node.types[i])) {
+                node.err = node.types[i]+" already registered";
+                break;
+            }
+        }
+    } catch(err) {
+        node.types = [];
+        if (err.code === 'ENOENT') {
+            node.err = "Error: "+file+" does not exist";
+        } else {
+            node.err = err.toString();
+        }
+    }
+    registry.addNodeSet(id,node);
+    return node;
+}
+
+/**
+ * Loads all palette nodes
+ * @param defaultNodesDir optional parameter, when set, it overrides the default
+ *                        location of nodeFiles - used by the tests
+ * @return a promise that resolves on completion of loading
+ */
+function load(defaultNodesDir,disableNodePathScan) {
+    return when.promise(function(resolve,reject) {
+        // Find all of the nodes to load
+        var nodeFiles;
+        if(defaultNodesDir) {
+            nodeFiles = getNodeFiles(path.resolve(defaultNodesDir));
+        } else {
+            nodeFiles = getNodeFiles(__dirname+"/../../nodes");
+        }
+        
+        if (settings.nodesDir) {
+            var dir = settings.nodesDir;
+            if (typeof settings.nodesDir == "string") {
+                dir = [dir];
+            }
+            for (var i=0;i<dir.length;i++) {
+                nodeFiles = nodeFiles.concat(getNodeFiles(dir[i]));
+            }
+        }
+        var nodes = [];
+        nodeFiles.forEach(function(file) {
+            try {
+                nodes.push(loadNodeConfig(file));
+            } catch(err) {
+                // 
+            }
+        });
+        
+        // TODO: disabling npm module loading if defaultNodesDir set
+        //       This indicates a test is being run - don't want to pick up
+        //       unexpected nodes.
+        //       Urgh.
+        if (!disableNodePathScan) {
+            // Find all of the modules containing nodes
+            var moduleFiles = scanTreeForNodesModules();
+            moduleFiles.forEach(function(moduleFile) {
+                nodes = nodes.concat(loadNodesFromModule(moduleFile.dir,moduleFile.package));
+            });
+        }
+        var promises = [];
+        nodes.forEach(function(node) {
+            if (!node.err) {
+                promises.push(loadNodeModule(node));
+            }
+        });
+        
+        //resolve([]);
+        when.settle(promises).then(function(results) {
+            // Trigger a load of the configs to get it precached
+            registry.getAllNodeConfigs();
+            
+            if (settings.available()) {
+                resolve(registry.saveNodeList());
+            } else {
+                resolve();
+            }
+        }); 
+    });
+}
+
+/**
+ * Loads the specified node into the runtime
+ * @param node a node info object - see loadNodeConfig
+ * @return a promise that resolves to an update node info object. The object
+ *         has the following properties added:
+ *            err: any error encountered whilst loading the node
+ *            
+ */
+function loadNodeModule(node) {
+    var nodeDir = path.dirname(node.file);
+    var nodeFn = path.basename(node.file);
+    if (!node.enabled) {
+        return when.resolve(node);
+    }
+    try {
+        var loadPromise = null;
+        var r = require(node.file);
+        if (typeof r === "function") {
+            var promise = r(require('../red'));
+            if (promise != null && typeof promise.then === "function") {
+                loadPromise = promise.then(function() {
+                    node.enabled = true;
+                    node.loaded = true;
+                    return node;
+                }).otherwise(function(err) {
+                    node.err = err;
+                    return node;
+                });
+            }
+        }
+        if (loadPromise == null) {
+            node.enabled = true;
+            node.loaded = true;
+            loadPromise = when.resolve(node);
+        }
+        return loadPromise;
+    } catch(err) {
+        node.err = err;
+        return when.resolve(node);
+    }
+}
+
+function loadNodeList(nodes) {
+    var promises = [];
+    nodes.forEach(function(node) {
+        if (!node.err) {
+            promises.push(loadNodeModule(node));
+        } else {
+            promises.push(node);
+        }
+    });
+    
+    return when.settle(promises).then(function(results) {
+        return registry.saveNodeList().then(function() {
+            var list = results.map(function(r) {
+                return filterNodeInfo(r.value);
+            });
+            return list;
+        });
+    });
+}
+
+function addNode(file) {
+    if (!settings.available()) {
+        throw new Error("Settings unavailable");
+    }
+    var nodes = [];
+    try { 
+        nodes.push(loadNodeConfig(file));
+    } catch(err) {
+        return when.reject(err);
+    }
+    return loadNodeList(nodes);
+}
+
+function addModule(module) {
+    if (!settings.available()) {
+        throw new Error("Settings unavailable");
+    }
+    var nodes = [];
+    if (registry.getModuleInfo(module)) {
+        return when.reject(new Error("Module already loaded"));
+    }
+    var moduleFiles = scanTreeForNodesModules(module);
+    if (moduleFiles.length === 0) {
+        var err = new Error("Cannot find module '" + module + "'");
+        err.code = 'MODULE_NOT_FOUND';
+        return when.reject(err);
+    }
+    moduleFiles.forEach(function(moduleFile) {
+        nodes = nodes.concat(loadNodesFromModule(moduleFile.dir,moduleFile.package));
+    });
+    return loadNodeList(nodes);
+}
+
+module.exports = {
+    init:init,
+    load:load,
+    clear: registry.clear,
+    registerType: registry.registerNodeConstructor,
+    get: registry.getNodeConstructor,
+    getNodeInfo: registry.getNodeInfo,
+    getNodeModuleInfo: registry.getModuleInfo,
+    getNodeList: registry.getNodeList,
+    getNodeConfigs: registry.getAllNodeConfigs,
+    getNodeConfig: registry.getNodeConfig,
+    addNode: addNode,
+    removeNode: registry.removeNode,
+    enableNode: registry.enableNodeSet,
+    disableNode: registry.disableNodeSet,
+    
+    addModule: addModule,
+    removeModule: registry.removeModule,
+    cleanNodeList: registry.cleanNodeList
+}