diff options
Diffstat (limited to 'dgbuilder/core_nodes/parsers')
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-CSV.html | 123 | ||||
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-CSV.js | 157 | ||||
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-HTML.html | 73 | ||||
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-HTML.js | 60 | ||||
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-JSON.html | 47 | ||||
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-JSON.js | 46 | ||||
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-XML.html | 48 | ||||
| -rw-r--r-- | dgbuilder/core_nodes/parsers/70-XML.js | 46 |
8 files changed, 600 insertions, 0 deletions
diff --git a/dgbuilder/core_nodes/parsers/70-CSV.html b/dgbuilder/core_nodes/parsers/70-CSV.html new file mode 100644 index 00000000..5632246f --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-CSV.html @@ -0,0 +1,123 @@ + +<!-- + Copyright 2014 IBM Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<script type="text/x-red" data-template-name="csv"> + <div class="form-row"> + <label for="node-input-temp"><i class="fa fa-list"></i> Columns</label> + <input type="text" id="node-input-temp" placeholder="comma-separated column names"> + </div> + <div class="form-row"> + <label for="node-input-select-sep"><i class="fa fa-text-width"></i> Separator</label> + <select style="width: 150px" id="node-input-select-sep"> + <option value=",">comma</option> + <option value="\t">tab</option> + <option value=" ">space</option> + <option value=";">semicolon</option> + <option value=":">colon</option> + <option value="">other...</option> + </select> + <input style="width: 40px;" type="text" id="node-input-sep" pattern="."> + </div> + + <div class="form-row"> + <label for="node-input-name"><i class="fa fa-tag"></i> Name</label> + <input type="text" id="node-input-name" placeholder="Name"> + </div> + <hr align="middle"/> + <div class="form-row"> + <label style="width: 100%;"><i class="fa fa-gears"></i> CSV-to-Object options</label> + <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Input</label> + <input style="width: 30px" type="checkbox" id="node-input-hdrin"><label style="width: auto;" for="node-input-hdrin">first row contains column names</span> + </div> + <div class="form-row"> + <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-out"></i> Output</label> + <select type="text" id="node-input-multi" style="width: 250px;"> + <option value="one">a message per row</option> + <option value="mult">a single message [array]</option> + </select> + </div> + <hr align="middle"/> + <div class="form-row"> + <label style="width: 100%;"><i class="fa fa-gears"></i> Object-to-CSV options</label> + <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-sign-in"></i> Output</label> + <input style="width: 30px" type="checkbox" id="node-input-hdrout"><label style="width: auto;" for="node-input-hdrout">include column name row</span> + </div> + <div class="form-row"> + <label style="margin-left: 10px; margin-right: -10px;"><i class="fa fa-align-left"></i> Newline</label> + <select style="width: 150px" id="node-input-ret"> + <option value='\n'>Linux (\n)</option> + <option value='\r'>Mac (\r)</option> + <option value='\r\n'>Windows (\r\n)</option> + </select> + </div> +</script> + +<script type="text/x-red" data-help-name="csv"> + <p>A function that parses the <b>msg.payload</b> to convert csv to/from a javascript object. + Places the result in the payload.</p> + <p>If the input is a string it tries to parse it as CSV and creates a javascript object.</p> + <p>If the input is a javascript object it tries to build a CSV string.</p> + <p>The columns template should contain an ordered list of column headers. For csv input these become the property names. + For csv output these specify the properties to extract from the object and the order for the csv.</p> + <p><b>Note:</b> the columns should always be specified comma separated - even if another separator is chosen for the data.</p> + </script> + +<script type="text/javascript"> + RED.nodes.registerType('csv',{ + category: 'function', + color:"#DEBD5C", + defaults: { + name: {value:""}, + sep: {value:',',required:true,validate:RED.validators.regex(/^.{1,2}$/)}, + //quo: {value:'"',required:true}, + hdrin: {value:""}, + hdrout: {value:""}, + multi: {value:"one",required:true}, + ret: {value:'\\n'}, + temp: {value:""} + }, + inputs:1, + outputs:1, + icon: "arrow-in.png", + label: function() { + return this.name||"csv"; + }, + labelStyle: function() { + return this.name?"node_label_italic":""; + }, + oneditprepare: function() { + if (this.sep == "," || this.sep == "\\t" || this.sep == ";" || this.sep == ":" || this.sep == " ") { + $("#node-input-select-sep").val(this.sep); + $("#node-input-sep").hide(); + } else { + $("#node-input-select-sep").val(""); + $("#node-input-sep").val(this.sep); + $("#node-input-sep").show(); + } + $("#node-input-select-sep").change(function() { + var v = $("#node-input-select-sep option:selected").val(); + $("#node-input-sep").val(v); + if (v == "") { + $("#node-input-sep").val(""); + $("#node-input-sep").show().focus(); + } else { + $("#node-input-sep").hide(); + } + }); + } + }); +</script> diff --git a/dgbuilder/core_nodes/parsers/70-CSV.js b/dgbuilder/core_nodes/parsers/70-CSV.js new file mode 100644 index 00000000..56aa7c72 --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-CSV.js @@ -0,0 +1,157 @@ +/** + * Copyright 2014 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +module.exports = function(RED) { + "use strict"; + function CSVNode(n) { + RED.nodes.createNode(this,n); + this.template = n.temp.split(","); + this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r"); + this.quo = '"'; + this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r"); + this.winflag = (this.ret === "\r\n"); + this.lineend = "\n"; + this.multi = n.multi || "one"; + this.hdrin = n.hdrin || false; + this.hdrout = n.hdrout || false; + this.goodtmpl = true; + var node = this; + + // pass in an array of column names to be trimed, de-quoted and retrimed + var clean = function(col) { + for (var t = 0; t < col.length; t++) { + col[t] = col[t].trim(); // remove leading and trailing whitespace + if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') { + // remove leading and trailing quotes (if they exist) - and remove whitepace again. + col[t] = col[t].substr(1,col[t].length -2).trim(); + } + } + if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; } + else { node.goodtmpl = true; } + return col; + } + node.template = clean(node.template); + + this.on("input", function(msg) { + if (msg.hasOwnProperty("payload")) { + if (typeof msg.payload == "object") { // convert object to CSV string + try { + var ou = ""; + if (node.hdrout) { + ou += node.template.join(node.sep) + node.ret; + } + if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; } + for (var s = 0; s < msg.payload.length; s++) { + for (var t=0; t < node.template.length; t++) { + + // aaargh - resorting to eval here - but fairly contained front and back. + var p = RED.util.ensureString(eval("msg.payload[s]."+node.template[t])); + + if (p === "undefined") { p = ""; } + if (p.indexOf(node.sep) != -1) { // add quotes if any "commas" + ou += node.quo + p + node.quo + node.sep; + } + else if (p.indexOf(node.quo) != -1) { // add double quotes if any quotes + p = p.replace(/"/g, '""'); + ou += node.quo + p + node.quo + node.sep; + } + else { ou += p + node.sep; } // otherwise just add + } + ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline" + } + node.send({payload:ou}); + } + catch(e) { node.log(e); } + } + else if (typeof msg.payload == "string") { // convert CSV string to object + try { + var f = true; // flag to indicate if inside or outside a pair of quotes true = outside. + var j = 0; // pointer into array of template items + var k = [""]; // array of data for each of the template items + var o = {}; // output object to build up + var a = []; // output array is needed for multiline option + var first = true; // is this the first line + var tmp = ""; + + // For now we are just going to assume that any \r or \n means an end of line... + // got to be a weird csv that has singleton \r \n in it for another reason... + + // Now process the whole file/line + for (var i = 0; i < msg.payload.length; i++) { + if ((node.hdrin === true) && first) { // if the template is in the first line + if ((msg.payload[i] === "\n")||(msg.payload[i] === "\r")) { // look for first line break + node.template = clean(tmp.split(node.sep)); + first = false; + } + else { tmp += msg.payload[i]; } + } + else { + if (msg.payload[i] === node.quo) { // if it's a quote toggle inside or outside + f = !f; + if (msg.payload[i-1] === node.quo) { k[j] += '\"'; } // if it's a quotequote then it's actually a quote + } + else if ((msg.payload[i] === node.sep) && f) { // if we are outside of quote (ie valid separator + if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } + if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) { + if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); } + o[node.template[j]] = k[j]; + } + j += 1; + k[j] = ""; + } + else if (f && ((msg.payload[i] === "\n") || (msg.payload[i] === "\r"))) { // handle multiple lines + //console.log(j,k,o,k[j]); + if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) { + if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); } + else { k[j].replace(/\r$/,''); } + o[node.template[j]] = k[j]; + } + if (JSON.stringify(o) !== "{}") { // don't send empty objects + if (node.multi === "one") { node.send({payload:o}); } // either send + else { a.push(o); } // or add to the array + } + j = 0; + k = [""]; + o = {}; + } + else { // just add to the part of the message + k[j] += msg.payload[i]; + } + } + } + // Finished so finalize and send anything left + //console.log(j,k,o,k[j]); + if (!node.goodtmpl) { node.template[j] = "col"+(j+1); } + if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) { + if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); } + else { k[j].replace(/\r$/,''); } + o[node.template[j]] = k[j]; + } + msg.payload = o; + if (JSON.stringify(o) !== "{}") { // don't send empty objects + if (node.multi === "one") { node.send({payload:o}); } // either send + else { a.push(o); } // or add to the aray + } + if (node.multi !== "one") { node.send({payload:a}); } // finally send the array + } + catch(e) { node.log(e); } + } + else { node.log("This node only handles csv strings or js objects."); } + } + }); + } + RED.nodes.registerType("csv",CSVNode); +} diff --git a/dgbuilder/core_nodes/parsers/70-HTML.html b/dgbuilder/core_nodes/parsers/70-HTML.html new file mode 100644 index 00000000..2b73b49c --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-HTML.html @@ -0,0 +1,73 @@ +<!-- + Copyright 2014 IBM Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<script type="text/x-red" data-template-name="html"> + <div class="form-row"> + <label for="node-input-tag"><i class="fa fa-filter"></i> Select</label> + <input type="text" id="node-input-tag" placeholder="h1"> + </div> + <div class="form-row"> + <label for="node-input-ret"><i class="fa fa-sign-out"></i> Output</label> + <select id="node-input-ret" style="width:73% !important"> + <option value="html">the html content of the elements</option> + <option value="text">only the text content of the elements</option> + <!-- <option value="attr">an object of any attributes</option> --> + <!-- <option value="val">return the value from a form element</option> --> + </select> + </div> + <div class="form-row"> + <label for="node-input-as"> </label> + <select id="node-input-as" style="width:73% !important"> + <option value="single">as a single message containing an array</option> + <option value="multi">as multiple messages, one for each element</option> + </select> + </div> + <br/> + <div class="form-row"> + <label for="node-input-name"><i class="fa fa-tag"></i> Name</label> + <input type="text" id="node-input-name" placeholder="Name"> + </div> + <div class="form-tips">Tip: The <b>Select</b> value is a <a href="http://api.jquery.com/category/selectors/" target="_new"><i><u>jQuery</u></i></a> style selector.</div> +</script> + +<script type="text/x-red" data-help-name="html"> + <p>Extracts elements from an html document held in <b>msg.payload</b> using a selector.</p> + <p>The selector uses the <a href="http://api.jquery.com/category/selectors/" target="_new">jQuery syntax</a>.</p> + <p>The result is either a single message with a payload containing an array of the matched elements, or multiple + messages that each contain a matched element.</p> +</script> + +<script type="text/javascript"> + RED.nodes.registerType('html',{ + category: 'function', + color:"#DEBD5C", + defaults: { + tag: {value:""}, + ret: {value:"html"}, + as: {value:"single"}, + name: {value:""} + }, + inputs:1, + outputs:1, + icon: "jq.png", + label: function() { + return this.name||this.tag||"html"; + }, + labelStyle: function() { + return this.name?"node_label_italic":""; + } + }); +</script> diff --git a/dgbuilder/core_nodes/parsers/70-HTML.js b/dgbuilder/core_nodes/parsers/70-HTML.js new file mode 100644 index 00000000..7a9450f3 --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-HTML.js @@ -0,0 +1,60 @@ +/** + * Copyright 2014 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +module.exports = function(RED) { + "use strict"; + var cheerio = require('cheerio'); + + function CheerioNode(n) { + RED.nodes.createNode(this,n); + this.tag = n.tag || "h1"; + this.ret = n.ret || "html"; + this.as = n.as || "single"; + var node = this; + this.on("input", function(msg) { + try { + var $ = cheerio.load(msg.payload); + var pay = []; + $(node.tag).each(function() { + if (node.as === "multi") { + var pay2 = null; + if (node.ret === "html") { pay2 = $(this).html(); } + if (node.ret === "text") { pay2 = $(this).text(); } + //if (node.ret === "attr") { pay2 = $(this)[0]["attribs"]; } + //if (node.ret === "val") { pay2 = $(this).val(); } + if (pay2) { + msg.payload = pay2; + node.send(msg); + } + } + if (node.as === "single") { + if (node.ret === "html") { pay.push( $(this).html() ); } + if (node.ret === "text") { pay.push( $(this).text() ); } + //if (node.ret === "attr") { pay.push( $(this)[0]["attribs"] ); } + //if (node.ret === "val") { pay.push( $(this).val() ); } + } + }); + if ((node.as === "single") && (pay.length !== 0)) { + msg.payload = pay; + node.send(msg); + } + } catch (error) { + node.log('Error: '+error.message); + } + }); + } + RED.nodes.registerType("html",CheerioNode); +} diff --git a/dgbuilder/core_nodes/parsers/70-JSON.html b/dgbuilder/core_nodes/parsers/70-JSON.html new file mode 100644 index 00000000..864974ab --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-JSON.html @@ -0,0 +1,47 @@ +<!-- + Copyright 2014 IBM Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<script type="text/x-red" data-template-name="json"> + <div class="form-row"> + <label for="node-input-name"><i class="fa fa-tag"></i> Name</label> + <input type="text" id="node-input-name" placeholder="Name"> + </div> +</script> + +<script type="text/x-red" data-help-name="json"> + <p>A function that parses the <b>msg.payload</b> to convert a JSON string to/from a javascript object. Places the result back into the payload.</p> + <p>If the input is a JSON string it tries to parse it to a javascript object.</p> + <p>If the input is a javascript object it creates a JSON string.</p> +</script> + +<script type="text/javascript"> + RED.nodes.registerType('json',{ + category: 'function', + color:"#DEBD5C", + defaults: { + name: {value:""} + }, + inputs:1, + outputs:1, + icon: "arrow-in.png", + label: function() { + return this.name||"json"; + }, + labelStyle: function() { + return this.name?"node_label_italic":""; + } + }); +</script> diff --git a/dgbuilder/core_nodes/parsers/70-JSON.js b/dgbuilder/core_nodes/parsers/70-JSON.js new file mode 100644 index 00000000..e216bd4f --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-JSON.js @@ -0,0 +1,46 @@ +/** + * Copyright 2014 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +module.exports = function(RED) { + "use strict"; + var util = require("util"); + + function JSONNode(n) { + RED.nodes.createNode(this,n); + var node = this; + this.on("input", function(msg) { + if (msg.hasOwnProperty("payload")) { + if (typeof msg.payload === "string") { + try { + msg.payload = JSON.parse(msg.payload); + node.send(msg); + } + catch(e) { node.log(e+ "\n"+msg.payload); } + } + else if (typeof msg.payload === "object") { + if (!Buffer.isBuffer(msg.payload) ) { + if (!util.isArray(msg.payload)) { + msg.payload = JSON.stringify(msg.payload); + node.send(msg); + } + } + } + else { node.log("dropped: "+msg.payload); } + } + }); + } + RED.nodes.registerType("json",JSONNode); +} diff --git a/dgbuilder/core_nodes/parsers/70-XML.html b/dgbuilder/core_nodes/parsers/70-XML.html new file mode 100644 index 00000000..8b0a2843 --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-XML.html @@ -0,0 +1,48 @@ +<!-- + Copyright 2014 IBM Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<script type="text/x-red" data-template-name="xml"> + <div class="form-row"> + <label for="node-input-name"><i class="fa fa-tag"></i> Name</label> + <input type="text" id="node-input-name" placeholder="Name"> + </div> +</script> + +<script type="text/x-red" data-help-name="xml"> + <p>A function that parses the <b>msg.payload</b> to convert xml to/from a javascript object. Places the result in the payload.</p> + <p>If the input is a string it tries to parse it as XML and creates a javascript object.</p> + <p>If the input is a javascript object it tries to build an XML string.</p> + <p>See <a href="https://github.com/Leonidas-from-XIV/node-xml2js/blob/master/README.md" target="_new">the xml2js docs <i>here</i></a> for more information.</p> +</script> + +<script type="text/javascript"> + RED.nodes.registerType('xml',{ + category: 'function', + color:"#DEBD5C", + defaults: { + name: {value:""} + }, + inputs:1, + outputs:1, + icon: "arrow-in.png", + label: function() { + return this.name||"xml"; + }, + labelStyle: function() { + return this.name?"node_label_italic":""; + } + }); +</script> diff --git a/dgbuilder/core_nodes/parsers/70-XML.js b/dgbuilder/core_nodes/parsers/70-XML.js new file mode 100644 index 00000000..931de7f5 --- /dev/null +++ b/dgbuilder/core_nodes/parsers/70-XML.js @@ -0,0 +1,46 @@ +/** + * Copyright 2014 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +module.exports = function(RED) { + "use strict"; + var xml2js = require('xml2js'); + var parseString = xml2js.parseString; + var builder = new xml2js.Builder({renderOpts:{pretty:false}}); + + function XMLNode(n) { + RED.nodes.createNode(this,n); + var node = this; + this.on("input", function(msg) { + if (msg.hasOwnProperty("payload")) { + if (typeof msg.payload == "object") { + msg.payload = builder.buildObject(msg.payload); + node.send(msg); + } + else if (typeof msg.payload == "string") { + parseString(msg.payload, {strict:true,async:true}, function (err, result) { + if (err) { node.error(err); } + else { + msg.payload = result; + node.send(msg); + } + }); + } + else { node.log("This node only handles xml strings or js objects."); } + } + }); + } + RED.nodes.registerType("xml",XMLNode); +} |