Address Manage Dictionary issues

A variety of bugs are addressed. See JIRA.

Issue-ID: CLAMP-849
Change-Id: I620c1ec774fdcec484f40fc638551960a98973c8
Signed-off-by: Ted Humphrey <Thomas.Humphrey@att.com>

2 files changed, 472 insertions, 0 deletions

diff --git a/ui-react/src/utils/CsvToJson.js b/ui-react/src/utils/CsvToJson.js
new file mode 100644
index 000000000..5ec19c9e2
--- /dev/null
+++ b/ui-react/src/utils/CsvToJson.js
@@ -0,0 +1,204 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP CLAMP
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END============================================
+ * ===================================================================
+ *
+ */
+
+export default function CsvToJson(rawCsvData, delimiter, internalDelimiter, csvHeaderNames, jsonKeyNames, mandatory) {
+
+	let printDictKeys = '';
+	let result = { jsonObjArray: [], errorMessages: '' };
+
+	// Validate that all parallel arrays passed in have same number of elements;
+	// this would be a developer error.
+
+	let checkLength = csvHeaderNames.length;
+
+	if (checkLength !== jsonKeyNames.length || checkLength !== mandatory.length) {
+		result.errorMessages = 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays parameters are not the same length';
+		return result;
+	}
+
+	if (checkLength < 1) {
+		result.errorMessages = 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays have no entries';
+		return result;
+	}
+
+	// Make a nice string to print in the error case to tell user what is the
+	//  required heaer row format
+
+	for (let i=0; i < csvHeaderNames.length; ++i) {
+		if (i === 0) {
+			printDictKeys = csvHeaderNames[i];
+		} else {
+			printDictKeys += ',' +  csvHeaderNames[i];
+		}
+	}
+
+	let dictElems = rawCsvData.split('\n');
+	let numColumns = 0;
+	let filteredDictElems = [];
+
+	// The task of the following loop is to convert raw CSV rows into easily parseable
+	// and streamlined versions of the rows with an internalDelimiter replacing the standard
+	// comma; it is presumed (and checked) that the internalDelimiter cannot exist as a valid
+	// sequence of characters in the user's data.
+
+	// This conversion process also strips leading and trailing whitespace from each row,
+	// discards empty rows, correctly interprets and removes all double quotes that programs like
+	// Excel use to support user columns that contain special characters, most notably, the comma
+	// delimiter. A double-quote that is contained within a double-quoted column value 
+	// must appear in this raw data as a sequence of two double quotes. Furthermore, any column
+	// value in the raw CSV data that does not contain a delimiter may or may not be enclosed in
+	// double quotes. It is the Excel convention to not use double qoutes unless necessary, and
+	// there is no reasonable way to tell Excel to surround every column value with double quotes. 
+	// Any files that were directly "exported" by CLAMP itself from the Managing Dictionaries
+	// capability, surround all columns with double quotes.
+
+	for (let i = 0; i < dictElems.length; i++) {
+
+		let oneRow = dictElems[i].trim();
+		let j = 0;
+		let inQuote = false
+		let nextChar = undefined;
+		let prevChar = null;
+
+		
+		if (oneRow === '') {
+			continue; // Skip blank rows
+		} else if (oneRow.indexOf(internalDelimiter) !== -1) {
+			result.errorMessages += '\nRow #' + i + ' contains illegal sequence of characters (' + internalDelimiter + ')';
+			break;
+		} else {
+			nextChar = oneRow[1];
+		}
+
+		let newStr = '';
+		numColumns = 1;
+
+		// This "while loop" performs the very meticulous task of removing double quotes that
+		// are used by Excel to encase special characters as user string value data,
+		// and manages to correctly identify columns that are defined with or without
+		// double quotes and to process the comma delimiter correctly when encountered
+		// as a user value within a column. Such a column would have to be encased in
+		// double quotes; a comma found outside double quotes IS a delimiter.
+
+		while (j < oneRow.length) {
+		 	if (oneRow[j] === '"') {
+		 		if (inQuote === false) {
+					if (prevChar !== delimiter && prevChar !== null) {
+						result.errorMessages += '\nMismatched double quotes or illegal whitespace around delimiter at row #' + (i + 1) + ' near column #' + numColumns;
+						break;
+					} else {
+						inQuote = true;
+					}
+				} else {
+					if (nextChar === '"') {
+						newStr += '"';
+						++j;
+					} else if ((nextChar !== delimiter) && (nextChar !== undefined)) {
+						result.errorMessages += '\nRow #' + (i + 1) + ' is badly formatted at column #' + numColumns + '. Perhaps an unescaped double quote.';
+						break;
+					} else if (nextChar === delimiter) {
+						++numColumns;
+						inQuote = false;
+						newStr += internalDelimiter;
+						prevChar = delimiter;
+						j += 2;
+						nextChar = oneRow[j+1];
+						continue;
+					} else {
+						++numColumns;
+						inQuote = false;
+						break;
+					}
+				}
+			} else {
+				if (oneRow[j] === delimiter && inQuote === false) {
+					newStr += internalDelimiter;
+					++numColumns;
+				} else {
+					newStr += oneRow[j];
+				}
+			}
+			prevChar = oneRow[j];
+			++j;
+			nextChar = oneRow[j+1]; // can result in undefined at the end
+		}
+
+		if (result.errorMessages === '' && inQuote !== false) {
+			result.errorMessages += '\nMismatched double quotes at row #' + (i + 1);
+			break;
+		} else if (result.errorMessages === '' && numColumns < jsonKeyNames.length) {
+			result.errorMessages += '\nNot enough columns (' + jsonKeyNames.length + ') at row #' + (i + 1);
+			break;
+		}
+
+		filteredDictElems.push(newStr);
+	}
+
+	if (result.errorMessages !== '') {
+		return result;
+	}
+
+	// Perform further checks on data that is now in JSON form
+	if (filteredDictElems.length < 2) {
+		result.errorMessages += '\nNot enough row data found in import file. Need at least a header row and one row of data';
+		return result;
+	}
+
+	// Now that we have something reliably parsed into sanitized columns lets run some checks
+	// and convert it all into an array of JSON objects to push to the back end if all the
+	// checks pass.
+
+	let headers = filteredDictElems[0].split(internalDelimiter);
+
+	// check that headers are included in proper order
+	for (let i=0; i < jsonKeyNames.length; ++i) {
+		if (csvHeaderNames[i] !== headers[i]) {
+			result.errorMessages += 'Row 1 header key at column #' + (i + 1) + ' is a mismatch. Expected row header must contain at least:\n' + printDictKeys; 
+			return result;
+		}
+	}
+
+	// Convert the ASCII rows of data into an array of JSON obects that omit the header
+	// row which is not sent to the back end.
+
+	for (let i = 1; i < filteredDictElems.length; i++) {
+		let data = filteredDictElems[i].split(internalDelimiter);
+		let obj = {};
+		for (let j = 0; j < data.length && j < jsonKeyNames.length; j++) {
+			let value = data[j].trim();
+			if (mandatory[j] === true && value === '') {
+				result.errorMessages += '\n' + csvHeaderNames[j] + ' at row #' + (i+1) + ' is empty but requires a value.';
+			}
+			obj[jsonKeyNames[j]] = value;
+		}
+		result.jsonObjArray.push(obj);
+	}
+
+	if (result.errorMessages !== '') {
+		// If we have errors, return empty parse result even though some things
+		// may have parsed properly. We do not want to encourage the caller
+		// to think the data is good for use.
+		result.jsonObjArray = [];
+	}
+
+	return result;
+}
diff --git a/ui-react/src/utils/CsvToJson.test.js b/ui-react/src/utils/CsvToJson.test.js
new file mode 100644
index 000000000..88fa7a472
--- /dev/null
+++ b/ui-react/src/utils/CsvToJson.test.js
@@ -0,0 +1,268 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP CLAMP
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ *                             reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END============================================
+ * ===================================================================
+ *
+ */
+
+import CsvToJson from './CsvToJson'
+
+describe('Verify CsvToJson', () => {
+
+	const hdrNames= [ 
+		"Element Short Name",
+		"Element Name",
+		"Element Description",
+		"Element Type",
+		"Sub-Dictionary"
+	];
+
+	const jsonKeyNames = [
+		"shortName",
+		"name",
+		"description",
+		"type",
+		"subDictionary"
+	];
+
+	const mandatory = [ true, true, true, true, false ];
+
+	it('Test CsvToJson No Error Case, Quoted Columns', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Type of Alert","string","","admin","2020-06-11T13:56:14.927437Z"';
+				
+		let expectedResult = {
+			 errorMessages: '',
+			 jsonObjArray: [
+				{
+					description: "Type of Alert",
+					name: "Alert Type",
+					shortName: "alertType",
+					subDictionary: "",
+					type: "string"
+				}
+			]
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson No Error Case, Unquoted Columns', () => {
+
+		let rawCsv = 'Element Short Name,Element Name,Element Description,Element Type,Sub-Dictionary\n';
+		rawCsv += 'alertType,Alert Type,Type of Alert,string,,admin,2020-06-11T13:56:14.927437Z';
+				
+		let expectedResult = {
+			 errorMessages: '',
+			 jsonObjArray: [
+				{
+					description: "Type of Alert",
+					name: "Alert Type",
+					shortName: "alertType",
+					subDictionary: "",
+					type: "string"
+				}
+			]
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Properly Escaped Double Quote and Delimiter', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert ""Type""","Type of Alert, Varies","string","","admin","2020-06-11T13:56:14.927437Z"';
+		
+		let errorMessage = '';
+				
+		let expectedResult = {
+			errorMessages: errorMessage,
+			jsonObjArray: [
+				{
+					description: "Type of Alert, Varies",
+					name: 'Alert "Type"',
+					shortName: 'alertType',
+					subDictionary: "",
+					type: "string",
+				}
+
+			]
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+
+	it('Test CsvToJson Error Header Mismatch Error Case', () => {
+
+		let rawCsv = '"Element Short Names","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Type of Alert","string","","admin","2020-06-11T13:56:14.927437Z"';
+		
+		let errorMessage = 'Row 1 header key at column #1 is a mismatch. Expected row header must contain at least:\n';
+		errorMessage += 'Element Short Name,Element Name,Element Description,Element Type,Sub-Dictionary';
+				
+		let expectedResult = {
+			 errorMessages: errorMessage,
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Mismatched Double Quotes in Column', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alert"Type","Alert Type","Type of Alert","string","","admin","2020-06-11T13:56:14.927437Z"';
+		
+		let errorMessage = '\nRow #2 is badly formatted at column #1. Perhaps an unescaped double quote.'
+				
+		let expectedResult = {
+			 errorMessages: errorMessage,
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Illegal Whitespace', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += 'alertType ,  "Alert Type","Type of Alert","string","","admin","2020-06-11T13:56:14.927437Z"';
+		
+		let errorMessage = '\nMismatched double quotes or illegal whitespace around delimiter at row #2 near column #2';
+				
+		let expectedResult = {
+			 errorMessages: errorMessage,
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Too Few Data Columns', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Type of Alert"';
+		
+		let errorMessage = '\nNot enough columns (5) at row #2';
+				
+		let expectedResult = {
+			errorMessages: errorMessage,
+			jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Wrong Header Column Order', () => {
+
+		let rawCsv = '"Element Name","Element Short Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Type of Alert","string","","admin","2020-06-11T13:56:14.927437Z"';
+		
+		let errorMessage = 'Row 1 header key at column #1 is a mismatch. Expected row header must contain at least:\n';
+		errorMessage += 'Element Short Name,Element Name,Element Description,Element Type,Sub-Dictionary';
+				
+		let expectedResult = {
+			errorMessages: errorMessage,
+			jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Not Enough Rows', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		
+		let errorMessage = '\nNot enough row data found in import file. Need at least a header row and one row of data';
+				
+		let expectedResult = {
+			errorMessages: errorMessage,
+			jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Mandatory Field Is Empty', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"","Alert Type","Type of Alert","string","","admin","2020-06-11T13:56:14.927437Z"';
+				
+		let expectedResult = {
+			 errorMessages: '\nElement Short Name at row #2 is empty but requires a value.',
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '|', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Mismatched Double Quotes At End', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Alert Type Description","string","admin","2020-06-11T13:56:14.927437Z';
+				
+		let expectedResult = {
+			 errorMessages: '\nMismatched double quotes at row #2',
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '||', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Mismatched Mandatory Array Parameters', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Alert Type Description","string","admin","2020-06-11T13:56:14.927437Z';
+				
+		let expectedResult = {
+			 errorMessages: 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays parameters are not the same length',
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '||', hdrNames, jsonKeyNames, [ true ])).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Empty Mandatory Array Parameters', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Alert Type Description","string","admin","2020-06-11T13:56:14.927437Z';
+				
+		let expectedResult = {
+			 errorMessages: 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays have no entries',
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '||', [], [], [])).toEqual(expectedResult);
+	});
+
+	it('Test CsvToJson Error Illegal Data Contains Internal Delimiter', () => {
+
+		let rawCsv = '"Element Short Name","Element Name","Element Description","Element Type","Sub-Dictionary"\n';
+		rawCsv += '"alertType","Alert Type","Alert Type||Description","string","admin","2020-06-11T13:56:14.927437Z';
+
+		let expectedResult = {
+			 errorMessages: '\nRow #1 contains illegal sequence of characters (||)',
+			 jsonObjArray: []
+		};
+
+		expect(CsvToJson(rawCsv, ',', '||', hdrNames, jsonKeyNames, mandatory)).toEqual(expectedResult);
+	});
+})