aboutsummaryrefslogtreecommitdiffstats
path: root/gui-clamp/ui-react/src/utils/CsvToJson.js
blob: 6d95b31194ef70687ac5add17d42fb294878dfc6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

@media only all and (prefers-color-scheme: dark) {
.highlight .hll { background-color: #49483e }
.highlight .c { color: #75715e } /* Comment */
.highlight .err { color: #960050; background-color: #1e0010 } /* Error */
.highlight .k { color: #66d9ef } /* Keyword */
.highlight .l { color: #ae81ff } /* Literal */
.highlight .n { color: #f8f8f2 } /* Name */
.highlight .o { color: #f92672 } /* Operator */
.highlight .p { color: #f8f8f2 } /* Punctuation */
.highlight .ch { color: #75715e } /* Comment.Hashbang */
.highlight .cm { color: #75715e } /* Comment.Multiline */
.highlight .cp { color: #75715e } /* Comment.Preproc */
.highlight .cpf { color: #75715e } /* Comment.PreprocFile */
.highlight .c1 { color: #75715e } /* Comment.Single */
.highlight .cs { color: #75715e } /* Comment.Special */
.highlight .gd { color: #f92672 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gi { color: #a6e22e } /* Generic.Inserted */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #75715e } /* Generic.Subheading */
.highlight .kc { color: #66d9ef } /* Keyword.Constant */
.highlight .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight .kn { color: #f92672 } /* Keyword.Namespace */
.highlight .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight .kt { color: #66d9ef } /* Keyword.Type */
.highlight .ld { color: #e6db74 } /* Literal.Date */
.highlight .m { color: #ae81ff } /* Literal.Number */
.highlight .s { color: #e6db74 } /* Literal.String */
.highlight .na { color: #a6e22e } /* Name.Attribute */
.highlight .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight .nc { color: #a6e22e } /* Name.Class */
.highlight .no { color: #66d9ef } /* Name.Constant */
.highlight .nd { color: #a6e22e } /* Name.Decorator */
.highlight .ni { color: #f8f8f2 } /* Name.Entity */
.highlight .ne { color: #a6e22e } /* Name.Exception */
.highlight .nf { color: #a6e22e } /* Name.Function */
.highlight .nl { color: #f8f8f2 } /* Name.Label */
.highlight .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight .nx { color: #a6e22e } /* Name.Other */
.highlight .py { color: #f8f8f2 } /* Name.Property */
.highlight .nt { color: #f92672 } /* Name.Tag */
.highlight .nv { color: #f8f8f2 } /* Name.Variable */
.highlight .ow { color: #f92672 } /* Operator.Word */
.highlight .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight .sc { color: #e6db74 } /* Literal.String.Char */
.highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight .se { color: #ae81ff } /* Literal.String.Escape */
.highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight .sx { color: #e6db74 } /* Literal.String.Other */
.highlight .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */
}
@media (prefers-color-scheme: light) {
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; b
/*-
 * ============LICENSE_START=======================================================
 * ONAP CLAMP
 * ================================================================================
 * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
 * ================================================================================
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * ============LICENSE_END============================================
 * ===================================================================
 *
 */

export default function CsvToJson(rawCsvData, delimiter, internalDelimiter, csvHeaderNames, jsonKeyNames, mandatory) {

  let printDictKeys = '';
  let result = { jsonObjArray: [], errorMessages: '' };

  // Validate that all parallel arrays passed in have same number of elements;
  // this would be a developer error.

  let checkLength = csvHeaderNames.length;

  if (checkLength !== jsonKeyNames.length || checkLength !== mandatory.length) {
    result.errorMessages = 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays parameters are not the same length';
    return result;
  }

  if (checkLength < 1) {
    result.errorMessages = 'interanl error: csvHeaderNames, jsonKeyNames, and mandatory arrays have no entries';
    return result;
  }

  // Make a nice string to print in the error case to tell user what is the
  //  required heaer row format

  for (let i = 0; i < csvHeaderNames.length; ++i) {
    if (i === 0) {
      printDictKeys = csvHeaderNames[i];
    } else {
      printDictKeys += ',' + csvHeaderNames[i];
    }
  }

  let dictElems = rawCsvData.split('\n');
  let numColumns = 0;
  let filteredDictElems = [];

  // The task of the following loop is to convert raw CSV rows into easily parseable
  // and streamlined versions of the rows with an internalDelimiter replacing the standard
  // comma; it is presumed (and checked) that the internalDelimiter cannot exist as a valid
  // sequence of characters in the user's data.

  // This conversion process also strips leading and trailing whitespace from each row,
  // discards empty rows, correctly interprets and removes all double quotes that programs like
  // Excel use to support user columns that contain special characters, most notably, the comma
  // delimiter. A double-quote that is contained within a double-quoted column value
  // must appear in this raw data as a sequence of two double quotes. Furthermore, any column
  // value in the raw CSV data that does not contain a delimiter may or may not be enclosed in
  // double quotes. It is the Excel convention to not use double qoutes unless necessary, and
  // there is no reasonable way to tell Excel to surround every column value with double quotes.
  // Any files that were directly "exported" by CLAMP itself from the Managing Dictionaries
  // capability, surround all columns with double quotes.

  for (let i = 0; i < dictElems.length; i++) {

    let oneRow = dictElems[i].trim();
    let j = 0;
    let inQuote = false
    let nextChar = undefined;
    let prevChar = null;


    if (oneRow === '') {
      continue; // Skip blank rows
    } else if (oneRow.indexOf(internalDelimiter) !== -1) {
      result.errorMessages += '\nRow #' + i + ' contains illegal sequence of characters (' + internalDelimiter + ')';
      break;
    } else {
      nextChar = oneRow[1];
    }

    let newStr = '';
    numColumns = 1;

    // This "while loop" performs the very meticulous task of removing double quotes that
    // are used by Excel to encase special characters as user string value data,
    // and manages to correctly identify columns that are defined with or without
    // double quotes and to process the comma delimiter correctly when encountered
    // as a user value within a column. Such a column would have to be encased in
    // double quotes; a comma found outside double quotes IS a delimiter.

    while (j < oneRow.length) {
      if (oneRow[j] === '"') {
        if (inQuote === false) {
          if (prevChar !== delimiter && prevChar !== null) {
            result.errorMessages += '\nMismatched double quotes or illegal whitespace around delimiter at row #' + (i + 1) + ' near column #' + numColumns;
            break;
          } else {
            inQuote = true;
          }
        } else {
          if (nextChar === '"') {
            newStr += '"';
            ++j;
          } else if ((nextChar !== delimiter) && (nextChar !== undefined)) {
            result.errorMessages += '\nRow #' + (i + 1) + ' is badly formatted at column #' + numColumns + '. Perhaps an unescaped double quote.';
            break;
          } else if (nextChar === delimiter) {
            ++numColumns;
            inQuote = false;
            newStr += internalDelimiter;
            prevChar = delimiter;
            j += 2;
            nextChar = oneRow[j + 1];
            continue;
          } else {
            ++numColumns;
            inQuote = false;
            break;
          }
        }
      } else {
        if (oneRow[j] === delimiter && inQuote === false) {
          newStr += internalDelimiter;
          ++numColumns;
        } else {
          newStr += oneRow[j];
        }
      }
      prevChar = oneRow[j];
      ++j;
      nextChar = oneRow[j + 1]; // can result in undefined at the end
    }

    if (result.errorMessages === '' && inQuote !== false) {
      result.errorMessages += '\nMismatched double quotes at row #' + (i + 1);
      break;
    } else if (result.errorMessages === '' && numColumns < jsonKeyNames.length) {
      result.errorMessages += '\nNot enough columns (' + jsonKeyNames.length + ') at row #' + (i + 1);
      break;
    }

    filteredDictElems.push(newStr);
  }

  if (result.errorMessages !== '') {
    return result;
  }

  // Perform further checks on data that is now in JSON form
  if (filteredDictElems.length < 2) {
    result.errorMessages += '\nNot enough row data found in import file. Need at least a header row and one row of data';
    return result;
  }

  // Now that we have something reliably parsed into sanitized columns lets run some checks
  // and convert it all into an array of JSON objects to push to the back end if all the
  // checks pass.

  let headers = filteredDictElems[0].split(internalDelimiter);

  // check that headers are included in proper order
  for (let i = 0; i < jsonKeyNames.length; ++i) {
    if (csvHeaderNames[i] !== headers[i]) {
      result.errorMessages += 'Row 1 header key at column #' + (i + 1) + ' is a mismatch. Expected row header must contain at least:\n' + printDictKeys;
      return result;
    }
  }

  // Convert the ASCII rows of data into an array of JSON obects that omit the header
  // row which is not sent to the back end.

  for (let i = 1; i < filteredDictElems.length; i++) {
    let data = filteredDictElems[i].split(internalDelimiter);
    let obj = {};
    for (let j = 0; j < data.length && j < jsonKeyNames.length; j++) {
      let value = data[j].trim();
      if (mandatory[j] === true && value === '') {
        result.errorMessages += '\n' + csvHeaderNames[j] + ' at row #' + (i + 1) + ' is empty but requires a value.';
      }
      obj[jsonKeyNames[j]] = value;
    }
    result.jsonObjArray.push(obj);
  }

  if (result.errorMessages !== '') {
    // If we have errors, return empty parse result even though some things
    // may have parsed properly. We do not want to encourage the caller
    // to think the data is good for use.
    result.jsonObjArray = [];
  }

  return result;
}