Update command and dataprovider components

Adjust license and changes

Issue-ID: SDNC-1092
Signed-off-by: herbert <herbert.eiselt@highstreet-technologies.com>
Change-Id: I8a0a339e4ecee828ae402f67d9dfc4dff7c3e59a
Signed-off-by: herbert <herbert.eiselt@highstreet-technologies.com>

8 files changed, 127 insertions, 32 deletions

diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java
index ff5406a99..c20a1ad06 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java
@@ -279,6 +279,9 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService {
 			return false;
 		}
 		AcknowledgedResponse response = null;
+		if(!ri.runPreInitCommands(this.dbClient)) {
+			return false;
+		}
 		for (ComponentName component : ri.getComponents()) {
 			try {
 				if (ri.hasOwnDbIndex(component)) {
@@ -309,6 +312,9 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService {
 				return false;
 			}
 		}
+		if(!ri.runPostInitCommands(this.dbClient)) {
+			return false;
+		}
 		return true;
 	}
 
@@ -325,6 +331,7 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService {
 		if (entries.size() <= 0) {
 			LOG.info("no aliases to clear");
 		} else {
+			//check for every component of release if alias exists
 			for (ComponentName component : ri.getComponents()) {
 				String aliasToDelete = ri.getAlias(component, dbPrefix);
 				AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete);
@@ -348,6 +355,7 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService {
 		if (entries2.size() <= 0) {
 			LOG.info("no indices to clear");
 		} else {
+			//check for every component of release if index exists
 			for (ComponentName component : ri.getComponents()) {
 				String indexToDelete = ri.getIndex(component, dbPrefix);
 				IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete);
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java
index abb372920..23a64f252 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java
@@ -146,7 +146,8 @@ public class Program {
 			System.exit(1);
 		}
 		try {
-			initLog(getOptionOrDefault(cmd, OPTION_SILENT_SHORT, false), null,getOptionOrDefault(cmd, OPTION_DEBUG_SHORT, false)?Level.DEBUG:Level.INFO);
+			initLog(getOptionOrDefault(cmd, OPTION_SILENT_SHORT, false), null,
+					getOptionOrDefault(cmd, OPTION_DEBUG_SHORT, false) ? Level.DEBUG : Level.INFO);
 		} catch (ParseException e2) {
 
 		}
@@ -241,9 +242,12 @@ public class Program {
 			throw new Exception("please add output file parameter");
 		}
 		DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) },
-				username, password,trustAll);
-		DataMigrationReport report = service.importData(filename,false);
+				username, password, trustAll);
+		DataMigrationReport report = service.importData(filename, false);
 		LOG.info(report);
+		if(!report.completed()) {
+			throw new Exception("db import seems to be not executed completed");
+		}
 	}
 
 	/**
@@ -260,9 +264,13 @@ public class Program {
 			throw new Exception("please add output file parameter");
 		}
 		DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) },
-				username, password,trustAll);
+				username, password, trustAll);
 		DataMigrationReport report = service.exportData(filename);
 		LOG.info(report);
+		if(!report.completed()) {
+			throw new Exception("db export seems to be not executed completed");
+		}
+
 	}
 
 	/**
@@ -279,10 +287,10 @@ public class Program {
 
 	/**
 	 * @param cmd
-	 * @throws ParseException 
 	 * @throws java.text.ParseException 
+	 * @throws Exception 
 	 */
-	private static void cmd_clear_db(CommandLine cmd) throws ParseException, java.text.ParseException {
+	private static void cmd_clear_db(CommandLine cmd) throws Exception {
 		Release r = getOptionOrDefault(cmd, OPTION_VERSION_SHORT, Release.CURRENT_RELEASE);
 		String dbUrl = getOptionOrDefault(cmd, "db", DEFAULT_DBURL);
 		String dbPrefix = getOptionOrDefault(cmd, "p", DEFAULT_DBPREFIX);
@@ -290,16 +298,18 @@ public class Program {
 		String password = getOptionOrDefault(cmd, "dbp", null);
 		boolean trustAll = getOptionOrDefault(cmd, OPTION_TRUSTINSECURESSL_SHORT, false);
 		DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) },
-				username, password,trustAll);
-		service.clearDatabase(r, dbPrefix);
+				username, password, trustAll);
+		if (!service.clearDatabase(r, dbPrefix)) {
+			throw new Exception("failed to init database");
+		}
 	}
 
 	/**
 	 * @param cmd
-	 * @throws ParseException 
 	 * @throws java.text.ParseException 
+	 * @throws Exception 
 	 */
-	private static void cmd_init_db(CommandLine cmd) throws ParseException, java.text.ParseException {
+	private static void cmd_init_db(CommandLine cmd) throws Exception {
 		Release r = getOptionOrDefault(cmd, OPTION_VERSION_SHORT, Release.CURRENT_RELEASE);
 		int numShards = getOptionOrDefault(cmd, OPTION_SHARDS_SHORT, DEFAULT_SHARDS);
 		int numReplicas = getOptionOrDefault(cmd, OPTION_REPLICAS_SHORT, DEFAULT_REPLICAS);
@@ -309,9 +319,11 @@ public class Program {
 		String password = getOptionOrDefault(cmd, "dbp", null);
 		boolean trustAll = getOptionOrDefault(cmd, OPTION_TRUSTINSECURESSL_SHORT, false);
 		DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) },
-				username, password,trustAll);
+				username, password, trustAll);
 		boolean forceRecreate = cmd.hasOption(OPTION_FORCE_RECREATE_SHORT);
-		service.initDatabase(r, numShards, numReplicas, dbPrefix, forceRecreate);
+		if (!service.initDatabase(r, numShards, numReplicas, dbPrefix, forceRecreate)) {
+			throw new Exception("failed to init database");
+		}
 
 	}
 
@@ -329,7 +341,8 @@ public class Program {
 		options.addOption(createOption("p", "prefix", true, "prefix for db indices", false));
 		options.addOption(createOption(OPTION_VERSION_SHORT, "version", true, "version", false));
 		options.addOption(createOption(OPTION_DEBUG_SHORT, "verbose", false, "verbose mode", false));
-		options.addOption(createOption(OPTION_TRUSTINSECURESSL_SHORT, "trust-insecure", false, "trust insecure ssl certs", false));
+		options.addOption(createOption(OPTION_TRUSTINSECURESSL_SHORT, "trust-insecure", false,
+				"trust insecure ssl certs", false));
 		options.addOption(createOption("w", "wait", true, "wait delay for yellow status", false));
 		options.addOption(
 				createOption(OPTION_FORCE_RECREATE_SHORT, "force-recreate", false, "delete if sth exists", false));
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java
index 5c5aa2a8b..f45d5e7d5 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java
@@ -24,6 +24,7 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup;
 import java.util.Map;
 import java.util.Set;
 
+import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo;
@@ -76,13 +77,16 @@ public abstract class ReleaseInformation {
 		return dbMap.get(name) == null ? null : dbMap.get(name).doctype;
 	}
 
-	/**
+	public String getDatabaseMapping(ComponentName name) {
+		return dbMap.get(name) == null ? null : dbMap.get(name).getMapping();
+	}
+		/**
 	 * get database doctype definition for component
 	 * @param name
 	 * @return mappings or null if not exists
 	 */
-	public String getDatabaseMapping(ComponentName name) {
-		return dbMap.get(name) == null ? null : dbMap.get(name).getMapping();
+	public String getDatabaseMapping(ComponentName name,boolean useStrict) {
+		return dbMap.get(name) == null ? null : dbMap.get(name).getMapping(useStrict);
 	}
 	/**
 	 * get database settings definition for component
@@ -152,6 +156,20 @@ public abstract class ReleaseInformation {
 		return true;
 		
 	}
+
+	/**
+	 * @param dbClient
+	 * @return if succeeded or not
+	 */
+	protected abstract boolean runPreInitCommands(HtDatabaseClient dbClient);
+
+
+	/**
+	 * 
+	 * @param dbClient
+	 * @return if succeeded or not
+	 */
+	protected abstract boolean runPostInitCommands(HtDatabaseClient dbClient);
 	
 
 }
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java
index a6e81343a..4d82a0901 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java
@@ -31,21 +31,22 @@ public class DatabaseInfo {
 	private final String mapping;
 	private final String settingsFormat;
 	private final String index;
+
 	public String getIndex(String version) {
 		return this.index + version;
 	}
 
 	public DatabaseInfo(String alias, String doctype, String mapping) {
-		this(alias,alias,doctype,mapping);
+		this(alias, alias, doctype, mapping);
 	}
 
-	public DatabaseInfo(String index,String alias, String doctype, String mapping) {
-		this(index,alias,doctype, mapping,
+	public DatabaseInfo(String index, String alias, String doctype, String mapping) {
+		this(index, alias, doctype, mapping,
 				"{\"index\":{\"number_of_shards\":%d,\"number_of_replicas\":%d},\"analysis\":{\"analyzer\":{\"content\":"
 						+ "{\"type\":\"custom\",\"tokenizer\":\"whitespace\"}}}}");
 	}
 
-	public DatabaseInfo(String index,String alias,String doctype, String mapping, String settingsformat) {
+	public DatabaseInfo(String index, String alias, String doctype, String mapping, String settingsformat) {
 		this.index = index;
 		this.alias = alias;
 		this.doctype = doctype;
@@ -53,10 +54,14 @@ public class DatabaseInfo {
 		this.settingsFormat = settingsformat;
 	}
 	
-	
 	public String getMapping() {
-		return this.mapping==null?null:String.format("{\"%s\":{\"properties\":%s}}",this.doctype, this.mapping);
+		return this.getMapping(false);
 	}
+	public String getMapping(boolean useStrict) {
+		return this.mapping == null ? null
+				: String.format("{\"%s\":{%s\"properties\":%s}}", this.doctype,useStrict?"\"dynamic\": \"strict\",":"", this.mapping);
+	}
+
 	public String getSettings(int shards, int replicas) {
 		return String.format(this.settingsFormat, shards, replicas);
 	}
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java
index a63f8d4c1..e923b2728 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java
@@ -5,6 +5,8 @@
  * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property.
  * All rights reserved.
  * ================================================================================
+ * Update Copyright (C) 2020 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -23,7 +25,6 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
@@ -44,15 +45,20 @@ public class MavenDatabasePluginInitFile {
     "{\"type\":\"custom\",\"tokenizer\":\"whitespace\"}}}}";
 
 	public static void create(Release release, String filename) throws IOException {
-		
+
 		ReleaseInformation ri = ReleaseInformation.getInstance(release);
 		Set<ComponentName> comps=ri.getComponents();
-		List<String> lines = new ArrayList<String>();
+		List<String> lines = new ArrayList<>();
 		for(ComponentName c:comps) {
 			lines.add(String.format("PUT:%s/:{"+settings+","+mappings+"}",ri.getIndex(c),shards,replicas,ri.getDatabaseMapping(c)));
 			lines.add(String.format("PUT:%s/_alias/%s/:{}", ri.getIndex(c),ri.getAlias(c)));
 		}
-		Files.write(new File(filename).toPath(),lines,StandardCharsets.UTF_8);
-		
+
+		File filePath = new File(filename);
+		if (filePath.getParentFile() != null && !filePath.getParentFile().exists()){
+			//Crate Directory if missing
+			filePath.getParentFile().mkdirs();
+		}
+		Files.write(filePath.toPath(), lines);
 	}
 }
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java
index 559468f79..1236bd8e5 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java
@@ -28,6 +28,7 @@ import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
+import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation;
 
 public class ElAltoReleaseInformation extends ReleaseInformation {
@@ -87,6 +88,15 @@ public class ElAltoReleaseInformation extends ReleaseInformation {
 		}
 		return c;
 	}
-	
+
+	@Override
+	protected boolean runPreInitCommands(HtDatabaseClient dbClient) {
+		return true;
+	}
+
+	@Override
+	protected boolean runPostInitCommands(HtDatabaseClient dbClient) {
+		return true;
+	}
 	
 }
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java
index f83263b62..4fab820b7 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java
@@ -21,18 +21,25 @@
  */
 package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.frankfurt;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
+import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest;
+import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation;
 
 public class FrankfurtReleaseInformation extends ReleaseInformation {
 
+	private final Logger LOG = LoggerFactory.getLogger(FrankfurtReleaseInformation.class);
 	private final Map<Release, Map<ComponentName, SearchHitConverter>> converters = new HashMap<>();
-	
+
 	public FrankfurtReleaseInformation() {
 		super(Release.FRANKFURT_R1, createDBMap());
 	}
@@ -67,11 +74,28 @@ public class FrankfurtReleaseInformation extends ReleaseInformation {
 
 	@Override
 	public SearchHitConverter getConverter(Release dst, ComponentName comp) {
-		SearchHitConverter c=this.converters.containsKey(dst)?this.converters.get(dst).get(comp):null;
-		if(c==null) {
-			c=super.getConverter(dst, comp);
+		SearchHitConverter c = this.converters.containsKey(dst) ? this.converters.get(dst).get(comp) : null;
+		if (c == null) {
+			c = super.getConverter(dst, comp);
 		}
 		return c;
 	}
 
+	@Override
+	protected boolean runPreInitCommands(HtDatabaseClient dbClient) {
+
+		ClusterSettingsResponse response = null;
+		try {
+			response = dbClient.setupClusterSettings(new ClusterSettingsRequest(false));
+		} catch (IOException e) {
+			LOG.warn("problem setting up cluster: {}", e);
+		}
+		return response == null ? false : response.isAcknowledged();
+	}
+
+	@Override
+	protected boolean runPostInitCommands(HtDatabaseClient dbClient) {
+		return true;
+	}
+
 }
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java
index 4912e4bc4..7b5e38116 100644
--- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java
+++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java
@@ -29,6 +29,7 @@ import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
+import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation;
 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo;
 
@@ -68,4 +69,14 @@ public class GuilinReleaseInformation extends ReleaseInformation {
 		return null;
 	}
 
+	@Override
+	protected boolean runPreInitCommands(HtDatabaseClient dbClient) {
+		return true;
+	}
+
+	@Override
+	protected boolean runPostInitCommands(HtDatabaseClient dbClient) {
+		return true;
+	}
+
 }