From 4eed58ba1a434261510c996316d2d201a40eb760 Mon Sep 17 00:00:00 2001 From: herbert Date: Fri, 28 Feb 2020 13:51:00 +0100 Subject: Update command and dataprovider components Adjust license and changes Issue-ID: SDNC-1092 Signed-off-by: herbert Change-Id: I8a0a339e4ecee828ae402f67d9dfc4dff7c3e59a Signed-off-by: herbert --- .../setup/DataMigrationProviderImpl.java | 8 +++++ .../sdnr/wt/dataprovider/setup/Program.java | 39 ++++++++++++++-------- .../wt/dataprovider/setup/ReleaseInformation.java | 24 +++++++++++-- .../wt/dataprovider/setup/data/DatabaseInfo.java | 17 ++++++---- .../setup/data/MavenDatabasePluginInitFile.java | 16 ++++++--- .../setup/elalto/ElAltoReleaseInformation.java | 12 ++++++- .../frankfurt/FrankfurtReleaseInformation.java | 32 +++++++++++++++--- .../setup/guilin/GuilinReleaseInformation.java | 11 ++++++ 8 files changed, 127 insertions(+), 32 deletions(-) (limited to 'sdnr/wt/data-provider/setup/src/main') diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java index ff5406a99..c20a1ad06 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java @@ -279,6 +279,9 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService { return false; } AcknowledgedResponse response = null; + if(!ri.runPreInitCommands(this.dbClient)) { + return false; + } for (ComponentName component : ri.getComponents()) { try { if (ri.hasOwnDbIndex(component)) { @@ -309,6 +312,9 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService { return false; } } + if(!ri.runPostInitCommands(this.dbClient)) { + return false; + } return true; } @@ -325,6 +331,7 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService { if (entries.size() <= 0) { LOG.info("no aliases to clear"); } else { + //check for every component of release if alias exists for (ComponentName component : ri.getComponents()) { String aliasToDelete = ri.getAlias(component, dbPrefix); AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete); @@ -348,6 +355,7 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService { if (entries2.size() <= 0) { LOG.info("no indices to clear"); } else { + //check for every component of release if index exists for (ComponentName component : ri.getComponents()) { String indexToDelete = ri.getIndex(component, dbPrefix); IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete); diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java index abb372920..23a64f252 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java @@ -146,7 +146,8 @@ public class Program { System.exit(1); } try { - initLog(getOptionOrDefault(cmd, OPTION_SILENT_SHORT, false), null,getOptionOrDefault(cmd, OPTION_DEBUG_SHORT, false)?Level.DEBUG:Level.INFO); + initLog(getOptionOrDefault(cmd, OPTION_SILENT_SHORT, false), null, + getOptionOrDefault(cmd, OPTION_DEBUG_SHORT, false) ? Level.DEBUG : Level.INFO); } catch (ParseException e2) { } @@ -241,9 +242,12 @@ public class Program { throw new Exception("please add output file parameter"); } DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) }, - username, password,trustAll); - DataMigrationReport report = service.importData(filename,false); + username, password, trustAll); + DataMigrationReport report = service.importData(filename, false); LOG.info(report); + if(!report.completed()) { + throw new Exception("db import seems to be not executed completed"); + } } /** @@ -260,9 +264,13 @@ public class Program { throw new Exception("please add output file parameter"); } DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) }, - username, password,trustAll); + username, password, trustAll); DataMigrationReport report = service.exportData(filename); LOG.info(report); + if(!report.completed()) { + throw new Exception("db export seems to be not executed completed"); + } + } /** @@ -279,10 +287,10 @@ public class Program { /** * @param cmd - * @throws ParseException * @throws java.text.ParseException + * @throws Exception */ - private static void cmd_clear_db(CommandLine cmd) throws ParseException, java.text.ParseException { + private static void cmd_clear_db(CommandLine cmd) throws Exception { Release r = getOptionOrDefault(cmd, OPTION_VERSION_SHORT, Release.CURRENT_RELEASE); String dbUrl = getOptionOrDefault(cmd, "db", DEFAULT_DBURL); String dbPrefix = getOptionOrDefault(cmd, "p", DEFAULT_DBPREFIX); @@ -290,16 +298,18 @@ public class Program { String password = getOptionOrDefault(cmd, "dbp", null); boolean trustAll = getOptionOrDefault(cmd, OPTION_TRUSTINSECURESSL_SHORT, false); DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) }, - username, password,trustAll); - service.clearDatabase(r, dbPrefix); + username, password, trustAll); + if (!service.clearDatabase(r, dbPrefix)) { + throw new Exception("failed to init database"); + } } /** * @param cmd - * @throws ParseException * @throws java.text.ParseException + * @throws Exception */ - private static void cmd_init_db(CommandLine cmd) throws ParseException, java.text.ParseException { + private static void cmd_init_db(CommandLine cmd) throws Exception { Release r = getOptionOrDefault(cmd, OPTION_VERSION_SHORT, Release.CURRENT_RELEASE); int numShards = getOptionOrDefault(cmd, OPTION_SHARDS_SHORT, DEFAULT_SHARDS); int numReplicas = getOptionOrDefault(cmd, OPTION_REPLICAS_SHORT, DEFAULT_REPLICAS); @@ -309,9 +319,11 @@ public class Program { String password = getOptionOrDefault(cmd, "dbp", null); boolean trustAll = getOptionOrDefault(cmd, OPTION_TRUSTINSECURESSL_SHORT, false); DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] { HostInfo.parse(dbUrl) }, - username, password,trustAll); + username, password, trustAll); boolean forceRecreate = cmd.hasOption(OPTION_FORCE_RECREATE_SHORT); - service.initDatabase(r, numShards, numReplicas, dbPrefix, forceRecreate); + if (!service.initDatabase(r, numShards, numReplicas, dbPrefix, forceRecreate)) { + throw new Exception("failed to init database"); + } } @@ -329,7 +341,8 @@ public class Program { options.addOption(createOption("p", "prefix", true, "prefix for db indices", false)); options.addOption(createOption(OPTION_VERSION_SHORT, "version", true, "version", false)); options.addOption(createOption(OPTION_DEBUG_SHORT, "verbose", false, "verbose mode", false)); - options.addOption(createOption(OPTION_TRUSTINSECURESSL_SHORT, "trust-insecure", false, "trust insecure ssl certs", false)); + options.addOption(createOption(OPTION_TRUSTINSECURESSL_SHORT, "trust-insecure", false, + "trust insecure ssl certs", false)); options.addOption(createOption("w", "wait", true, "wait delay for yellow status", false)); options.addOption( createOption(OPTION_FORCE_RECREATE_SHORT, "force-recreate", false, "delete if sth exists", false)); diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java index 5c5aa2a8b..f45d5e7d5 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java @@ -24,6 +24,7 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup; import java.util.Map; import java.util.Set; +import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; @@ -76,13 +77,16 @@ public abstract class ReleaseInformation { return dbMap.get(name) == null ? null : dbMap.get(name).doctype; } - /** + public String getDatabaseMapping(ComponentName name) { + return dbMap.get(name) == null ? null : dbMap.get(name).getMapping(); + } + /** * get database doctype definition for component * @param name * @return mappings or null if not exists */ - public String getDatabaseMapping(ComponentName name) { - return dbMap.get(name) == null ? null : dbMap.get(name).getMapping(); + public String getDatabaseMapping(ComponentName name,boolean useStrict) { + return dbMap.get(name) == null ? null : dbMap.get(name).getMapping(useStrict); } /** * get database settings definition for component @@ -152,6 +156,20 @@ public abstract class ReleaseInformation { return true; } + + /** + * @param dbClient + * @return if succeeded or not + */ + protected abstract boolean runPreInitCommands(HtDatabaseClient dbClient); + + + /** + * + * @param dbClient + * @return if succeeded or not + */ + protected abstract boolean runPostInitCommands(HtDatabaseClient dbClient); } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java index a6e81343a..4d82a0901 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/DatabaseInfo.java @@ -31,21 +31,22 @@ public class DatabaseInfo { private final String mapping; private final String settingsFormat; private final String index; + public String getIndex(String version) { return this.index + version; } public DatabaseInfo(String alias, String doctype, String mapping) { - this(alias,alias,doctype,mapping); + this(alias, alias, doctype, mapping); } - public DatabaseInfo(String index,String alias, String doctype, String mapping) { - this(index,alias,doctype, mapping, + public DatabaseInfo(String index, String alias, String doctype, String mapping) { + this(index, alias, doctype, mapping, "{\"index\":{\"number_of_shards\":%d,\"number_of_replicas\":%d},\"analysis\":{\"analyzer\":{\"content\":" + "{\"type\":\"custom\",\"tokenizer\":\"whitespace\"}}}}"); } - public DatabaseInfo(String index,String alias,String doctype, String mapping, String settingsformat) { + public DatabaseInfo(String index, String alias, String doctype, String mapping, String settingsformat) { this.index = index; this.alias = alias; this.doctype = doctype; @@ -53,10 +54,14 @@ public class DatabaseInfo { this.settingsFormat = settingsformat; } - public String getMapping() { - return this.mapping==null?null:String.format("{\"%s\":{\"properties\":%s}}",this.doctype, this.mapping); + return this.getMapping(false); } + public String getMapping(boolean useStrict) { + return this.mapping == null ? null + : String.format("{\"%s\":{%s\"properties\":%s}}", this.doctype,useStrict?"\"dynamic\": \"strict\",":"", this.mapping); + } + public String getSettings(int shards, int replicas) { return String.format(this.settingsFormat, shards, replicas); } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java index a63f8d4c1..e923b2728 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MavenDatabasePluginInitFile.java @@ -5,6 +5,8 @@ * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property. * All rights reserved. * ================================================================================ + * Update Copyright (C) 2020 AT&T Intellectual Property. All rights reserved. + * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -23,7 +25,6 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; import java.util.List; @@ -44,15 +45,20 @@ public class MavenDatabasePluginInitFile { "{\"type\":\"custom\",\"tokenizer\":\"whitespace\"}}}}"; public static void create(Release release, String filename) throws IOException { - + ReleaseInformation ri = ReleaseInformation.getInstance(release); Set comps=ri.getComponents(); - List lines = new ArrayList(); + List lines = new ArrayList<>(); for(ComponentName c:comps) { lines.add(String.format("PUT:%s/:{"+settings+","+mappings+"}",ri.getIndex(c),shards,replicas,ri.getDatabaseMapping(c))); lines.add(String.format("PUT:%s/_alias/%s/:{}", ri.getIndex(c),ri.getAlias(c))); } - Files.write(new File(filename).toPath(),lines,StandardCharsets.UTF_8); - + + File filePath = new File(filename); + if (filePath.getParentFile() != null && !filePath.getParentFile().exists()){ + //Crate Directory if missing + filePath.getParentFile().mkdirs(); + } + Files.write(filePath.toPath(), lines); } } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java index 559468f79..1236bd8e5 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java @@ -28,6 +28,7 @@ import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; +import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; public class ElAltoReleaseInformation extends ReleaseInformation { @@ -87,6 +88,15 @@ public class ElAltoReleaseInformation extends ReleaseInformation { } return c; } - + + @Override + protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + return true; + } + + @Override + protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + return true; + } } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java index f83263b62..4fab820b7 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java @@ -21,18 +21,25 @@ */ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.frankfurt; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; +import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest; +import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; public class FrankfurtReleaseInformation extends ReleaseInformation { + private final Logger LOG = LoggerFactory.getLogger(FrankfurtReleaseInformation.class); private final Map> converters = new HashMap<>(); - + public FrankfurtReleaseInformation() { super(Release.FRANKFURT_R1, createDBMap()); } @@ -67,11 +74,28 @@ public class FrankfurtReleaseInformation extends ReleaseInformation { @Override public SearchHitConverter getConverter(Release dst, ComponentName comp) { - SearchHitConverter c=this.converters.containsKey(dst)?this.converters.get(dst).get(comp):null; - if(c==null) { - c=super.getConverter(dst, comp); + SearchHitConverter c = this.converters.containsKey(dst) ? this.converters.get(dst).get(comp) : null; + if (c == null) { + c = super.getConverter(dst, comp); } return c; } + @Override + protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + + ClusterSettingsResponse response = null; + try { + response = dbClient.setupClusterSettings(new ClusterSettingsRequest(false)); + } catch (IOException e) { + LOG.warn("problem setting up cluster: {}", e); + } + return response == null ? false : response.isAcknowledged(); + } + + @Override + protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + return true; + } + } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java index 4912e4bc4..7b5e38116 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java @@ -29,6 +29,7 @@ import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; +import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; @@ -68,4 +69,14 @@ public class GuilinReleaseInformation extends ReleaseInformation { return null; } + @Override + protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + return true; + } + + @Override + protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + return true; + } + } -- cgit 1.2.3-korg