diff options
Diffstat (limited to 'sdnr/wt/data-provider/setup/src')
19 files changed, 1800 insertions, 465 deletions
diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java index 7d2adc5ad..466a3e290 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java @@ -21,127 +21,33 @@ */ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import org.json.JSONObject; -import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; -import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit; -import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult; -import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo; -import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry; -import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList; -import org.onap.ccsdk.features.sdnr.wt.common.database.data.EsVersion; -import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry; -import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList; -import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateAliasRequest; -import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateIndexRequest; -import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteAliasRequest; -import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteIndexRequest; -import org.onap.ccsdk.features.sdnr.wt.common.database.responses.AcknowledgedResponse; -import org.onap.ccsdk.features.sdnr.wt.common.database.responses.GetInfoResponse; -import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListAliasesResponse; -import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListIndicesResponse; -import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData; -import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; -import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; -import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup; -import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database.ElasticsearchDataMigrationProvider; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database.MariaDbDataMigrationProvider; public class DataMigrationProviderImpl implements DataMigrationProviderService { + private final DataMigrationProviderService dbProvider; - private static final Logger LOG = LoggerFactory.getLogger(DataMigrationProviderImpl.class); - - private final HtDatabaseClient dbClient; - - public DataMigrationProviderImpl(HostInfo[] hosts, String username, String password, boolean trustAll, + public DataMigrationProviderImpl(SdnrDbType type, String url, String username, String password, boolean trustAll, long timeoutms) throws Exception { - - dbClient = HtDatabaseClient.getClient(hosts, username, password, trustAll, timeoutms); + if (type == SdnrDbType.ELASTICSEARCH) { + dbProvider = new ElasticsearchDataMigrationProvider(url, username, password, trustAll, timeoutms); + } else { + dbProvider = new MariaDbDataMigrationProvider(url, username, password, trustAll, timeoutms); + } } @Override public DataMigrationReport importData(String filename, boolean dryrun) throws Exception { - return this.importData(filename, dryrun, Release.CURRENT_RELEASE); + return this.dbProvider.importData(filename, dryrun, Release.CURRENT_RELEASE); } + @Override public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception { - DataMigrationReport report = new DataMigrationReport(); - File file = new File(filename); - if (!file.exists()) { - if (dryrun) { - report.error("file %s not found", filename); - return report; - } - throw new FileNotFoundException(filename); - } - DataContainer container = null; - try { - container = DataContainer.load(file); - } catch (Exception e) { - if (dryrun) { - report.error("problem loading file %s: %s", filename, e.getMessage()); - return report; - } - throw new Exception("problem loading file " + filename, e); - } - ReleaseInformation ri = ReleaseInformation.getInstance(forRelease); - SearchHitConverter converter; - Set<ComponentName> components = ri.getComponents(); - //for all db components of dest architecture - for (ComponentName component : components) { - //convert to ComponentData for current release with existing ComponentData of the container - converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component); - if (converter == null) { - continue; - } - ComponentData data = converter.convert(container); - if (data != null) { - String indexName = ri.getAlias(component); - String dataTypeName = ri.getDataType(component); - if (dryrun) { - report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName); - } else { - LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName); - } - for (SearchHit item : data) { - if (!dryrun) { - String id = this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(), - item.getSourceAsString(), true); - if (!item.getId().equals(id)) { - LOG.warn("entry for {} with original id {} was written with another id {}", - component.getValue(), item.getId(), id); - } - } - } - } else { - if (dryrun) { - report.error("unable to convert data for " + component.getValue() + " from version " - + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n"); - } else { - LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(), - container.getRelease().getValue(), forRelease.getValue()); - } - } - } - LOG.info("import of {} completed", filename); - if (dryrun) { - report.log("import of %s completed", filename); - } - report.setCompleted(true); - return report; + return this.dbProvider.importData(filename, dryrun, forRelease); } @@ -151,49 +57,7 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService { */ @Override public DataMigrationReport exportData(String filename) { - DataMigrationReport report = new DataMigrationReport(); - - DataContainer container = new DataContainer(); - - filename = this.checkFilenameForWrite(filename); - LOG.info("output will be written to {}", filename); - //autodetect version - Release dbRelease = this.autoDetectRelease(); - if (dbRelease == null) { - report.error("unbable to detect db release. is database initialized?"); - return report; - } - ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease); - boolean componentsSucceeded = true; - for (ComponentName c : ri.getComponents()) { - ComponentData data = new ComponentData(c); - SearchResult<SearchHit> result = this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false); - data.addAll(result.getHits()); - container.addComponent(c, data); - } - try { - Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8); - report.setCompleted(componentsSucceeded); - } catch (IOException e) { - LOG.warn("problem writing data to {}: {}", filename, e); - } - return report; - } - - private String checkFilenameForWrite(String filename) { - File f = new File(filename); - if (!f.exists()) { - return filename; - } - return this.checkFilenameForWrite(filename, 0); - } - - private String checkFilenameForWrite(String filename, int apdx) { - File f = new File(String.format("$s.$d", filename, apdx)); - if (!f.exists()) { - return filename; - } - return this.checkFilenameForWrite(filename, apdx + 1); + return this.dbProvider.exportData(filename); } @Override @@ -202,226 +66,20 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService { } + @Override public Release autoDetectRelease() { - EsVersion dbVersion = this.readActualVersion(); - AliasesEntryList aliases = this.readAliases(); - IndicesEntryList indices = this.readIndices(); - if (indices == null) { - return null; - } - List<Release> foundReleases = new ArrayList<>(); - //if there are active aliases reduce indices to the active ones - if (aliases != null && aliases.size() > 0) { - indices = indices.subList(aliases.getLinkedIndices()); - } - for (Release r : Release.values()) { - if (r.isDbInRange(dbVersion)) { - ReleaseInformation ri = ReleaseInformation.getInstance(r); - if (ri != null && ri.containsIndices(indices)) { - foundReleases.add(r); - } - } - } - if (foundReleases.size() == 1) { - return foundReleases.get(0); - } - LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(), - foundReleases); - return null; - } - - private EsVersion readActualVersion() { - try { - GetInfoResponse response = this.dbClient.getInfo(); - return response.getVersion(); - } catch (Exception e) { - LOG.warn(e.getMessage()); - } - return null; - } - - private AliasesEntryList readAliases() { - AliasesEntryList entries = null; - try { - ListAliasesResponse response = this.dbClient.getAliases(); - entries = response.getEntries(); - } catch (ParseException | IOException e) { - LOG.error(e.getMessage()); - } - return entries; - } - - private IndicesEntryList readIndices() { - IndicesEntryList entries = null; - try { - ListIndicesResponse response = this.dbClient.getIndices(); - entries = response.getEntries(); - } catch (ParseException | IOException e) { - LOG.error(e.getMessage()); - } - return entries; + return this.dbProvider.autoDetectRelease(); } - @Override public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate, long timeoutms) { - if (timeoutms > 0) { - this.dbClient.waitForYellowStatus(timeoutms); - } - EsVersion dbVersion = this.readActualVersion(); - if (dbVersion == null) { - return false; - } - LOG.info("detected database version {}", dbVersion); - if (release == null) { - release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion); - if (release == null) { - LOG.warn("unable to autodetect release for this database version for release {}", - ReleaseGroup.CURRENT_RELEASE.name()); - return false; - } - LOG.info("autodetect release {}", release); - } - if (!release.isDbInRange(dbVersion)) { - LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release); - return false; - } - if (forceRecreate) { - this.clearDatabase(release, dbPrefix, 0); - } - ReleaseInformation ri = ReleaseInformation.getInstance(release); - AliasesEntryList aliases = this.readAliases(); - IndicesEntryList indices = this.readIndices(); - if (aliases == null || indices == null) { - return false; - } - AcknowledgedResponse response = null; - if (!ri.runPreInitCommands(this.dbClient)) { - return false; - } - for (ComponentName component : ri.getComponents()) { - try { - if (ri.hasOwnDbIndex(component)) { - //check if index already exists - String indexName = ri.getIndex(component, dbPrefix); - String aliasName = ri.getAlias(component, dbPrefix); - if (indices.findByIndex(indexName) == null) { - LOG.info("creating index for {}", component); - CreateIndexRequest request = new CreateIndexRequest(ri.getIndex(component, dbPrefix)); - request.mappings(new JSONObject(ri.getDatabaseMapping(component))); - request.settings(new JSONObject(ri.getDatabaseSettings(component, numShards, numReplicas))); - response = this.dbClient.createIndex(request); - LOG.info(response.isAcknowledged() ? "succeeded" : "failed"); - } else { - LOG.info("index {} for {} already exists", indexName, component); - } - //check if alias already exists - if (aliases.findByAlias(aliasName) == null) { - LOG.info("creating alias for {}", component); - response = this.dbClient.createAlias(new CreateAliasRequest(indexName, aliasName)); - LOG.info(response.isAcknowledged() ? "succeeded" : "failed"); - } else { - LOG.info("alias {} for index {} for {} already exists", aliasName, indexName, component); - } - } - } catch (IOException e) { - LOG.error(e.getMessage()); - return false; - } - } - if (!ri.runPostInitCommands(this.dbClient)) { - return false; - } - return true; + return this.dbProvider.initDatabase(release, numShards, numReplicas, dbPrefix, forceRecreate, timeoutms); } @Override public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) { - - if (timeoutms > 0) { - this.dbClient.waitForYellowStatus(timeoutms); - } - //check aliases - AliasesEntryList entries = this.readAliases(); - IndicesEntryList entries2 = this.readIndices(); - if (entries == null) { - return false; - } - if (release == null) { - EsVersion dbVersion = this.readActualVersion(); - if (dbVersion == null) { - return false; - } - LOG.info("detected database version {}", dbVersion); - release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion); - if (release == null) { - LOG.warn("unable to autodetect release for this database version for release {}", - ReleaseGroup.CURRENT_RELEASE.name()); - return false; - } - LOG.info("autodetect release {}", release); - } - ReleaseInformation ri = ReleaseInformation.getInstance(release); - AcknowledgedResponse response; - if (entries.size() <= 0) { - LOG.info("no aliases to clear"); - } else { - //check for every component of release if alias exists - for (ComponentName component : ri.getComponents()) { - String aliasToDelete = ri.getAlias(component, dbPrefix); - AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete); - if (entryToDelete != null) { - try { - LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex()); - response = this.dbClient.deleteAlias( - new DeleteAliasRequest(entryToDelete.getIndex(), entryToDelete.getAlias())); - LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed"); - } catch (IOException e) { - LOG.error(e.getMessage()); - return false; - } - } - else { - //try to find malformed typed index with alias name - IndicesEntry entry2ToDelete = entries2.findByIndex(aliasToDelete); - if (entry2ToDelete != null) { - try { - LOG.info("deleting index {}", entry2ToDelete.getName()); - response = this.dbClient.deleteIndex(new DeleteIndexRequest(entry2ToDelete.getName())); - LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed"); - } catch (IOException e) { - LOG.error(e.getMessage()); - return false; - } - } - } - } - } - if (entries2 == null) { - return false; - } - if (entries2.size() <= 0) { - LOG.info("no indices to clear"); - } else { - //check for every component of release if index exists - for (ComponentName component : ri.getComponents()) { - String indexToDelete = ri.getIndex(component, dbPrefix); - IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete); - if (entryToDelete != null) { - try { - LOG.info("deleting index {}", entryToDelete.getName()); - response = this.dbClient.deleteIndex(new DeleteIndexRequest(entryToDelete.getName())); - LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed"); - } catch (IOException e) { - LOG.error(e.getMessage()); - return false; - } - } - } - } - - return true; + return this.dbProvider.clearDatabase(release, dbPrefix, timeoutms); } /** @@ -429,34 +87,6 @@ public class DataMigrationProviderImpl implements DataMigrationProviderService { * @return */ public boolean clearCompleteDatabase(long timeoutms) { - if (timeoutms > 0) { - this.dbClient.waitForYellowStatus(timeoutms); - } - //check aliases and indices - AliasesEntryList aliases = this.readAliases(); - IndicesEntryList indices = this.readIndices(); - if (aliases == null || indices == null) { - return false; - } - for (AliasesEntry alias : aliases) { - try { - LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex()); - this.dbClient.deleteAlias(new DeleteAliasRequest(alias.getIndex(), alias.getAlias())); - } catch (IOException e) { - LOG.error("problem deleting alias {}: {}", alias.getAlias(), e); - return false; - } - } - for (IndicesEntry index : indices) { - try { - LOG.info("deleting index {}", index.getName()); - this.dbClient.deleteIndex(new DeleteIndexRequest(index.getName())); - } catch (IOException e) { - LOG.error("problem deleting index {}: {}", index.getName(), e); - return false; - } - } - return true; + return this.clearCompleteDatabase(timeoutms); } - } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderService.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderService.java index 0f1fc6c73..53ab32636 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderService.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderService.java @@ -22,7 +22,6 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup; import java.io.FileNotFoundException; - import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; @@ -31,7 +30,7 @@ public interface DataMigrationProviderService { /** * import data from file and write these to database - * + * * @param filename source * @param dryrun only loading file and check consistency, not pushing into database * @return report @@ -40,19 +39,22 @@ public interface DataMigrationProviderService { */ DataMigrationReport importData(String filename, boolean dryrun) throws FileNotFoundException, Exception; + DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception; /** * export current data to file - * + * * @param filename */ DataMigrationReport exportData(String filename); /** - * + * * @return */ Release getCurrentVersion(); + Release autoDetectRelease(); + /** * @param release * @param numShards @@ -67,8 +69,9 @@ public interface DataMigrationProviderService { /** * clean up the database all data will be removed complete structure will be destroyed - * + * * @return */ boolean clearDatabase(Release release, String dbPrefix, long timeoutms); + } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java index 54204f170..56973f459 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/Program.java @@ -37,7 +37,7 @@ import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import org.apache.log4j.RollingFileAppender; -import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.MavenDatabasePluginInitFile; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; @@ -77,7 +77,8 @@ public class Program { private static final int DEFAULT_SHARDS = 5; private static final int DEFAULT_REPLICAS = 1; private static final int DEFAULT_DATABASEWAIT_SECONDS = 30; - private static final String DEFAULT_DBURL = "http://sdnrdb:9200"; + private static final String DEFAULT_DBURL_ELASTICSEARCH = "http://sdnrdb:9200"; + private static final String DEFAULT_DBURL_MARIADB = "jdbc:mysql://sdnrdb:3306/sdnrdb"; private static final String DEFAULT_DBPREFIX = ""; private static final boolean DEFAULT_TRUSTINSECURESSL = false; @@ -103,6 +104,8 @@ public class Program { private static final String OPTION_DATABASE_LONG = "dburl"; private static final String OPTION_COMMAND_SHORT = "c"; private static final String OPTION_COMMAND_LONG = "cmd"; + private static final String OPTION_DATABASETYPE_SHORT = "dbt"; + private static final String OPTION_DATABASETYPE_LONG = "db-type"; private static final String OPTION_DATABASEUSER_SHORT = "dbu"; private static final String OPTION_DATABASEUSER_LONG = "db-username"; private static final String OPTION_DATABASEPASSWORD_SHORT = "dbp"; @@ -141,6 +144,9 @@ public class Program { if (def instanceof Release) { return cmd.hasOption(option) ? (T) Release.getValueBySuffix(cmd.getOptionValue(option)) : def; } + if (def instanceof SdnrDbType) { + return cmd.hasOption(option) ? (T) SdnrDbType.valueOf(cmd.getOptionValue(option).toUpperCase()) : def; + } if (cmd.hasOption(option) && cmd.getOptionValue(option) != null) { if (option.equals(OPTION_VERSION_SHORT)) { String v = cmd.getOptionValue(option); @@ -303,7 +309,7 @@ public class Program { if (filename == null) { throw new Exception("please add output file parameter"); } - DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] {HostInfo.parse(options.getUrl())}, + DataMigrationProviderImpl service = new DataMigrationProviderImpl(options.getType(), options.getUrl(), options.getUsername(), options.getPassword(), options.doTrustAll(), options.getTimeoutMs()); DataMigrationReport report = service.importData(filename, false); LOG.info(report); @@ -319,7 +325,7 @@ public class Program { if (filename == null) { throw new Exception("please add output file parameter"); } - DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] {HostInfo.parse(options.getUrl())}, + DataMigrationProviderImpl service = new DataMigrationProviderImpl(options.getType(), options.getUrl(), options.getUsername(), options.getPassword(), options.doTrustAll(), options.getTimeoutMs()); DataMigrationReport report = service.exportData(filename); LOG.info(report); @@ -342,7 +348,7 @@ public class Program { Release r = getOptionOrDefault(cmd, OPTION_VERSION_SHORT, (Release) null); DatabaseOptions options = new DatabaseOptions(cmd); String dbPrefix = getOptionOrDefault(cmd, OPTION_DATABASEPREFIX_SHORT, DEFAULT_DBPREFIX); - DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] {HostInfo.parse(options.getUrl())}, + DataMigrationProviderImpl service = new DataMigrationProviderImpl(options.getType(), options.getUrl(), options.getUsername(), options.getPassword(), options.doTrustAll(), options.getTimeoutMs()); if (!service.clearDatabase(r, dbPrefix, options.getTimeoutMs())) { throw new Exception("failed to init database"); @@ -352,7 +358,7 @@ public class Program { private static void cmd_clear_db_complete(CommandLine cmd) throws Exception { DatabaseOptions options = new DatabaseOptions(cmd); - DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] {HostInfo.parse(options.getUrl())}, + DataMigrationProviderImpl service = new DataMigrationProviderImpl(options.getType(), options.getUrl(), options.getUsername(), options.getPassword(), options.doTrustAll(), options.getTimeoutMs()); if (!service.clearCompleteDatabase(options.getTimeoutMs())) { throw new Exception("failed to init database"); @@ -366,7 +372,7 @@ public class Program { int numReplicas = getOptionOrDefault(cmd, OPTION_REPLICAS_SHORT, DEFAULT_REPLICAS); DatabaseOptions options = new DatabaseOptions(cmd); String dbPrefix = getOptionOrDefault(cmd, OPTION_DATABASEPREFIX_SHORT, DEFAULT_DBPREFIX); - DataMigrationProviderImpl service = new DataMigrationProviderImpl(new HostInfo[] {HostInfo.parse(options.getUrl())}, + DataMigrationProviderImpl service = new DataMigrationProviderImpl(options.getType(),options.getUrl(), options.getUsername(), options.getPassword(), options.doTrustAll(), options.getTimeoutMs()); boolean forceRecreate = cmd.hasOption(OPTION_FORCE_RECREATE_SHORT); if (!service.initDatabase(r, numShards, numReplicas, dbPrefix, forceRecreate, options.getTimeoutMs())) { @@ -380,6 +386,8 @@ public class Program { Options result = new Options(); result.addOption(createOption(OPTION_COMMAND_SHORT, OPTION_COMMAND_LONG, true, "command to execute", false)); result.addOption(createOption(OPTION_DATABASE_SHORT, OPTION_DATABASE_LONG, true, "database url", false)); + result.addOption(createOption(OPTION_DATABASETYPE_SHORT, OPTION_DATABASETYPE_LONG, true, + "database type (elasticsearch|mariadb)", false)); result.addOption(createOption(OPTION_DATABASEUSER_SHORT, OPTION_DATABASEUSER_LONG, true, "database basic auth username", false)); result.addOption(createOption(OPTION_DATABASEPASSWORD_SHORT, OPTION_DATABASEPASSWORD_LONG, true, @@ -432,10 +440,14 @@ public class Program { private final String password; private final boolean trustAll; private final long timeoutMs; + private final SdnrDbType type; public String getUrl() { return this.url; } + public SdnrDbType getType() { + return this.type; + } public String getUsername() { return this.username; } @@ -448,8 +460,11 @@ public class Program { public long getTimeoutMs() { return this.timeoutMs; } + public DatabaseOptions(CommandLine cmd) throws ParseException { - this.url = getOptionOrDefault(cmd, OPTION_DATABASE_SHORT, DEFAULT_DBURL); + this.type = getOptionOrDefault(cmd, OPTION_DATABASETYPE_LONG, SdnrDbType.ELASTICSEARCH); + this.url = getOptionOrDefault(cmd, OPTION_DATABASE_SHORT, + this.type == SdnrDbType.ELASTICSEARCH ? DEFAULT_DBURL_ELASTICSEARCH : DEFAULT_DBURL_MARIADB); this.username = getOptionOrDefault(cmd, OPTION_DATABASEUSER_SHORT, null); this.password = getOptionOrDefault(cmd, OPTION_DATABASEPASSWORD_SHORT, null); this.trustAll = getOptionOrDefault(cmd, OPTION_TRUSTINSECURESSL_SHORT, DEFAULT_TRUSTINSECURESSL); diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java index 8362b0030..6e5bdc522 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/ReleaseInformation.java @@ -21,13 +21,17 @@ */ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup; +import java.util.HashMap; import java.util.Map; import java.util.Set; import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.MariaDBTableInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.elalto.ElAltoReleaseInformation; @@ -42,15 +46,26 @@ public abstract class ReleaseInformation { // variables private final Release release; private final Map<ComponentName, DatabaseInfo> dbMap; + private Map<ComponentName, MariaDBTableInfo> mariadbMap; // end of variables // constructors public ReleaseInformation(Release r, Map<ComponentName, DatabaseInfo> dbMap) { + this(r, dbMap, new HashMap<>()); + } + + public ReleaseInformation(Release r, Map<ComponentName, DatabaseInfo> dbMap, + Map<ComponentName, MariaDBTableInfo> mariadbMap) { this.release = r; this.dbMap = dbMap; + this.mariadbMap = mariadbMap; } // end of constructors + protected Release getReleas() { + return this.release; + } + /** * get database alias for component * @@ -120,6 +135,18 @@ public abstract class ReleaseInformation { return dbMap.get(name) == null ? null : dbMap.get(name).getSettings(shards, replicas); } + public String getDatabaseMapping(ComponentName name, SdnrDbType dbType) { + switch (dbType) { + case ELASTICSEARCH: + return this.getDatabaseMapping(name); + case MARIADB: + return mariadbMap.get(name) == null ? null + : mariadbMap.get(name).getMapping(this.release.getDBSuffix()); + default: + return null; + } + } + /** * get converter for component data * @@ -172,7 +199,7 @@ public abstract class ReleaseInformation { * @param indices * @return true if components of this release are covered by the given indices */ - protected boolean containsIndices(IndicesEntryList indices) { + public boolean containsIndices(IndicesEntryList indices) { if (this.dbMap.size() <= 0) { return false; @@ -191,13 +218,19 @@ public abstract class ReleaseInformation { * @param dbClient * @return if succeeded or not */ - protected abstract boolean runPreInitCommands(HtDatabaseClient dbClient); + public abstract boolean runPreInitCommands(HtDatabaseClient dbClient); + + public abstract boolean runPreInitCommands(SqlDBClient dbClient); /** * * @param dbClient * @return if succeeded or not */ - protected abstract boolean runPostInitCommands(HtDatabaseClient dbClient); + public abstract boolean runPostInitCommands(HtDatabaseClient dbClient); + + public abstract boolean runPostInitCommands(SqlDBClient dbClient); + + } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MariaDBTableInfo.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MariaDBTableInfo.java new file mode 100644 index 000000000..956790942 --- /dev/null +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/MariaDBTableInfo.java @@ -0,0 +1,42 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : ccsdk features + * ================================================================================ + * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property. + * All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + */ +package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data; + +/** + * @author Michael Dürre + * + */ +public class MariaDBTableInfo { + public final String alias; + protected final String mapping; + + public MariaDBTableInfo(String alias, String mapping) { + this.alias = alias; + this.mapping = mapping; + } + + public String getMapping(String dbSuffix) { + return this.mapping.replaceAll("%s", dbSuffix); + } + + +} diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/Release.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/Release.java index 119479b97..22338e89a 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/Release.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/Release.java @@ -22,34 +22,51 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data; import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry; -import org.onap.ccsdk.features.sdnr.wt.common.database.data.EsVersion; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; + public enum Release { - EL_ALTO("el alto", "_v1", new EsVersion(2, 2, 0), new EsVersion(2, 2, 0)), - FRANKFURT_R1("frankfurt-R1", "-v2", new EsVersion(6, 4, 3), new EsVersion(6, 8, 6)), - FRANKFURT_R2("frankfurt-R2", "-v3", new EsVersion(7, 0, 1), new EsVersion(7, 6, 1)), - GUILIN_R1("guilin-R1", "-v4", new EsVersion(7,1,1), new EsVersion(7,6,1)), - HONOLULU_R1("honolulu-R1", "-v5", new EsVersion(7,1,1), new EsVersion(8,0,0), false), - ISTANBUL_R1("istanbul-R1", "-v6", new EsVersion(7,1,1), new EsVersion(8,0,0), false); + EL_ALTO("el alto", "_v1", new DatabaseVersion(2, 2, 0), new DatabaseVersion(2, 2, 0)), + FRANKFURT_R1("frankfurt-R1", "-v2", new DatabaseVersion(6, 4, 3), new DatabaseVersion(6, 8, 6)), + FRANKFURT_R2("frankfurt-R2", "-v3", new DatabaseVersion(7, 0, 1), new DatabaseVersion(7, 6, 1)), + GUILIN_R1("guilin-R1", "-v4", new DatabaseVersion(7,1,1), new DatabaseVersion(7,6,1)), + HONOLULU_R1("honolulu-R1", "-v5", new DatabaseVersion(7,1,1), new DatabaseVersion(8,0,0), false), + ISTANBUL_R1("istanbul-R1", "-v6", new DatabaseVersion(7,1,1), new DatabaseVersion(8,0,0), false, + new DatabaseVersion(10,2,7), new DatabaseVersion(10,6,0), false); public static final Release CURRENT_RELEASE = Release.ISTANBUL_R1; private final String value; private final String dbSuffix; - private final EsVersion minDbVersion; - private final EsVersion maxDbVersion; + private final DatabaseVersion minDbVersion; + private final DatabaseVersion maxDbVersion; + private final DatabaseVersion minMariaDbVersion; + private final DatabaseVersion maxMariaDbVersion; private final boolean includeEndVersion; + private final boolean mariaDbIncludeEndVersion; + + private Release(String s, String dbsuffix, DatabaseVersion minDbVersion, DatabaseVersion maxDbVersion) { + this(s, dbsuffix, minDbVersion, maxDbVersion, true, null, null, false); + } + + private Release(String s, String dbsuffix, DatabaseVersion minDbVersion, DatabaseVersion maxDbVersion, + boolean includeEnd) { + this(s, dbsuffix, minDbVersion, maxDbVersion, includeEnd, null, null, false); + } - private Release(String s, String dbsuffix, EsVersion minDbVersion, EsVersion maxDbVersion) { - this(s, dbsuffix, minDbVersion, maxDbVersion, true); - } - private Release(String s, String dbsuffix, EsVersion minDbVersion, EsVersion maxDbVersion, boolean includeEnd) { + private Release(String s, String dbsuffix, DatabaseVersion minDbVersion, DatabaseVersion maxDbVersion, + boolean includeEnd, DatabaseVersion minMariaDbVersion, DatabaseVersion maxMariaDbVersion, + boolean mariaDbIncludeEnd) { this.value = s; this.dbSuffix = dbsuffix; this.minDbVersion = minDbVersion; this.maxDbVersion = maxDbVersion; this.includeEndVersion = includeEnd; + this.minMariaDbVersion = minMariaDbVersion; + this.maxMariaDbVersion = maxMariaDbVersion; + this.mariaDbIncludeEndVersion = mariaDbIncludeEnd; } @Override @@ -94,16 +111,24 @@ public enum Release { return this.dbSuffix; } - public EsVersion getDBVersion() { + public DatabaseVersion getDBVersion() { return this.minDbVersion; } - public boolean isDbInRange(EsVersion dbVersion) { - if(this.includeEndVersion) { - return dbVersion.isNewerOrEqualThan(minDbVersion) && dbVersion.isOlderOrEqualThan(maxDbVersion); - } - else { - return dbVersion.isNewerOrEqualThan(minDbVersion) && dbVersion.isOlderThan(maxDbVersion); + public boolean isDbInRange(DatabaseVersion dbVersion, SdnrDbType type) { + if (type == SdnrDbType.ELASTICSEARCH) { + if (this.includeEndVersion) { + return dbVersion.isNewerOrEqualThan(minDbVersion) && dbVersion.isOlderOrEqualThan(maxDbVersion); + } else { + return dbVersion.isNewerOrEqualThan(minDbVersion) && dbVersion.isOlderThan(maxDbVersion); + } + } else { + if (this.mariaDbIncludeEndVersion) { + return dbVersion.isNewerOrEqualThan(minMariaDbVersion) + && dbVersion.isOlderOrEqualThan(maxMariaDbVersion); + } else { + return dbVersion.isNewerOrEqualThan(minMariaDbVersion) && dbVersion.isOlderThan(maxMariaDbVersion); + } } } } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/ReleaseGroup.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/ReleaseGroup.java index 664dce677..58e8d2055 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/ReleaseGroup.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/data/ReleaseGroup.java @@ -23,7 +23,9 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data; import java.util.ArrayList; import java.util.List; -import org.onap.ccsdk.features.sdnr.wt.common.database.data.EsVersion; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; + /** * @author Michael Dürre @@ -51,10 +53,10 @@ public enum ReleaseGroup { * @param dbVersion * @return */ - public Release getLatestCompatibleRelease(EsVersion dbVersion) { + public Release getLatestCompatibleRelease(DatabaseVersion dbVersion, SdnrDbType type) { Release match = null; for (Release r : this.releases) { - if (r.isDbInRange(dbVersion)) { + if (r.isDbInRange(dbVersion, type)) { match = r; } } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/database/ElasticsearchDataMigrationProvider.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/database/ElasticsearchDataMigrationProvider.java new file mode 100644 index 000000000..3cfa8c17f --- /dev/null +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/database/ElasticsearchDataMigrationProvider.java @@ -0,0 +1,465 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : ccsdk features + * ================================================================================ + * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property. + * All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + */ +package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import org.json.JSONObject; +import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; +import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit; +import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult; +import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList; +import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateAliasRequest; +import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateIndexRequest; +import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteAliasRequest; +import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteIndexRequest; +import org.onap.ccsdk.features.sdnr.wt.common.database.responses.AcknowledgedResponse; +import org.onap.ccsdk.features.sdnr.wt.common.database.responses.GetInfoResponse; +import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListAliasesResponse; +import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListIndicesResponse; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.DataMigrationProviderService; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ElasticsearchDataMigrationProvider implements DataMigrationProviderService { + + + private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDataMigrationProvider.class); + private final HtDatabaseClient dbClient; + + public ElasticsearchDataMigrationProvider(String url, String username, String password, boolean trustAll, + long timeoutms) throws Exception { + dbClient = HtDatabaseClient.getClient(new HostInfo[] {HostInfo.parse(url)}, username, password, trustAll, + true, timeoutms); + } + + @Override + public DataMigrationReport importData(String filename, boolean dryrun) throws Exception { + return this.importData(filename, dryrun, Release.CURRENT_RELEASE); + } + + @Override + public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception { + DataMigrationReport report = new DataMigrationReport(); + File file = new File(filename); + if (!file.exists()) { + if (dryrun) { + report.error("file %s not found", filename); + return report; + } + throw new FileNotFoundException(filename); + } + DataContainer container = null; + try { + container = DataContainer.load(file); + } catch (Exception e) { + if (dryrun) { + report.error("problem loading file %s: %s", filename, e.getMessage()); + return report; + } + throw new Exception("problem loading file " + filename, e); + } + ReleaseInformation ri = ReleaseInformation.getInstance(forRelease); + SearchHitConverter converter; + Set<ComponentName> components = ri.getComponents(); + //for all db components of dest architecture + for (ComponentName component : components) { + //convert to ComponentData for current release with existing ComponentData of the container + converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component); + if (converter == null) { + continue; + } + ComponentData data = converter.convert(container); + if (data != null) { + String indexName = ri.getAlias(component); + String dataTypeName = ri.getDataType(component); + if (dryrun) { + report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName); + } else { + LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName); + } + for (SearchHit item : data) { + if (!dryrun) { + String id = this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(), + item.getSourceAsString(), true); + if (!item.getId().equals(id)) { + LOG.warn("entry for {} with original id {} was written with another id {}", + component.getValue(), item.getId(), id); + } + } + } + } else { + if (dryrun) { + report.error("unable to convert data for " + component.getValue() + " from version " + + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n"); + } else { + LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(), + container.getRelease().getValue(), forRelease.getValue()); + } + } + } + LOG.info("import of {} completed", filename); + if (dryrun) { + report.log("import of %s completed", filename); + } + report.setCompleted(true); + return report; + } + + + /** + * export data if file exists .1 (.n) will be created + * + */ + @Override + public DataMigrationReport exportData(String filename) { + DataMigrationReport report = new DataMigrationReport(); + + DataContainer container = new DataContainer(); + + filename = this.checkFilenameForWrite(filename); + LOG.info("output will be written to {}", filename); + //autodetect version + Release dbRelease = this.autoDetectRelease(); + if (dbRelease == null) { + report.error("unbable to detect db release. is database initialized?"); + return report; + } + ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease); + boolean componentsSucceeded = true; + for (ComponentName c : ri.getComponents()) { + ComponentData data = new ComponentData(c); + SearchResult<SearchHit> result = this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false); + data.addAll(result.getHits()); + container.addComponent(c, data); + } + try { + Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8); + report.setCompleted(componentsSucceeded); + } catch (IOException e) { + LOG.warn("problem writing data to {}: {}", filename, e); + } + return report; + } + + private String checkFilenameForWrite(String filename) { + File f = new File(filename); + if (!f.exists()) { + return filename; + } + return this.checkFilenameForWrite(filename, 0); + } + + private String checkFilenameForWrite(String filename, int apdx) { + File f = new File(String.format("$s.$d", filename, apdx)); + if (!f.exists()) { + return filename; + } + return this.checkFilenameForWrite(filename, apdx + 1); + } + + @Override + public Release getCurrentVersion() { + return Release.CURRENT_RELEASE; + } + + + @Override + public Release autoDetectRelease() { + DatabaseVersion dbVersion = this.readActualVersion(); + AliasesEntryList aliases = this.readAliases(); + IndicesEntryList indices = this.readIndices(); + if (indices == null) { + return null; + } + List<Release> foundReleases = new ArrayList<>(); + //if there are active aliases reduce indices to the active ones + if (aliases != null && aliases.size() > 0) { + indices = indices.subList(aliases.getLinkedIndices()); + } + for (Release r : Release.values()) { + if (r.isDbInRange(dbVersion, SdnrDbType.ELASTICSEARCH)) { + ReleaseInformation ri = ReleaseInformation.getInstance(r); + if (ri != null && ri.containsIndices(indices)) { + foundReleases.add(r); + } + } + } + if (foundReleases.size() == 1) { + return foundReleases.get(0); + } + LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(), + foundReleases); + return null; + } + + private DatabaseVersion readActualVersion() { + try { + GetInfoResponse response = this.dbClient.getInfo(); + return response.getVersion(); + } catch (Exception e) { + LOG.warn(e.getMessage()); + } + return null; + } + + private AliasesEntryList readAliases() { + AliasesEntryList entries = null; + try { + ListAliasesResponse response = this.dbClient.getAliases(); + entries = response.getEntries(); + } catch (ParseException | IOException e) { + LOG.error(e.getMessage()); + } + return entries; + } + + private IndicesEntryList readIndices() { + IndicesEntryList entries = null; + try { + ListIndicesResponse response = this.dbClient.getIndices(); + entries = response.getEntries(); + } catch (ParseException | IOException e) { + LOG.error(e.getMessage()); + } + return entries; + } + + + @Override + public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate, + long timeoutms) { + if (timeoutms > 0) { + this.dbClient.waitForYellowStatus(timeoutms); + } + DatabaseVersion dbVersion = this.readActualVersion(); + if (dbVersion == null) { + return false; + } + LOG.info("detected database version {}", dbVersion); + if (release == null) { + release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.ELASTICSEARCH); + if (release == null) { + LOG.warn("unable to autodetect release for this database version for release {}", + ReleaseGroup.CURRENT_RELEASE.name()); + return false; + } + LOG.info("autodetect release {}", release); + } + if (!release.isDbInRange(dbVersion, SdnrDbType.ELASTICSEARCH)) { + LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release); + return false; + } + if (forceRecreate) { + this.clearDatabase(release, dbPrefix, 0); + } + ReleaseInformation ri = ReleaseInformation.getInstance(release); + AliasesEntryList aliases = this.readAliases(); + IndicesEntryList indices = this.readIndices(); + if (aliases == null || indices == null) { + return false; + } + AcknowledgedResponse response = null; + if (!ri.runPreInitCommands(this.dbClient)) { + return false; + } + for (ComponentName component : ri.getComponents()) { + try { + if (ri.hasOwnDbIndex(component)) { + //check if index already exists + String indexName = ri.getIndex(component, dbPrefix); + String aliasName = ri.getAlias(component, dbPrefix); + if (indices.findByIndex(indexName) == null) { + LOG.info("creating index for {}", component); + CreateIndexRequest request = new CreateIndexRequest(ri.getIndex(component, dbPrefix)); + request.mappings(new JSONObject(ri.getDatabaseMapping(component))); + request.settings(new JSONObject(ri.getDatabaseSettings(component, numShards, numReplicas))); + response = this.dbClient.createIndex(request); + LOG.info(response.isAcknowledged() ? "succeeded" : "failed"); + } else { + LOG.info("index {} for {} already exists", indexName, component); + } + //check if alias already exists + if (aliases.findByAlias(aliasName) == null) { + LOG.info("creating alias for {}", component); + response = this.dbClient.createAlias(new CreateAliasRequest(indexName, aliasName)); + LOG.info(response.isAcknowledged() ? "succeeded" : "failed"); + } else { + LOG.info("alias {} for index {} for {} already exists", aliasName, indexName, component); + } + } + } catch (IOException e) { + LOG.error(e.getMessage()); + return false; + } + } + if (!ri.runPostInitCommands(this.dbClient)) { + return false; + } + return true; + } + + @Override + public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) { + + if (timeoutms > 0) { + this.dbClient.waitForYellowStatus(timeoutms); + } + //check aliases + AliasesEntryList entries = this.readAliases(); + IndicesEntryList entries2 = this.readIndices(); + if (entries == null) { + return false; + } + if (release == null) { + DatabaseVersion dbVersion = this.readActualVersion(); + if (dbVersion == null) { + return false; + } + LOG.info("detected database version {}", dbVersion); + release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.ELASTICSEARCH); + if (release == null) { + LOG.warn("unable to autodetect release for this database version for release {}", + ReleaseGroup.CURRENT_RELEASE.name()); + return false; + } + LOG.info("autodetect release {}", release); + } + ReleaseInformation ri = ReleaseInformation.getInstance(release); + AcknowledgedResponse response; + if (entries.size() <= 0) { + LOG.info("no aliases to clear"); + } else { + //check for every component of release if alias exists + for (ComponentName component : ri.getComponents()) { + String aliasToDelete = ri.getAlias(component, dbPrefix); + AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete); + if (entryToDelete != null) { + try { + LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex()); + response = this.dbClient.deleteAlias( + new DeleteAliasRequest(entryToDelete.getIndex(), entryToDelete.getAlias())); + LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed"); + } catch (IOException e) { + LOG.error(e.getMessage()); + return false; + } + } else { + //try to find malformed typed index with alias name + IndicesEntry entry2ToDelete = entries2.findByIndex(aliasToDelete); + if (entry2ToDelete != null) { + try { + LOG.info("deleting index {}", entry2ToDelete.getName()); + response = this.dbClient.deleteIndex(new DeleteIndexRequest(entry2ToDelete.getName())); + LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed"); + } catch (IOException e) { + LOG.error(e.getMessage()); + return false; + } + } + } + } + } + if (entries2 == null) { + return false; + } + if (entries2.size() <= 0) { + LOG.info("no indices to clear"); + } else { + //check for every component of release if index exists + for (ComponentName component : ri.getComponents()) { + String indexToDelete = ri.getIndex(component, dbPrefix); + IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete); + if (entryToDelete != null) { + try { + LOG.info("deleting index {}", entryToDelete.getName()); + response = this.dbClient.deleteIndex(new DeleteIndexRequest(entryToDelete.getName())); + LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed"); + } catch (IOException e) { + LOG.error(e.getMessage()); + return false; + } + } + } + } + + return true; + } + + /** + * @param timeoutms + * @return + */ + public boolean clearCompleteDatabase(long timeoutms) { + if (timeoutms > 0) { + this.dbClient.waitForYellowStatus(timeoutms); + } + //check aliases and indices + AliasesEntryList aliases = this.readAliases(); + IndicesEntryList indices = this.readIndices(); + if (aliases == null || indices == null) { + return false; + } + for (AliasesEntry alias : aliases) { + try { + LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex()); + this.dbClient.deleteAlias(new DeleteAliasRequest(alias.getIndex(), alias.getAlias())); + } catch (IOException e) { + LOG.error("problem deleting alias {}: {}", alias.getAlias(), e); + return false; + } + } + for (IndicesEntry index : indices) { + try { + LOG.info("deleting index {}", index.getName()); + this.dbClient.deleteIndex(new DeleteIndexRequest(index.getName())); + } catch (IOException e) { + LOG.error("problem deleting index {}: {}", index.getName(), e); + return false; + } + } + return true; + } + +} diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/database/MariaDbDataMigrationProvider.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/database/MariaDbDataMigrationProvider.java new file mode 100644 index 000000000..d46fc4e2e --- /dev/null +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/database/MariaDbDataMigrationProvider.java @@ -0,0 +1,433 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : ccsdk features + * ================================================================================ + * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property. + * All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + */ +package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.SQLException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit; +import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry; +import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.DataMigrationProviderService; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MariaDbDataMigrationProvider implements DataMigrationProviderService { + + + private static final Logger LOG = LoggerFactory.getLogger(MariaDbDataMigrationProvider.class); + private static final SdnrDbType DBTYPE = SdnrDbType.MARIADB; + private final SqlDBClient dbClient; + + public MariaDbDataMigrationProvider(String url, String username, String password, boolean trustAll, + long timeoutms) throws Exception { + dbClient = new SqlDBClient(url, username, password); + } + + @Override + public DataMigrationReport importData(String filename, boolean dryrun) throws Exception { + return this.importData(filename, dryrun, Release.CURRENT_RELEASE); + } + + @Override + public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception { + DataMigrationReport report = new DataMigrationReport(); + File file = new File(filename); + if (!file.exists()) { + if (dryrun) { + report.error("file %s not found", filename); + return report; + } + throw new FileNotFoundException(filename); + } + DataContainer container = null; + try { + container = DataContainer.load(file); + } catch (Exception e) { + if (dryrun) { + report.error("problem loading file %s: %s", filename, e.getMessage()); + return report; + } + throw new Exception("problem loading file " + filename, e); + } + ReleaseInformation ri = ReleaseInformation.getInstance(forRelease); + SearchHitConverter converter; + Set<ComponentName> components = ri.getComponents(); + //for all db components of dest architecture + for (ComponentName component : components) { + //convert to ComponentData for current release with existing ComponentData of the container + converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component); + if (converter == null) { + continue; + } + ComponentData data = converter.convert(container); + if (data != null) { + String indexName = ri.getAlias(component); + String dataTypeName = ri.getDataType(component); + if (dryrun) { + report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName); + } else { + LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName); + } + for (SearchHit item : data) { + if (!dryrun) { + String id = null;//this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(), item.getSourceAsString(), true); + if (!item.getId().equals(id)) { + LOG.warn("entry for {} with original id {} was written with another id {}", + component.getValue(), item.getId(), id); + } + } + } + } else { + if (dryrun) { + report.error("unable to convert data for " + component.getValue() + " from version " + + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n"); + } else { + LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(), + container.getRelease().getValue(), forRelease.getValue()); + } + } + } + LOG.info("import of {} completed", filename); + if (dryrun) { + report.log("import of %s completed", filename); + } + report.setCompleted(true); + return report; + } + + + /** + * export data if file exists .1 (.n) will be created + * + */ + @Override + public DataMigrationReport exportData(String filename) { + DataMigrationReport report = new DataMigrationReport(); + + DataContainer container = new DataContainer(); + + filename = this.checkFilenameForWrite(filename); + LOG.info("output will be written to {}", filename); + //autodetect version + Release dbRelease = this.autoDetectRelease(); + if (dbRelease == null) { + report.error("unbable to detect db release. is database initialized?"); + return report; + } + ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease); + boolean componentsSucceeded = true; + for (ComponentName c : ri.getComponents()) { + ComponentData data = new ComponentData(c); + SearchResult<SearchHit> result = null;//this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false); + data.addAll(result.getHits()); + container.addComponent(c, data); + } + try { + Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8); + report.setCompleted(componentsSucceeded); + } catch (IOException e) { + LOG.warn("problem writing data to {}: {}", filename, e); + } + return report; + } + + private String checkFilenameForWrite(String filename) { + File f = new File(filename); + if (!f.exists()) { + return filename; + } + return this.checkFilenameForWrite(filename, 0); + } + + private String checkFilenameForWrite(String filename, int apdx) { + File f = new File(String.format("$s.$d", filename, apdx)); + if (!f.exists()) { + return filename; + } + return this.checkFilenameForWrite(filename, apdx + 1); + } + + @Override + public Release getCurrentVersion() { + return Release.CURRENT_RELEASE; + } + + + @Override + public Release autoDetectRelease() { + DatabaseVersion dbVersion; + try { + dbVersion = this.dbClient.readActualVersion(); + } catch (SQLException | ParseException e) { + LOG.error("unable to detect db version", e); + return null; + } + AliasesEntryList aliases = this.dbClient.readViews(); + IndicesEntryList indices = this.dbClient.readTables(); + if (indices == null) { + return null; + } + List<Release> foundReleases = new ArrayList<>(); + //if there are active aliases reduce indices to the active ones + if (aliases != null && aliases.size() > 0) { + indices = indices.subList(aliases.getLinkedIndices()); + } + for (Release r : Release.values()) { + if (r.isDbInRange(dbVersion, SdnrDbType.MARIADB)) { + ReleaseInformation ri = ReleaseInformation.getInstance(r); + if (ri != null && ri.containsIndices(indices)) { + foundReleases.add(r); + } + } + } + if (foundReleases.size() == 1) { + return foundReleases.get(0); + } + LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(), + foundReleases); + return null; + } + + @Override + public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate, + long timeoutms) { + if (timeoutms > 0) { + this.dbClient.waitForYellowStatus(timeoutms); + } + DatabaseVersion dbVersion; + try { + dbVersion = this.dbClient.readActualVersion(); + } catch (SQLException | ParseException e1) { + LOG.error("unable to detect db version", e1); + return false; + } + if (dbVersion == null) { + return false; + } + LOG.info("detected database version {}", dbVersion); + if (release == null) { + release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.MARIADB); + if (release == null) { + LOG.warn("unable to autodetect release for this database version for release {}", + ReleaseGroup.CURRENT_RELEASE.name()); + return false; + } + LOG.info("autodetect release {}", release); + } + if (!release.isDbInRange(dbVersion, SdnrDbType.MARIADB)) { + LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release); + return false; + } + if (forceRecreate) { + this.clearDatabase(release, dbPrefix, 0); + } + ReleaseInformation ri = ReleaseInformation.getInstance(release); + AliasesEntryList views = this.dbClient.readViews(); + IndicesEntryList indices = this.dbClient.readTables(); + if (views == null || indices == null) { + return false; + } + boolean response = false; + if (!ri.runPreInitCommands(this.dbClient)) { + return false; + } + for (ComponentName component : ri.getComponents()) { + try { + if (ri.hasOwnDbIndex(component)) { + //check if index already exists + String tableName = ri.getIndex(component, dbPrefix); + String viewName = ri.getAlias(component, dbPrefix); + if (indices.findByIndex(tableName) == null) { + LOG.info("creating index for {}", component); + response = this.dbClient.createTable(ri.getIndex(component, dbPrefix), ri.getDatabaseMapping(component, DBTYPE)); + LOG.info(response ? "succeeded" : "failed"); + } else { + LOG.info("index {} for {} already exists", tableName, component); + } + //check if alias already exists + if (views.findByAlias(viewName) == null) { + LOG.info("creating alias for {}", component); + response = this.dbClient.createView(tableName, viewName); + LOG.info(response ? "succeeded" : "failed"); + } else { + LOG.info("view {} for table {} for {} already exists", viewName, tableName, component); + } + } + } catch (SQLException e) { + LOG.error(e.getMessage()); + return false; + } + } + if (!ri.runPostInitCommands(this.dbClient)) { + return false; + } + return true; + } + + @Override + public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) { + + if (timeoutms > 0) { + this.dbClient.waitForYellowStatus(timeoutms); + } + //check aliases + AliasesEntryList entries = this.dbClient.readViews(); + IndicesEntryList entries2 = this.dbClient.readTables(); + if (entries == null) { + return false; + } + if (release == null) { + DatabaseVersion dbVersion; + try { + dbVersion = this.dbClient.readActualVersion(); + } catch (SQLException | ParseException e) { + LOG.error("unable to detect db version", e); + return false; + } + LOG.info("detected database version {}", dbVersion); + release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.MARIADB); + if (release == null) { + LOG.warn("unable to autodetect release for this database version for release {}", + ReleaseGroup.CURRENT_RELEASE.name()); + return false; + } + LOG.info("autodetect release {}", release); + } + ReleaseInformation ri = ReleaseInformation.getInstance(release); + boolean response; + if (entries.size() <= 0) { + LOG.info("no aliases to clear"); + } else { + //check for every component of release if alias exists + for (ComponentName component : ri.getComponents()) { + String aliasToDelete = ri.getAlias(component, dbPrefix); + AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete); + if (entryToDelete != null) { + try { + LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex()); + response = this.dbClient.deleteView(entryToDelete.getAlias()); + LOG.info(response ? "succeeded" : "failed"); + } catch (SQLException e) { + LOG.error(e.getMessage()); + return false; + } + } else { + //try to find malformed typed index with alias name + IndicesEntry entry2ToDelete = entries2.findByIndex(aliasToDelete); + if (entry2ToDelete != null) { + try { + LOG.info("deleting index {}", entry2ToDelete.getName()); + response = this.dbClient.deleteTable(entry2ToDelete.getName()); + LOG.info(response ? "succeeded" : "failed"); + } catch (SQLException e) { + LOG.error(e.getMessage()); + return false; + } + } + } + } + } + if (entries2 == null) { + return false; + } + if (entries2.size() <= 0) { + LOG.info("no indices to clear"); + } else { + //check for every component of release if index exists + for (ComponentName component : ri.getComponents()) { + String indexToDelete = ri.getIndex(component, dbPrefix); + IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete); + if (entryToDelete != null) { + try { + LOG.info("deleting index {}", entryToDelete.getName()); + response = this.dbClient.deleteTable(entryToDelete.getName()); + LOG.info(response ? "succeeded" : "failed"); + } catch (SQLException e) { + LOG.error(e.getMessage()); + return false; + } + } + } + } + + return true; + } + + /** + * @param timeoutms + * @return + */ + public boolean clearCompleteDatabase(long timeoutms) { + if (timeoutms > 0) { + this.dbClient.waitForYellowStatus(timeoutms); + } + //check aliases and indices + AliasesEntryList aliases = this.dbClient.readViews(); + IndicesEntryList indices = this.dbClient.readTables(); + if (aliases == null || indices == null) { + return false; + } + for (AliasesEntry alias : aliases) { + try { + LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex()); + this.dbClient.deleteView(alias.getAlias()); + } catch (SQLException e) { + LOG.error("problem deleting alias {}: {}", alias.getAlias(), e); + return false; + } + } + for (IndicesEntry index : indices) { + try { + LOG.info("deleting index {}", index.getName()); + this.dbClient.deleteTable(index.getName()); + } catch (SQLException e) { + LOG.error("problem deleting index {}: {}", index.getName(), e); + return false; + } + } + return true; + } + +} diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java index 739afaf0e..4165080b5 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/elalto/ElAltoReleaseInformation.java @@ -23,13 +23,14 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.elalto; import java.util.HashMap; import java.util.Map; +import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; -import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; -import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; public class ElAltoReleaseInformation extends ReleaseInformation { @@ -96,13 +97,22 @@ public class ElAltoReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + public boolean runPreInitCommands(HtDatabaseClient dbClient) { return true; } @Override - protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + public boolean runPostInitCommands(HtDatabaseClient dbClient) { return true; } + @Override + public boolean runPreInitCommands(SqlDBClient dbClient) { + return false; + } + + @Override + public boolean runPostInitCommands(SqlDBClient dbClient) { + return false; + } } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java index 7b8c40bca..83ad6c4e7 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformation.java @@ -24,16 +24,17 @@ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.frankfurt; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; +import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest; +import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; -import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest; -import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse; -import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; public class FrankfurtReleaseInformation extends ReleaseInformation { @@ -82,7 +83,7 @@ public class FrankfurtReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + public boolean runPreInitCommands(HtDatabaseClient dbClient) { ClusterSettingsResponse response = null; try { @@ -94,8 +95,17 @@ public class FrankfurtReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + public boolean runPostInitCommands(HtDatabaseClient dbClient) { return true; } + @Override + public boolean runPreInitCommands(SqlDBClient dbClient) { + return false; + } + + @Override + public boolean runPostInitCommands(SqlDBClient dbClient) { + return false; + } } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformationR2.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformationR2.java index f972777b9..8882a8b68 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformationR2.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/frankfurt/FrankfurtReleaseInformationR2.java @@ -27,6 +27,7 @@ import java.util.Map; import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest; import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; @@ -59,10 +60,10 @@ public class FrankfurtReleaseInformationR2 extends ReleaseInformation { "{\"date\": {\"type\": \"keyword\"},\"model-identifier\": {\"type\": \"keyword\"},\"manufacturer-identifier\": {\"type\": \"keyword\"},\"type-name\": {\"type\": \"keyword\"},\"description\": {\"type\": \"keyword\"},\"uuid\": {\"type\": \"keyword\"},\"version\": {\"type\": \"keyword\"},\"parent-uuid\": {\"type\": \"keyword\"},\"contained-holder\": {\"type\": \"keyword\"},\"node-id\": {\"type\": \"keyword\"},\"tree-level\": {\"type\": \"long\"},\"part-type-id\": {\"type\": \"keyword\"},\"serial\": {\"type\": \"keyword\"}}")); map.put(ComponentName.HISTORICAL_PERFORMANCE_15M, new DatabaseInfo7("historicalperformance15min", "historicalperformance15min", - "{\"node-name\":{\"type\": \"keyword\"},\"timestamp\":{\"type\": \"date\"},\"suspect-interval-flag\":{\"type\":\"boolean\"},\"scanner-id\":{\"type\": \"keyword\"},\"uuid-interface\":{\"type\": \"keyword\"},\"layer-protocol-name\":{\"type\": \"keyword\"},\"granularity-period\":{\"type\": \"keyword\"},\"radio-signal-id\":{\"type\": \"keyword\"}}")); + "{\"node-name\":{\"type\": \"keyword\"},\"time-stamp\":{\"type\": \"date\"},\"suspect-interval-flag\":{\"type\":\"boolean\"},\"scanner-id\":{\"type\": \"keyword\"},\"uuid-interface\":{\"type\": \"keyword\"},\"layer-protocol-name\":{\"type\": \"keyword\"},\"granularity-period\":{\"type\": \"keyword\"},\"radio-signal-id\":{\"type\": \"keyword\"}}")); map.put(ComponentName.HISTORICAL_PERFORMANCE_24H, new DatabaseInfo7("historicalperformance24h", "historicalperformance24h", - "{\"node-name\":{\"type\": \"keyword\"},\"timestamp\":{\"type\": \"date\"},\"suspect-interval-flag\":{\"type\":\"boolean\"},\"scanner-id\":{\"type\": \"keyword\"},\"uuid-interface\":{\"type\": \"keyword\"},\"layer-protocol-name\":{\"type\": \"keyword\"},\"granularity-period\":{\"type\": \"keyword\"},\"radio-signal-id\":{\"type\": \"keyword\"}}")); + "{\"node-name\":{\"type\": \"keyword\"},\"time-stamp\":{\"type\": \"date\"},\"suspect-interval-flag\":{\"type\":\"boolean\"},\"scanner-id\":{\"type\": \"keyword\"},\"uuid-interface\":{\"type\": \"keyword\"},\"layer-protocol-name\":{\"type\": \"keyword\"},\"granularity-period\":{\"type\": \"keyword\"},\"radio-signal-id\":{\"type\": \"keyword\"}}")); map.put(ComponentName.REQUIRED_NETWORKELEMENT, new DatabaseInfo7("networkelement-connection", "networkelement-connection", "{\"node-id\": {\"type\": \"keyword\"},\"host\": {\"type\": \"keyword\"},\"port\": {\"type\": \"long\"},\"username\": {\"type\": \"keyword\"},\"password\": {\"type\": \"keyword\"},\"core-model-capability\": {\"type\": \"keyword\"},\"device-type\": {\"type\": \"keyword\"},\"is-required\": {\"type\": \"boolean\"},\"status\": {\"type\": \"keyword\"}}")); @@ -83,7 +84,7 @@ public class FrankfurtReleaseInformationR2 extends ReleaseInformation { } @Override - protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + public boolean runPreInitCommands(HtDatabaseClient dbClient) { ClusterSettingsResponse response = null; try { @@ -95,8 +96,18 @@ public class FrankfurtReleaseInformationR2 extends ReleaseInformation { } @Override - protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + public boolean runPostInitCommands(HtDatabaseClient dbClient) { return true; } + @Override + public boolean runPreInitCommands(SqlDBClient dbClient) { + return false; + } + + @Override + public boolean runPostInitCommands(SqlDBClient dbClient) { + return false; + } + } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java index e057d82f2..5efaf9331 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/guilin/GuilinReleaseInformation.java @@ -25,6 +25,7 @@ import java.io.IOException; import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest; import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter; @@ -51,7 +52,7 @@ public class GuilinReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + public boolean runPreInitCommands(HtDatabaseClient dbClient) { ClusterSettingsResponse response = null; try { response = dbClient.setupClusterSettings(new ClusterSettingsRequest(false).maxCompilationsPerMinute(400)); @@ -62,8 +63,18 @@ public class GuilinReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + public boolean runPostInitCommands(HtDatabaseClient dbClient) { return true; } + @Override + public boolean runPreInitCommands(SqlDBClient dbClient) { + return false; + } + + @Override + public boolean runPostInitCommands(SqlDBClient dbClient) { + return false; + } + } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/honolulu/HonoluluReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/honolulu/HonoluluReleaseInformation.java index 41f8175db..148c004a6 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/honolulu/HonoluluReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/honolulu/HonoluluReleaseInformation.java @@ -26,6 +26,7 @@ import java.util.Map; import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest; import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; @@ -61,7 +62,7 @@ public class HonoluluReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPreInitCommands(HtDatabaseClient dbClient) { + public boolean runPreInitCommands(HtDatabaseClient dbClient) { ClusterSettingsResponse response = null; try { response = dbClient.setupClusterSettings(new ClusterSettingsRequest(false).maxCompilationsPerMinute(400)); @@ -72,8 +73,18 @@ public class HonoluluReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + public boolean runPostInitCommands(HtDatabaseClient dbClient) { return true; } + @Override + public boolean runPreInitCommands(SqlDBClient dbClient) { + return false; + } + + @Override + public boolean runPostInitCommands(SqlDBClient dbClient) { + return false; + } + } diff --git a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/istanbul/IstanbulReleaseInformation.java b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/istanbul/IstanbulReleaseInformation.java index 67200bb2f..0dcd3ae3c 100644 --- a/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/istanbul/IstanbulReleaseInformation.java +++ b/sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/istanbul/IstanbulReleaseInformation.java @@ -21,33 +21,154 @@ */ package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.istanbul; -import java.io.IOException; +import java.util.HashMap; import java.util.Map; import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient; -import org.onap.ccsdk.features.sdnr.wt.common.database.requests.ClusterSettingsRequest; -import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ClusterSettingsResponse; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DatabaseInfo7; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.KeepDataSearchHitConverter; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.MariaDBTableInfo; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.honolulu.HonoluluReleaseInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Entity; public class IstanbulReleaseInformation extends ReleaseInformation { - private final Logger LOG = LoggerFactory.getLogger(IstanbulReleaseInformation.class); + private static final String TIMEZONE_TYPE_FORMAT = + "CHAR(6) DEFAULT NULL CHECK (`%s` regexp '^[+-]\\\\d\\\\d:\\\\d\\\\d$')"; + private static final String TABLENAME_CONTROLLER_FORMAT = "controller%s"; + private static final String TABLEMAPPING_CONTROLLER = + "`id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,`desc` VARCHAR(255) CHARACTER SET utf8 ,primary key(id)"; + private static final String TABLEMAPPING_CONNECTIONLOG_FORMAT = "`id` int(11) NOT NULL AUTO_INCREMENT,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + "`timestamp` DATETIME(3) ,\n" + + "`status` VARCHAR(100) CHARACTER SET utf8 ,\n" + "`node-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_EVENTLOG_FORMAT = "`id` int(11) NOT NULL AUTO_INCREMENT,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + + "`source-type` VARCHAR(100) CHARACTER SET utf8 ,\n" + "`object-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`timestamp` DATETIME(3) ,\n" + "`timestamp-tz` " + String.format(TIMEZONE_TYPE_FORMAT, "timestamp-tz") + + " ,\n" + "`node-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`counter` INTEGER ,\n" + + "`attribute-name` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`new-value` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_FAULTCURRENT_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + + "`object-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`timestamp` DATETIME(3) ,\n" + "`timestamp-tz` " + + String.format(TIMEZONE_TYPE_FORMAT, "timestamp-tz") + " ,\n" + + "`node-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`counter` INTEGER ,\n" + + "`severity` VARCHAR(100) CHARACTER SET utf8 ,\n" + "`problem` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_FAULTLOG_FORMAT = "`id` int(11) NOT NULL AUTO_INCREMENT,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + + "`source-type` VARCHAR(100) CHARACTER SET utf8 ,\n" + "`object-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`timestamp` DATETIME(3) ,\n" + "`timestamp-tz` " + String.format(TIMEZONE_TYPE_FORMAT, "timestamp-tz") + + " ,\n" + "`node-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`counter` INTEGER ,\n" + + "`severity` VARCHAR(100) CHARACTER SET utf8 ,\n" + "`problem` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_GUICUTTHROUGH_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + + "`name` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`weburi` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_HISTORICALPM15M_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + "`performance-data` JSON ,\n" + + "`granularity-period` VARCHAR(100) CHARACTER SET utf8 ,\n" + "`time-stamp` DATETIME(3) ,\n" + + "`time-stamp-tz` " + String.format(TIMEZONE_TYPE_FORMAT, "time-stamp-tz") + " ,\n" + + "`node-name` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`uuid-interface` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`layer-protocol-name` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`radio-signal-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`suspect-interval-flag` BOOLEAN ,\n" + + "`scanner-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_HISTORICALPM24H_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + "`performance-data` JSON ,\n" + + "`granularity-period` VARCHAR(100) CHARACTER SET utf8 ,\n" + "`time-stamp` DATETIME(3) ,\n" + + "`time-stamp-tz` " + String.format(TIMEZONE_TYPE_FORMAT, "time-stamp-tz") + " ,\n" + + "`node-name` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`uuid-interface` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`layer-protocol-name` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`radio-signal-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`suspect-interval-flag` BOOLEAN ,\n" + + "`scanner-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_INVENTORY_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + + "`version` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`type-name` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`date` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`description` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`node-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`uuid` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`part-type-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`model-identifier` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`manufacturer-identifier` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`tree-level` BIGINT ,\n" + + "`parent-uuid` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`contained-holder` JSON ,\n" + + "`serial` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`manufacturer-name` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_MAINTENANCE_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + "`active` BOOLEAN ,\n" + + "`node-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`description` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`problem` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`start` DATETIME(3) ,\n" + "`start-tz` " + + String.format(TIMEZONE_TYPE_FORMAT, "start-tz") + " ,\n" + "`end` DATETIME(3) ,\n" + "`end-tz` " + + String.format(TIMEZONE_TYPE_FORMAT, "end-tz") + " ,\n" + + "`object-id-ref` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_MEDIATORSERVER_FORMAT = + "`id` int(11) NOT NULL AUTO_INCREMENT,\n" + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + + "`name` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`url` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_NETWORKELEMENT_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + + "`password` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`host` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`port` BIGINT ,\n" + "`status` VARCHAR(100) CHARACTER SET utf8 ,\n" + + "`node-id` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`username` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`is-required` BOOLEAN ,\n" + "`core-model-capability` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "`device-type` VARCHAR(100) CHARACTER SET utf8 ,\n" + + "`device-function` VARCHAR(512) CHARACTER SET utf8 ,\n" + "`node-details` JSON ,\n" + + "`tls-key` VARCHAR(255) CHARACTER SET utf8 ,\n" + "`mount-method` VARCHAR(255) CHARACTER SET utf8 ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + private static final String TABLEMAPPING_USERDATA_FORMAT = "`id` VARCHAR(255) CHARACTER SET utf8 NOT NULL,\n" + + "`controller-id` VARCHAR(40) CHARACTER SET utf8 NOT NULL,\n" + "`value` JSON ,\n" + + "primary key(id),foreign key(`controller-id`) references `controller%s`(id)"; + + public IstanbulReleaseInformation() { - super(Release.ISTANBUL_R1,createDBMap()); + super(Release.ISTANBUL_R1, createDBMap(), createMariaDBMap(Release.ISTANBUL_R1.getDBSuffix())); + } + private static Map<ComponentName, MariaDBTableInfo> createMariaDBMap(String suffix) { + Map<ComponentName, MariaDBTableInfo> map = new HashMap<>(); + map.put(ComponentName.CONNECTIONLOG, + new MariaDBTableInfo(Entity.Connectionlog.getName(), TABLEMAPPING_CONNECTIONLOG_FORMAT)); + map.put(ComponentName.EVENTLOG, new MariaDBTableInfo(Entity.Eventlog.getName(), TABLEMAPPING_EVENTLOG_FORMAT)); + map.put(ComponentName.FAULTCURRENT, + new MariaDBTableInfo(Entity.Faultcurrent.getName(), TABLEMAPPING_FAULTCURRENT_FORMAT)); + map.put(ComponentName.FAULTLOG, new MariaDBTableInfo(Entity.Faultlog.getName(), TABLEMAPPING_FAULTLOG_FORMAT)); + map.put(ComponentName.GUICUTTHROUGH, + new MariaDBTableInfo(Entity.Guicutthrough.getName(), TABLEMAPPING_GUICUTTHROUGH_FORMAT)); + map.put(ComponentName.HISTORICAL_PERFORMANCE_15M, + new MariaDBTableInfo(Entity.Historicalperformance15min.getName(), TABLEMAPPING_HISTORICALPM15M_FORMAT)); + map.put(ComponentName.HISTORICAL_PERFORMANCE_24H, + new MariaDBTableInfo(Entity.Historicalperformance24h.getName(), TABLEMAPPING_HISTORICALPM24H_FORMAT)); + map.put(ComponentName.INVENTORY, + new MariaDBTableInfo(Entity.Inventoryequipment.getName(), TABLEMAPPING_INVENTORY_FORMAT)); + map.put(ComponentName.MAINTENANCE, + new MariaDBTableInfo(Entity.Maintenancemode.getName(), TABLEMAPPING_MAINTENANCE_FORMAT)); + map.put(ComponentName.MEDIATOR_SERVER, + new MariaDBTableInfo(Entity.MediatorServer.getName(), TABLEMAPPING_MEDIATORSERVER_FORMAT)); + map.put(ComponentName.REQUIRED_NETWORKELEMENT, + new MariaDBTableInfo(Entity.NetworkelementConnection.getName(), TABLEMAPPING_NETWORKELEMENT_FORMAT)); + map.put(ComponentName.USERDATA, new MariaDBTableInfo(Entity.Userdata.getName(), TABLEMAPPING_USERDATA_FORMAT)); + return map; } private static Map<ComponentName, DatabaseInfo> createDBMap() { - Map<ComponentName, DatabaseInfo> map= HonoluluReleaseInformation.createDBMap(); + Map<ComponentName, DatabaseInfo> map = HonoluluReleaseInformation.createDBMap(); map.put(ComponentName.USERDATA, new DatabaseInfo7("userdata", "userdata", "{}")); + map.put(ComponentName.REQUIRED_NETWORKELEMENT, new DatabaseInfo7("networkelement-connection", + "networkelement-connection", + "{\"node-id\": {\"type\": \"keyword\"},\"host\": {\"type\": \"keyword\"},\"port\": " + + "{\"type\": \"long\"},\"username\": {\"type\": \"keyword\"},\"password\": {\"type\": \"keyword\"}," + + "\"core-model-capability\": {\"type\": \"keyword\"},\"device-type\": {\"type\": \"keyword\"}," + + "\"device-function\": {\"type\": \"keyword\"},\"is-required\": {\"type\": \"boolean\"}," + + "\"status\": {\"type\": \"keyword\"},\"tls-key\": {\"type\": \"keyword\"}," + + "\"mount-method\": {\"type\":\"keyword\"}}")); return map; } @@ -60,18 +181,24 @@ public class IstanbulReleaseInformation extends ReleaseInformation { } @Override - protected boolean runPreInitCommands(HtDatabaseClient dbClient) { - ClusterSettingsResponse response = null; - try { - response = dbClient.setupClusterSettings(new ClusterSettingsRequest(false).maxCompilationsPerMinute(400)); - } catch (IOException e) { - LOG.warn("problem setting up cluster: {}", e); - } - return response == null ? false : response.isAcknowledged(); + public boolean runPreInitCommands(HtDatabaseClient dbClient) { + return true; + } + + @Override + public boolean runPostInitCommands(HtDatabaseClient dbClient) { + return true; + } + + @Override + public boolean runPreInitCommands(SqlDBClient dbClient) { + boolean success = dbClient.createTable( + String.format(TABLENAME_CONTROLLER_FORMAT, this.getReleas().getDBSuffix()), TABLEMAPPING_CONTROLLER); + return success; } @Override - protected boolean runPostInitCommands(HtDatabaseClient dbClient) { + public boolean runPostInitCommands(SqlDBClient dbClient) { return true; } diff --git a/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/MariaDBTestBase.java b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/MariaDBTestBase.java new file mode 100644 index 000000000..264f1203c --- /dev/null +++ b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/MariaDBTestBase.java @@ -0,0 +1,187 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : ccsdk features + * ================================================================================ + * Copyright (C) 2021 highstreet technologies GmbH Intellectual Property. + * All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + */ +package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup; + +import static org.junit.Assert.fail; +import java.io.File; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import org.onap.ccsdk.features.sdnr.wt.common.configuration.ConfigurationFileRepresentation; +import org.onap.ccsdk.features.sdnr.wt.common.configuration.subtypes.Section; +import org.onap.ccsdk.features.sdnr.wt.common.configuration.subtypes.Section.EnvGetter; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBConfig; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.data.SqlDBDataProvider; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.database.SqlDBMapper; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.database.SqlDBMapper.UnableToMapClassException; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionlogEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Entity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.EventlogEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultcurrentEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultlogEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GuicutthroughEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.InventoryEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.MaintenanceEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.MediatorServerEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnectionEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Pmdata15mEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Pmdata24hEntity; +import ch.vorburger.exec.ManagedProcessException; +import ch.vorburger.mariadb4j.DB; +import ch.vorburger.mariadb4j.DBConfigurationBuilder; + +public class MariaDBTestBase { + + private final SqlDBDataProvider dbProvider; + private final DB db; + private static final Map<String, String> envDefaultValues = initEnvDefaultValues(); + private static final String SDNRDBDATABASETEST="test"; + private static final String TESTPROPFILE = "test.properties"; + + public MariaDBTestBase() throws ManagedProcessException { + this(new Random().nextInt(1000) + 50000); + } + private static String dbUrl(String host, int port, String dbName) { + return String.format("jdbc:mysql://%s:%d/%s", host,port,dbName); + } + private static Map<String, String> initEnvDefaultValues() { + Map<String, String> defaults = new HashMap<>(); + defaults.put("SDNRDBURL", dbUrl("localhost",3306,SDNRDBDATABASETEST)); + defaults.put("SDNRDBDATABASE", "test"); + + return defaults; + } + public MariaDBTestBase(String host, int port) { + this(host, port, SDNRDBDATABASETEST ); + } + public MariaDBTestBase(String host, int port, String dbName) { + EnvGetter env = Section.getEnvGetter(); + envDefaultValues.put("SDNRDBURL", dbUrl(host,port,dbName)); + envDefaultValues.put("SDNRDBUSERNAME", "sdnrdb"); + envDefaultValues.put("SDNRDBPASSWORD", "sdnrdb"); + Section.setEnvGetter((envname) -> { + return envDefaultValues.getOrDefault(envname, env.getenv(envname)); + }); + SqlDBConfig config = new SqlDBConfig(new ConfigurationFileRepresentation(TESTPROPFILE)); + config.setControllerId("test123"); + this.db = null; + this.dbProvider = new SqlDBDataProvider(config, false); + //testCreateTable(this.dbProvider.getDBService()); + } + + public MariaDBTestBase(int port) throws ManagedProcessException { + EnvGetter env = Section.getEnvGetter(); + envDefaultValues.put("SDNRDBURL",dbUrl("localhost",port,SDNRDBDATABASETEST)); + Section.setEnvGetter((envname) -> { + return envDefaultValues.getOrDefault(envname, env.getenv(envname)); + }); + SqlDBConfig config = new SqlDBConfig(new ConfigurationFileRepresentation(TESTPROPFILE)); + //start db server + this.db = startDatabase(port); + //create db with name sdnrdb + this.dbProvider = new SqlDBDataProvider(config, false); + //testCreateTable(this.dbProvider.getDBService()); + } + + public void close() throws ManagedProcessException { + if (db != null) { + this.db.stop(); + } + File f = new File(TESTPROPFILE); + if(f.exists()) { + f.delete(); + } + } + + public SqlDBDataProvider getDbProvider() { + return dbProvider; + } + + public DB getDb() { + return db; + } + + private static DB startDatabase(int port) throws ManagedProcessException { + // Start MariaDB4j database + DBConfigurationBuilder dbconfig = DBConfigurationBuilder.newBuilder(); + dbconfig.setPort(port); // 0 => autom. detect free port + DB db = DB.newEmbeddedDB(dbconfig.build()); + db.start(); + return db; + } + public static void testCreateTable(SqlDBClient dbService) { + createTableOdl(dbService); + createTable(dbService, ConnectionlogEntity.class, Entity.Connectionlog, true); + createTable(dbService, EventlogEntity.class, Entity.Eventlog, true); + createTable(dbService, FaultcurrentEntity.class, Entity.Faultcurrent, false); + createTable(dbService, FaultlogEntity.class, Entity.Faultlog, true); + createTable(dbService, GuicutthroughEntity.class, Entity.Guicutthrough, true); + createTable(dbService, Pmdata15mEntity.class, Entity.Historicalperformance15min, false); + createTable(dbService, Pmdata24hEntity.class, Entity.Historicalperformance24h, false); + createTable(dbService, InventoryEntity.class, Entity.Inventoryequipment, false); + createTable(dbService, MaintenanceEntity.class, Entity.Maintenancemode, false); + createTable(dbService, MediatorServerEntity.class, Entity.MediatorServer, true); + createTable(dbService, NetworkElementConnectionEntity.class, Entity.NetworkelementConnection, false); + } + public static boolean createTableOdl(SqlDBClient dbService) { + String createStatement = null; + createStatement = SqlDBMapper.createTableOdl(); + System.out.println(createStatement); + try { + return dbService.write(createStatement); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } + return false; + } + + public static boolean createTable(SqlDBClient dbService, Class<?> cls, Entity entity, boolean autoIndex) { + String createStatement = null; + try { + createStatement = SqlDBMapper.createTable(cls, entity, "", autoIndex); + } catch (UnableToMapClassException e) { + fail(e.getMessage()); + } + System.out.println(createStatement); + try { + return dbService.write(createStatement); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } + return false; + } + public String getDBUrl() { + return envDefaultValues.get("SDNRDBURL"); + } + public String getDBUsername() { + return envDefaultValues.getOrDefault("SDNRDBUSERNAME",""); + } + public String getDBPassword() { + return envDefaultValues.getOrDefault("SDNRDBPASSWORD",""); + } +} diff --git a/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMariaDBIstanbul.java b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMariaDBIstanbul.java new file mode 100644 index 000000000..072ce2f6b --- /dev/null +++ b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMariaDBIstanbul.java @@ -0,0 +1,90 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : ccsdk features + * ================================================================================ + * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property. + * All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + */ +package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.data.SqlDBDataProvider; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; +import ch.vorburger.exec.ManagedProcessException; + +public class TestMariaDBIstanbul { + + private static final String MARIADB_USERNAME = "sdnrdb"; + private static final String MARIADB_PASSWORD = "sdnrdb"; + // private static final String MARIADB_HOST = "10.20.11.159"; + private static final String MARIADB_HOST = "sdnrdb"; + private static final int MARIADB_PORT = 3306; + private static final String MARIADB_DATABASENAME = "sdnrdb"; + + private static SqlDBClient dbService; + + private static MariaDBTestBase testBase; + private static SqlDBDataProvider dbProvider; + + @BeforeClass + public static void init() throws Exception { + + //testBase = new MariaDBTestBase(MARIADB_HOST,MARIADB_PORT, MARIADB_DATABASENAME); + testBase = new MariaDBTestBase(); + dbProvider = testBase.getDbProvider(); + dbProvider.waitForDatabaseReady(30, TimeUnit.SECONDS); + // dbProvider.setControllerId(); + + } + + @AfterClass + public static void close() { + try { + testBase.close(); + } catch (ManagedProcessException e) { + e.printStackTrace(); + } + } + + @Test + public void testCreate() { + DataMigrationProviderImpl provider = null; + try { + provider = new DataMigrationProviderImpl(SdnrDbType.MARIADB, testBase.getDBUrl(), testBase.getDBUsername(), + testBase.getDBPassword(), true, 30000); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + boolean success = provider.initDatabase(Release.ISTANBUL_R1, 1, 1, "", false, 1000); + assertTrue("init database failed",success); + try { + dbProvider.setControllerId(); + } catch (SQLException e) { + fail(e.getMessage()); + } + + } +} diff --git a/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMariaDBMapper.java b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMariaDBMapper.java new file mode 100644 index 000000000..f39f9b088 --- /dev/null +++ b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMariaDBMapper.java @@ -0,0 +1,228 @@ +/* + * ============LICENSE_START======================================================= + * ONAP : ccsdk features + * ================================================================================ + * Copyright (C) 2021 highstreet technologies GmbH Intellectual Property. + * All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + * + */ +package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup; + +import static org.junit.Assert.fail; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.lang.reflect.InvocationTargetException; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.data.SqlDBDataProvider; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.database.SqlDBMapper; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.database.SqlDBMapper.UnableToMapClassException; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.query.InsertQuery; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.query.SelectQuery; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionLogStatus; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionlogEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Entity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.EventlogEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultcurrentEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultlogEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GuicutthroughEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.InventoryEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.MaintenanceEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.MediatorServerEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnection; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnectionBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnectionEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementDeviceType; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Pmdata15mEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Pmdata24hEntity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.network.element.connection.entity.NodeDetails; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.network.element.connection.entity.NodeDetailsBuilder; +import org.opendaylight.yangtools.yang.binding.DataObject; +import org.opendaylight.yangtools.yang.common.Uint32; +import ch.vorburger.exec.ManagedProcessException; + +public class TestMariaDBMapper { + + + private static final String MARIADB_USERNAME = "sdnrdb"; + private static final String MARIADB_PASSWORD = "sdnrdb"; + // private static final String MARIADB_HOST = "10.20.11.159"; + private static final String MARIADB_HOST = "sdnrdb"; + private static final int MARIADB_PORT = 3306; + private static final String MARIADB_DATABASENAME = "sdnrdb"; + + // private static DbLibService dbService; + + private static MariaDBTestBase testBase; + private static SqlDBDataProvider dbProvider; + + @BeforeClass + public static void init() throws Exception { + + testBase = new MariaDBTestBase(); + dbProvider = testBase.getDbProvider(); + dbProvider.waitForDatabaseReady(30, TimeUnit.SECONDS); + // dbProvider.setControllerId(); + + } + + @AfterClass + public static void close() { + try { + testBase.close(); + } catch (ManagedProcessException e) { + e.printStackTrace(); + } + } + + @Test + public void testCreateSdnrDBTables() { + createTables(dbProvider.getDBService()); + } + + public static void createTables(SqlDBClient dbService) { + createTableOdl(dbService); + createTable(dbService, ConnectionlogEntity.class, Entity.Connectionlog, true); + createTable(dbService, EventlogEntity.class, Entity.Eventlog, true); + createTable(dbService, FaultcurrentEntity.class, Entity.Faultcurrent, false); + createTable(dbService, FaultlogEntity.class, Entity.Faultlog, true); + createTable(dbService, GuicutthroughEntity.class, Entity.Guicutthrough, true); + createTable(dbService, Pmdata15mEntity.class, Entity.Historicalperformance15min, false); + createTable(dbService, Pmdata24hEntity.class, Entity.Historicalperformance24h, false); + createTable(dbService, InventoryEntity.class, Entity.Inventoryequipment, false); + createTable(dbService, MaintenanceEntity.class, Entity.Maintenancemode, false); + createTable(dbService, MediatorServerEntity.class, Entity.MediatorServer, true); + createTable(dbService, NetworkElementConnectionEntity.class, Entity.NetworkelementConnection, false); + } + + //@Test + public void testInsert() { + NetworkElementConnectionBuilder builder = new NetworkElementConnectionBuilder(); + builder.setId("ROADM-A"); + builder.setNodeId("ROADM-A"); + builder.setCoreModelCapability("2017-03-12"); + builder.setDeviceType(NetworkElementDeviceType.OROADM); + builder.setHost("10.20.30.40"); + builder.setIsRequired(true); + NodeDetails nodeDetails = new NodeDetailsBuilder().setAvailableCapabilities(Arrays.asList( + "urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&also-supported=report-all," + + "report-all-tagged,trim,explicit", + "urn:ietf:params:netconf:capability:validate:1.1", "urn:ietf:params:netconf:capability:url:1.0")) + .build(); + builder.setNodeDetails(nodeDetails); + builder.setPassword("password"); + builder.setPort(Uint32.valueOf(50000)); + builder.setStatus(ConnectionLogStatus.Connected); + builder.setUsername("admin"); + try { + writeEntry(builder.build(), Entity.NetworkelementConnection); + } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException + | JsonProcessingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + //@Test + public void testRead() { + List<NetworkElementConnection> con = + readEntry(Entity.NetworkelementConnection, NetworkElementConnection.class, "ROADM-A"); + System.out.println(con); + } + + private <T extends DataObject> List<T> readEntry(Entity entity, Class<T> clazz, String id) { + final SelectQuery selectStatement = new SelectQuery(entity.getName()); + System.out.println(selectStatement); + try { + return SqlDBMapper.read(dbProvider.getDBService().read(selectStatement.toSql()), clazz); + + } catch (SQLException | IllegalAccessException | IllegalArgumentException | InvocationTargetException + | InstantiationException | SecurityException | NoSuchMethodException | JsonProcessingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } + return null; + } + + private <T extends DataObject> boolean writeEntry(T data, Entity entity) throws IllegalAccessException, + IllegalArgumentException, InvocationTargetException, JsonProcessingException { + + final InsertQuery<T> insertStatement = new InsertQuery<T>(entity, data, dbProvider.getControllerId()); + System.out.println(insertStatement); + try { + return dbProvider.getDBService().write(insertStatement.toSql()); + + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } + + return false; + } + + private static boolean createTableOdl(SqlDBClient dbService) { + String createStatement = null; + createStatement = SqlDBMapper.createTableOdl(); + System.out.println(createStatement); + try { + return dbService.write(createStatement); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } + return false; + } + + private static boolean createTable(SqlDBClient dbService, Class<?> cls, Entity entity, boolean autoIndex) { + String createStatement = null; + try { + createStatement = SqlDBMapper.createTable(cls, entity, "", autoIndex); + } catch (UnableToMapClassException e) { + fail(e.getMessage()); + } + System.out.println(createStatement); + try { + return dbService.write(createStatement); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } + return false; + } + + private static Properties getConfig() { + Properties config = new Properties(); + config.setProperty("org.onap.ccsdk.sli.dbtype", "jdbc"); + config.setProperty("org.onap.ccsdk.sli.jdbc.hosts", MARIADB_HOST); + config.setProperty("org.onap.ccsdk.sli.jdbc.url", + String.format("jdbc:mysql://dbhost:%d/%s", MARIADB_PORT, MARIADB_DATABASENAME)); + config.setProperty("org.onap.ccsdk.sli.jdbc.driver", "org.mariadb.jdbc.Driver"); + config.setProperty("org.onap.ccsdk.sli.jdbc.database", MARIADB_DATABASENAME); + config.setProperty("org.onap.ccsdk.sli.jdbc.user", MARIADB_USERNAME); + config.setProperty("org.onap.ccsdk.sli.jdbc.password", MARIADB_PASSWORD); + return config; + } +} diff --git a/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMigrationProvider.java b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMigrationProvider.java index 311582a08..412c8b857 100644 --- a/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMigrationProvider.java +++ b/sdnr/wt/data-provider/setup/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/TestMigrationProvider.java @@ -26,6 +26,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.junit.Test; import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport; import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release; @@ -41,8 +42,9 @@ public class TestMigrationProvider { @Test public void testCreateImport() throws Exception { - DataMigrationProviderImpl provider = new DataMigrationProviderImpl(hosts, null, null, true, 5000); - DataMigrationReport report=null; + DataMigrationProviderImpl provider = + new DataMigrationProviderImpl(SdnrDbType.ELASTICSEARCH, hosts[0].toUrl(), null, null, true, 5000); + DataMigrationReport report = null; try { |