From ee75b19ee79f49b28a3137b6dcfdf02d6120f3de Mon Sep 17 00:00:00 2001 From: Michael Dürre Date: Wed, 26 Feb 2020 06:41:50 +0100 Subject: add data migration tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add tool for data migration and initialization Issue-ID: SDNC-1085 Signed-off-by: Michael Dürre Change-Id: Ibd1ffeffa95d2897ae65f7d964e98941d810ffcb Signed-off-by: Michael Dürre --- .../data/HtDatabaseMaintenanceService.java | 8 ++++- .../sdnr/wt/dataprovider/data/QueryByFilter.java | 6 ++-- .../sdnr/wt/dataprovider/impl/EsConfig.java | 11 ++++--- .../provider/src/main/resources/es-init.sh | 8 ++--- .../wt/dataprovider/test/TestCRUDforDatabase.java | 27 +++++++++++++++ .../wt/dataprovider/test/TestFilterConversion.java | 7 ++++ .../wt/dataprovider/test/TestTimestampFilter.java | 19 +++++++++-- .../provider/src/test/resources/log4j.properties | 33 ------------------- .../provider/src/test/resources/log4j2.xml | 38 ---------------------- .../src/test/resources/simplelogger.properties | 5 +-- 10 files changed, 76 insertions(+), 86 deletions(-) delete mode 100644 sdnr/wt/data-provider/provider/src/test/resources/log4j.properties delete mode 100644 sdnr/wt/data-provider/provider/src/test/resources/log4j2.xml (limited to 'sdnr/wt/data-provider/provider/src') diff --git a/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/HtDatabaseMaintenanceService.java b/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/HtDatabaseMaintenanceService.java index 45d989d6a..5ff7a5dec 100644 --- a/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/HtDatabaseMaintenanceService.java +++ b/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/HtDatabaseMaintenanceService.java @@ -120,6 +120,7 @@ public class HtDatabaseMaintenanceService implements HtDatabaseMaintenance { } } + } /** @@ -153,6 +154,7 @@ public class HtDatabaseMaintenanceService implements HtDatabaseMaintenance { * @param mountPointNodeName * @return */ + @SuppressWarnings("null") private boolean isRequireNe(String mountPointNodeName) { NetworkElementConnectionEntity ne = null; if (requiredNeRW != null) { @@ -161,7 +163,11 @@ public class HtDatabaseMaintenanceService implements HtDatabaseMaintenance { } else { LOG.warn("cannot read db. no db reader writer initialized"); } - return ne != null; + if (ne != null && ne.isIsRequired() != null) { + return ne.isIsRequired(); + } else { + return false; + } } } diff --git a/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/QueryByFilter.java b/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/QueryByFilter.java index db527959c..6b3c89af9 100644 --- a/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/QueryByFilter.java +++ b/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/data/QueryByFilter.java @@ -17,6 +17,7 @@ ******************************************************************************/ package org.onap.ccsdk.features.sdnr.wt.dataprovider.data; +import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.List; @@ -40,6 +41,7 @@ import org.slf4j.LoggerFactory; public class QueryByFilter { private static final Logger LOG = LoggerFactory.getLogger(DataObjectAcessorPm.class); + private static final List timestampValueNames = Arrays.asList("timestamp","start","end"); private QueryByFilter() { //Hide @@ -276,7 +278,7 @@ public class QueryByFilter { // v=v.toLowerCase(); } if (DbFilter.hasSearchParams(v)) { - if(p!=null && "timestamp".equals(p.toLowerCase())) { + if(p!=null && timestampValueNames.contains(p.toLowerCase())) { query = fromTimestampSearchFilter(p,v); if(query!=null) { return query; @@ -311,7 +313,7 @@ public class QueryByFilter { // v=v.toLowerCase(); } if(DbFilter.hasSearchParams(v)) { - if(p!=null && "timestamp".equals(p.toLowerCase())) { + if(p!=null && timestampValueNames.contains(p.toLowerCase())) { tmpQuery=fromTimestampSearchFilter(p,v); if(tmpQuery!=null) { query.must(tmpQuery); diff --git a/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/impl/EsConfig.java b/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/impl/EsConfig.java index a846cd8f3..ccfbe98fd 100644 --- a/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/impl/EsConfig.java +++ b/sdnr/wt/data-provider/provider/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/impl/EsConfig.java @@ -53,7 +53,10 @@ public class EsConfig implements Configuration, IEsConfig { private static final String DEFAULT_ARCHIVE_INTERVAL_SEC = "0"; /** keep data for this time [in seconds] 30 days */ private static final String DEFAULT_ARCHIVE_LIMIT_SEC = String.valueOf(60L * 60L * 24L * 30L); - private static final String DEFAULT_KEY_NODE = "elasticsearchnode"; + //private static final String DEFAULT_KEY_NODE = "elasticsearchnode"; + private static final String DEFAULT_VALUE_NODE = "elasticsearchnode"; + private static final String DEFAULT_VALUE_DBUSERNAME = "${SDNRDBUSERNAME}"; + private static final String DEFAULT_VALUE_DBPASSWORD = "${SDNRDBPASSWORD}"; private final ConfigurationFileRepresentation configuration; @@ -138,9 +141,9 @@ public class EsConfig implements Configuration, IEsConfig { configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_CLUSTER, DEFAULT_VALUE_CLUSTER); configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_ARCHIVE_INTERVAL, DEFAULT_ARCHIVE_INTERVAL_SEC); - configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_NODE, DEFAULT_KEY_NODE); - configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_AUTH_USERNAME, ""); - configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_AUTH_PASSWORD, ""); + configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_NODE, DEFAULT_VALUE_NODE); + configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_AUTH_USERNAME, DEFAULT_VALUE_DBUSERNAME); + configuration.setPropertyIfNotAvailable(SECTION_MARKER_ES, PROPERTY_KEY_AUTH_PASSWORD, DEFAULT_VALUE_DBPASSWORD); } @Override diff --git a/sdnr/wt/data-provider/provider/src/main/resources/es-init.sh b/sdnr/wt/data-provider/provider/src/main/resources/es-init.sh index 8873b0b75..67d495aa0 100755 --- a/sdnr/wt/data-provider/provider/src/main/resources/es-init.sh +++ b/sdnr/wt/data-provider/provider/src/main/resources/es-init.sh @@ -36,10 +36,10 @@ INITFILENAME="Init.script" set_definition() { def "connectionlog" '{"node-id": {"type": "keyword"},"timestamp": {"type": "date"},"status": {"type": "keyword"}}' - def "maintenancemode" '{"node-id": {"type": "keyword"},"active": {"type": "boolean"}},"date_detection":false}}' - def "faultlog" '{"node-id": {"type": "keyword"},"severity": {"type": "keyword"},"timestamp": {"type": "date"},"problem": {"type": "keyword"},"counter": {"type": "keyword"},"object-id":{"type": "keyword"},"source-type":{"type": "keyword"}}' - def "faultcurrent" '{"node-id": {"type": "keyword"},"severity": {"type": "keyword"},"timestamp": {"type": "date"},"problem": {"type": "keyword"},"counter": {"type": "keyword"},"object-id":{"type": "keyword"}}' - def "eventlog" '{"node-id": {"type": "keyword"},"timestamp": {"type": "date"},"new-value": {"type": "keyword"},"attribute-name": {"type": "keyword"},"counter": {"type": "keyword"},"object-id": {"type": "keyword"}}' + def "maintenancemode" '{"node-id": {"type": "keyword"},"start": {"type": "date"},"end": {"type": "date"},"description": {"type": "keyword"},"active": {"type": "boolean"}},"date_detection":false}}' + def "faultlog" '{"node-id": {"type": "keyword"},"severity": {"type": "keyword"},"timestamp": {"type": "date"},"problem": {"type": "keyword"},"counter": {"type": "long"},"object-id":{"type": "keyword"},"source-type":{"type": "keyword"}}' + def "faultcurrent" '{"node-id": {"type": "keyword"},"severity": {"type": "keyword"},"timestamp": {"type": "date"},"problem": {"type": "keyword"},"counter": {"type": "long"},"object-id":{"type": "keyword"}}' + def "eventlog" '{"node-id": {"type": "keyword"},"source-type": {"type": "keyword"},"timestamp": {"type": "date"},"new-value": {"type": "keyword"},"attribute-name": {"type": "keyword"},"counter": {"type": "long"},"object-id": {"type": "keyword"}}' def "inventoryequipment" '{"date": {"type": "keyword"},"model-identifier": {"type": "keyword"},"manufacturer-identifier": {"type": "keyword"},"type-name": {"type": "keyword"},"description": {"type": "keyword"},"uuid": {"type": "keyword"},"version": {"type": "keyword"},"parent-uuid": {"type": "keyword"},"contained-holder": {"type": "keyword"},"node-id": {"type": "keyword"},"tree-level": {"type": "long"},"part-type-id": {"type": "keyword"},"serial": {"type": "keyword"}}' def "historicalperformance24h" '{"node-name":{"type": "keyword"},"timestamp":{"type": "date"},"suspect-interval-flag":{"type":"boolean"},"scanner-id":{"type": "keyword"},"uuid-interface":{"type": "keyword"},"layer-protocol-name":{"type": "keyword"},"granularity-period":{"type": "keyword"},"radio-signal-id":{"type": "keyword"}}' def "historicalperformance15min" '{"node-name":{"type": "keyword"},"timestamp":{"type": "date"},"suspect-interval-flag":{"type":"boolean"},"scanner-id":{"type": "keyword"},"uuid-interface":{"type": "keyword"},"layer-protocol-name":{"type": "keyword"},"granularity-period":{"type": "keyword"},"radio-signal-id":{"type": "keyword"}}' diff --git a/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestCRUDforDatabase.java b/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestCRUDforDatabase.java index d898644c9..b98af452f 100644 --- a/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestCRUDforDatabase.java +++ b/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestCRUDforDatabase.java @@ -35,6 +35,7 @@ import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo; import org.onap.ccsdk.features.sdnr.wt.common.database.queries.QueryBuilders; import org.onap.ccsdk.features.sdnr.wt.common.database.requests.BaseRequest; import org.onap.ccsdk.features.sdnr.wt.dataprovider.data.ElasticSearchDataProvider; +import org.onap.ccsdk.features.sdnr.wt.dataprovider.yangtools.YangToolsMapper; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.DateAndTime; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.CreateMaintenanceInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.CreateMaintenanceInputBuilder; @@ -52,6 +53,9 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.pro import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.DeleteNetworkElementConnectionInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.DeleteNetworkElementConnectionInputBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.Entity; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.EntityInput; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.Faultlog; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.FaultlogEntity; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.GranularityPeriodType; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.ReadConnectionlogListInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.ReadConnectionlogListInputBuilder; @@ -105,6 +109,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.pro import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.UpdateNetworkElementConnectionInput; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.UpdateNetworkElementConnectionInputBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.UpdateNetworkElementConnectionOutputBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.entity.input.Filter; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.entity.input.FilterBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.entity.input.Pagination; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.entity.input.PaginationBuilder; @@ -1338,6 +1343,28 @@ public class TestCRUDforDatabase { } + @Test + public void testOutputCamelCase() { + try { + String jsonString="{\n" + + "\"timestamp\": \"2020-02-20T09:31:22.3Z\",\n" + + "\"object-id\": \"LP-MWPS-RADIO\",\n" + + "\"severity\": \"Critical\",\n" + + "\"counter\": 10,\n" + + "\"implemented-interface\": \"org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.Faultlog\",\n" + + "\"source-type\": \"Netconf\",\n" + + "\"node-id\": \"sim4\",\n" + + "\"problem\": \"signalIsLost\"\n" + + "}"; + YangToolsMapper yangtoolsMapper = new YangToolsMapper(); + FaultlogEntity log = yangtoolsMapper.readValue( jsonString, Faultlog.class ); + System.out.println(log ); + System.out.println(yangtoolsMapper.writeValueAsString(log)); + } catch (IOException e) { + fail(e.getMessage()); + } + + } private Pagination getPagination(long pageSize, int page) { return new PaginationBuilder().setPage(BigInteger.valueOf(page)).setSize(pageSize).build(); diff --git a/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestFilterConversion.java b/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestFilterConversion.java index 5ce8fcc0b..32a5773d7 100644 --- a/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestFilterConversion.java +++ b/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestFilterConversion.java @@ -60,4 +60,11 @@ public class TestFilterConversion { QueryByFilter.setSortOrder(query, sortorder); assertNotNull(QueryByFilter.getSortOrder(sortorder, PROPERTY)); } + @Test + public void testSortorder() { + String f = "{\"input\":{\"filter\":[],\"sortorder\":[{\"property\":\"source-type\",\"sortorder\":\"ascending\"}],\"pagination\":{\"size\":10,\"page\":1}}}"; + + QueryBuilder query = QueryByFilter.setSortOrder(QueryByFilter.fromFilter(null), Arrays.asList(new SortorderBuilder().setProperty("source-type").setSortorder(SortOrder.Ascending).build())); + System.out.println(query.toJSON()); + } } diff --git a/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestTimestampFilter.java b/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestTimestampFilter.java index c32149ba5..f41ee44b1 100644 --- a/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestTimestampFilter.java +++ b/sdnr/wt/data-provider/provider/src/test/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/test/TestTimestampFilter.java @@ -32,8 +32,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.pro import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev190801.entity.input.FilterBuilder; public class TestTimestampFilter { - - @Test public void testTimestampRange() { @@ -122,6 +120,23 @@ public class TestTimestampFilter { query= QueryByFilter.fromFilter(filters ); assertRange(query.getInner(), PROPERTY_TIMESTAMP, "2050-10-14T12:42:56.0Z", "2050-10-14T12:42:57.0Z"); } + + @Test + public void testExtra() { + final String PROPERTY_TIMESTAMP = "end"; + List filters = null; + QueryBuilder query= null; + filters = Arrays.asList(new FilterBuilder().setProperty(PROPERTY_TIMESTAMP).setFiltervalue("2020-02-19T*").build()); + query= QueryByFilter.fromFilter(filters ); + assertRange(query.getInner(), PROPERTY_TIMESTAMP, "2020-02-19T00:00:00.0Z", "2020-02-20T00:00:00.0Z"); + filters = Arrays.asList(new FilterBuilder().setProperty(PROPERTY_TIMESTAMP).setFiltervalue("2020-02-19*").build()); + query= QueryByFilter.fromFilter(filters ); + assertRange(query.getInner(), PROPERTY_TIMESTAMP, "2020-02-19T00:00:00.0Z", "2020-02-20T00:00:00.0Z"); + filters = Arrays.asList(new FilterBuilder().setProperty(PROPERTY_TIMESTAMP).setFiltervalue("2020*").build()); + query= QueryByFilter.fromFilter(filters ); + assertRange(query.getInner(), PROPERTY_TIMESTAMP, "2020-01-01T00:00:00.0Z", "2021-01-01T00:00:00.0Z"); + + } private void assertRange(JSONObject rangeQuery,String property,String lower,String upper) { System.out.println("==test for "+rangeQuery.toString()); assertTrue(rangeQuery.has("range")); diff --git a/sdnr/wt/data-provider/provider/src/test/resources/log4j.properties b/sdnr/wt/data-provider/provider/src/test/resources/log4j.properties deleted file mode 100644 index 2a2f8099e..000000000 --- a/sdnr/wt/data-provider/provider/src/test/resources/log4j.properties +++ /dev/null @@ -1,33 +0,0 @@ -# -# ============LICENSE_START======================================================= -# ONAP : ccsdk features -# ================================================================================ -# Copyright (C) 2020 AT&T Intellectual Property. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END======================================================= -# -# - -log4j.rootLogger=INFO, out - -log4j.logger.org.apache.camel.impl.converter=WARN -log4j.logger.org.apache.camel.management=WARN -log4j.logger.org.apache.camel.impl.DefaultPackageScanClassResolver=WARN -log4j.logger.org.springframework=ERROR - -# CONSOLE appender not used by default -log4j.appender.out=org.apache.log4j.ConsoleAppender -log4j.appender.out.layout=org.apache.log4j.PatternLayout -log4j.appender.out.layout.ConversionPattern=%d [%-15.15t] %-5p %-30.30c{1} - %m%n - diff --git a/sdnr/wt/data-provider/provider/src/test/resources/log4j2.xml b/sdnr/wt/data-provider/provider/src/test/resources/log4j2.xml deleted file mode 100644 index 846a9fc10..000000000 --- a/sdnr/wt/data-provider/provider/src/test/resources/log4j2.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/sdnr/wt/data-provider/provider/src/test/resources/simplelogger.properties b/sdnr/wt/data-provider/provider/src/test/resources/simplelogger.properties index 980da34f8..9943c2fc3 100644 --- a/sdnr/wt/data-provider/provider/src/test/resources/simplelogger.properties +++ b/sdnr/wt/data-provider/provider/src/test/resources/simplelogger.properties @@ -19,9 +19,10 @@ # # -org.slf4j.simpleLogger.defaultLogLevel=debug +org.slf4j.simpleLogger.defaultLogLevel=INFO org.slf4j.simpleLogger.showDateTime=true #org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z #org.slf4j.simpleLogger.showThreadName=true org.slf4j.simpleLogger.showLogName=true -org.slf4j.simpleLogger.showShortLogName=false \ No newline at end of file +org.slf4j.simpleLogger.showShortLogName=false +org.slf4j.simpleLogger.org.onap.ccsdk.features.sdnr.wt.dataprovider=DEBUG -- cgit 1.2.3-korg