From 2340495da15890f2e8fd23d962b13623c8a5f0dc Mon Sep 17 00:00:00 2001 From: Michael Dürre Date: Wed, 8 Apr 2020 08:17:02 +0200 Subject: fix db version conflict issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit merged write and refresh request together Issue-ID: SDNC-1146 Signed-off-by: Michael Dürre Change-Id: I6d65f372ee27a84c9b55371cefd1d7ecb9fac9fb --- .../sdnr/wt/common/database/HtDatabaseClient.java | 40 +++++++-------- .../wt/common/database/requests/BaseRequest.java | 41 +++++++++++++--- .../database/requests/CreateIndexRequest.java | 3 +- .../database/requests/DeleteByQueryRequest.java | 9 +++- .../wt/common/database/requests/DeleteRequest.java | 16 +++++- .../wt/common/database/requests/IndexRequest.java | 27 ++++++++-- .../database/requests/UpdateByQueryRequest.java | 14 +++++- .../wt/common/database/requests/UpdateRequest.java | 57 ++++++++++++++++++---- .../wt/common/database/responses/BaseResponse.java | 4 ++ .../common/database/responses/DeleteResponse.java | 6 ++- 10 files changed, 169 insertions(+), 48 deletions(-) (limited to 'sdnr/wt/common/src/main/java/org/onap') diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/HtDatabaseClient.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/HtDatabaseClient.java index e7b15bd0f..9a369ea5b 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/HtDatabaseClient.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/HtDatabaseClient.java @@ -130,7 +130,7 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A public @Nullable String doWriteRaw(String indexName,String dataTypeName, @Nullable String esId, String json) { IndexResponse response = null; - IndexRequest indexRequest = new IndexRequest(indexName,dataTypeName,esId); + IndexRequest indexRequest = new IndexRequest(indexName,dataTypeName,esId,this.doRefreshAfterWrite); indexRequest.source(json); try { response = this.index(indexRequest ); @@ -142,9 +142,9 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A LOG.warn("Response null during write: {} {}", esId, json); return null; } - if(this.doRefreshAfterWrite) { - this.doRefresh(dataTypeName); - } +// if(this.doRefreshAfterWrite) { +// this.doRefresh(dataTypeName); +// } return response.getId(); } @@ -167,16 +167,16 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A @Override public boolean doRemove(String dataTypeName, String esId) { - DeleteRequest deleteRequest = new DeleteRequest(dataTypeName,dataTypeName,esId); + DeleteRequest deleteRequest = new DeleteRequest(dataTypeName,dataTypeName,esId,this.doRefreshAfterWrite); DeleteResponse response = null; try { response = this.delete(deleteRequest); } catch (IOException e) { LOG.warn("Problem deleting from db: {}",e.getMessage()); } - if(this.doRefreshAfterWrite) { - this.doRefresh(dataTypeName); - } +// if(this.doRefreshAfterWrite) { +// this.doRefresh(dataTypeName); +// } return response!=null?response.isDeleted():false; } @@ -257,7 +257,7 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A return null; } boolean success = false; - UpdateRequest request = new UpdateRequest(dataTypeName, dataTypeName, esId); + UpdateRequest request = new UpdateRequest(dataTypeName, dataTypeName, esId,this.doRefreshAfterWrite); request.source(new JSONObject(json),onlyForInsert); try { UpdateResponse response = this.update(request); @@ -265,15 +265,15 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A } catch (IOException e) { LOG.warn("Problem updating {} with id {} and data {}: {}", dataTypeName, esId, json, e); } - if(this.doRefreshAfterWrite) { - this.doRefresh(dataTypeName); - } +// if(this.doRefreshAfterWrite) { +// this.doRefresh(dataTypeName); +// } return success ? esId : null; } @Override public boolean doUpdate(String dataTypeName, String json, QueryBuilder query) { boolean success = false; - UpdateByQueryRequest request = new UpdateByQueryRequest(dataTypeName, dataTypeName ); + UpdateByQueryRequest request = new UpdateByQueryRequest(dataTypeName, dataTypeName ,this.doRefreshAfterWrite); request.source(new JSONObject(json),query); try { UpdateByQueryResponse response = this.update(request); @@ -281,9 +281,9 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A } catch (IOException e) { LOG.warn("Problem updating items in {} with query {} and data {}: {}", dataTypeName, query, json, e); } - if(this.doRefreshAfterWrite) { - this.doRefresh(dataTypeName); - } +// if(this.doRefreshAfterWrite) { +// this.doRefresh(dataTypeName); +// } return success; } @@ -292,7 +292,7 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A @Override public int doRemove(String dataTypeName, QueryBuilder query) { int del=0; - DeleteByQueryRequest request = new DeleteByQueryRequest(dataTypeName); + DeleteByQueryRequest request = new DeleteByQueryRequest(dataTypeName,this.doRefreshAfterWrite); request.source(query); try { DeleteByQueryResponse response = this.deleteByQuery(request); @@ -300,9 +300,9 @@ public class HtDatabaseClient extends ExtRestClient implements DatabaseClient, A } catch (IOException e) { LOG.warn("Problem delete in {} with query {}:{} ", dataTypeName, query.toJSON(), e); } - if(this.doRefreshAfterWrite) { - this.doRefresh(dataTypeName); - } +// if(this.doRefreshAfterWrite) { +// this.doRefresh(dataTypeName); +// } return del; } diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/BaseRequest.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/BaseRequest.java index 061941bfb..e7261f3e5 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/BaseRequest.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/BaseRequest.java @@ -35,12 +35,31 @@ public abstract class BaseRequest { private static final Logger LOG = LoggerFactory.getLogger(BaseRequest.class); + public static final int DEFAULT_RETRIES = 1; + protected final Request request; private String query; + private final boolean refresh; + public BaseRequest(String method, String endpoint) { - LOG.debug("create request {} {}", method, endpoint); - this.request = new Request(method, endpoint); - query=null; + LOG.debug("create request {} {}" ,method, endpoint); + this.refresh = false; + this.request = new Request(method, endpoint); + query = null; + } + + public BaseRequest(String method, String endpoint, boolean refresh) { + LOG.debug("create request {} {} with refresh={}", method, endpoint, refresh); + this.refresh = refresh; + this.request = new Request(method, String.format("%s?refresh=%s", endpoint, String.valueOf(refresh))); + query = null; + } + + public BaseRequest(String method, String endpoint, boolean refresh, int retries) { + LOG.debug("create request {} {} with refresh={}", method, endpoint, refresh); + this.refresh = refresh; + this.request = new Request(method, String.format("%s?refresh=%s&retry_on_conflict=%d", endpoint, String.valueOf(refresh),retries)); + query = null; } public Request getInner() { @@ -58,9 +77,11 @@ public abstract class BaseRequest { } return value; } + @Override public String toString() { - return this.request.getMethod() + " "+this.request.getEndpoint()+ " : "+(this.query!=null?this.query:"no query"); + return this.request.getMethod() + " " + this.request.getEndpoint() + " : " + + (this.query != null ? this.query : "no query"); } protected void setQuery(QueryBuilder query) { @@ -72,8 +93,16 @@ public abstract class BaseRequest { } public void setQuery(String content) { - this.query=content; - LOG.trace("query={}",content); + this.query = content; + LOG.trace("query={}", content); this.request.setJsonEntity(this.query); } + + protected String getQuery() { + return this.query; + } + + protected boolean doRefresh() { + return this.refresh; + } } diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/CreateIndexRequest.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/CreateIndexRequest.java index 934c0f6df..6f060196e 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/CreateIndexRequest.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/CreateIndexRequest.java @@ -47,9 +47,10 @@ public class CreateIndexRequest extends BaseRequest{ super.setQuery(o); } @SuppressWarnings("hiding") - public void mappings(JSONObject mappings) { + public CreateIndexRequest mappings(JSONObject mappings) { this.mappings=mappings; this.setRequest(); + return this; } public void settings(JSONObject settings) { diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteByQueryRequest.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteByQueryRequest.java index ded69dd17..15c876e48 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteByQueryRequest.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteByQueryRequest.java @@ -26,11 +26,16 @@ import org.onap.ccsdk.features.sdnr.wt.common.database.queries.QueryBuilder; public class DeleteByQueryRequest extends BaseRequest { public DeleteByQueryRequest(String alias) { - super("POST",String.format("/%s/_delete_by_query",alias)); + this(alias, false); } - public void source(QueryBuilder query) { + public DeleteByQueryRequest(String alias, boolean refresh) { + super("POST",String.format("/%s/_delete_by_query",alias), refresh); + } + + public DeleteByQueryRequest source(QueryBuilder query) { this.setQuery(query); + return this; } diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteRequest.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteRequest.java index 325b529e5..63202b169 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteRequest.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/DeleteRequest.java @@ -23,8 +23,22 @@ package org.onap.ccsdk.features.sdnr.wt.common.database.requests; public class DeleteRequest extends BaseRequest { + private final String alias; + private final String esId; public DeleteRequest(String alias,String dataType,String esId) { - super("DELETE",String.format("/%s/%s/%s",alias,dataType,BaseRequest.urlEncodeValue(esId))); + this(alias, dataType, esId, false); + } + public DeleteRequest(String alias,String dataType,String esId, boolean refresh) { + super("DELETE",String.format("/%s/%s/%s",alias,dataType,BaseRequest.urlEncodeValue(esId)), refresh); + this.alias = alias; + this.esId = esId; + } + protected String getAlias() { + return this.alias; + } + + protected String getEsId() { + return this.esId; } } diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/IndexRequest.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/IndexRequest.java index cf4aabbbb..a3ee578a9 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/IndexRequest.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/IndexRequest.java @@ -23,20 +23,39 @@ package org.onap.ccsdk.features.sdnr.wt.common.database.requests; import javax.annotation.Nullable; -public class IndexRequest extends BaseRequest{ +public class IndexRequest extends BaseRequest { + + private final String alias; + private final String esId; public IndexRequest(String alias, String dataType) { - this(alias,dataType,null); + this(alias, dataType, null); + } + + public IndexRequest(String alias, String dataType, @Nullable String esId) { + super("POST", esId != null ? String.format("/%s/%s/%s", alias, dataType, BaseRequest.urlEncodeValue(esId)) + : String.format("/%s/%s", alias, dataType)); + this.alias = alias; + this.esId = esId; } - public IndexRequest(String alias,String dataType, @Nullable String esId) { - super("POST",esId!=null?String.format("/%s/%s/%s",alias,dataType,BaseRequest.urlEncodeValue(esId)):String.format("/%s/%s",alias,dataType)); + public IndexRequest(String alias, String dataType, @Nullable String esId, boolean refresh) { + super("POST", esId != null ? String.format("/%s/%s/%s", alias, dataType, BaseRequest.urlEncodeValue(esId)) + : String.format("/%s/%s", alias, dataType),refresh); + this.alias = alias; + this.esId = esId; } public void source(String content) { super.setQuery(content); } + protected String getAlias() { + return this.alias; + } + protected String getEsId() { + return this.esId; + } } diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateByQueryRequest.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateByQueryRequest.java index 1eb6037f4..8bca04ff6 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateByQueryRequest.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateByQueryRequest.java @@ -29,9 +29,15 @@ import org.onap.ccsdk.features.sdnr.wt.common.database.queries.QueryBuilders; public class UpdateByQueryRequest extends BaseRequest { private JSONObject params; - + private final String alias; + public UpdateByQueryRequest(String alias, String dataType) { - super("POST", String.format("/%s/%s/_update_by_query", alias, dataType)); + this(alias, dataType, false); + } + + public UpdateByQueryRequest(String alias, String dataType, boolean refresh) { + super("POST", String.format("/%s/%s/_update_by_query", alias, dataType), refresh); + this.alias = alias; this.params = null; } @@ -103,4 +109,8 @@ public class UpdateByQueryRequest extends BaseRequest { } + protected String getAlias() { + return this.alias; + } + } diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateRequest.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateRequest.java index 7445e155d..16923b238 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateRequest.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/requests/UpdateRequest.java @@ -33,12 +33,36 @@ public class UpdateRequest extends BaseRequest { private static final Logger LOG = LoggerFactory.getLogger(UpdateRequest.class); private JSONObject params; + private String alias; + private String esId; + private int retries; public UpdateRequest(String alias, String dataType, String esId) { - super("POST", String.format("/%s/%s/%s/_update", alias, dataType, BaseRequest.urlEncodeValue(esId))); + this(alias, dataType, esId, BaseRequest.DEFAULT_RETRIES); + } + public UpdateRequest(String alias, String dataType, String esId, boolean refresh) { + this(alias, dataType, esId, BaseRequest.DEFAULT_RETRIES, refresh); + } + public UpdateRequest(String alias, String dataType, String esId, int retries) { + this(alias, dataType, esId, retries, false); + } + + public UpdateRequest(String alias, String dataType, String esId, int retries, boolean refresh) { + this(String.format("/%s/%s/%s/_update", alias, dataType, BaseRequest.urlEncodeValue(esId)), refresh); + this.alias = alias; + this.esId = esId; + this.retries = retries; + } + + public UpdateRequest(String uri, boolean refresh) { + super("POST", uri, refresh, BaseRequest.DEFAULT_RETRIES); this.params = null; + this.retries = 1; + } + + private UpdateRequest withParam(String key, JSONObject p) { if (this.params == null) { this.params = new JSONObject(); @@ -54,16 +78,18 @@ public class UpdateRequest extends BaseRequest { this.params.put(key, p); return this; } + public void source(JSONObject map) { - this.source(map,null); + this.source(map, null); } + public void source(JSONObject map, List onlyForInsert) { JSONObject outer = new JSONObject(); JSONObject script = new JSONObject(); script.put("lang", "painless"); - script.put("source", this.createInline(map,onlyForInsert)); - if(this.params!=null) { - script.put("params",this.params); + script.put("source", this.createInline(map, onlyForInsert)); + if (this.params != null) { + script.put("params", this.params); } outer.put("script", script); outer.put("upsert", map); @@ -72,16 +98,16 @@ public class UpdateRequest extends BaseRequest { } private String createInline(JSONObject map, List onlyForInsert) { - if(onlyForInsert==null) { + if (onlyForInsert == null) { onlyForInsert = new ArrayList(); } - String s = "",k=""; + String s = "", k = ""; Object value; String pkey; int i = 0; for (Object key : map.keySet()) { - k=String.valueOf(key); - if(onlyForInsert.contains(k)) { + k = String.valueOf(key); + if (onlyForInsert.contains(k)) { continue; } value = map.get(k); @@ -93,7 +119,7 @@ public class UpdateRequest extends BaseRequest { this.withParam(pkey, (JSONArray) value); } - s += String.format("ctx._source['%s']=%s;", key, "params."+pkey); + s += String.format("ctx._source['%s']=%s;", key, "params." + pkey); } else { s += String.format("ctx._source['%s']=%s;", key, escpaped(value)); } @@ -113,4 +139,15 @@ public class UpdateRequest extends BaseRequest { } + protected String getAlias() { + return this.alias; + } + + protected String getEsId() { + return this.esId; + } + + protected int getRetries() { + return this.retries; + } } diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/BaseResponse.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/BaseResponse.java index 2fe81bb1b..a3a1b16e1 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/BaseResponse.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/BaseResponse.java @@ -49,6 +49,10 @@ public class BaseResponse { } JSONObject getJson(Response response) { + if(response==null) { + LOG.warn("unable to parse response. response is null."); + return null; + } try { String sresponse = EntityUtils.toString(response.getEntity()); LOG.debug("parsing response={}", sresponse); diff --git a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/DeleteResponse.java b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/DeleteResponse.java index c7bba0e7d..d4dee0441 100644 --- a/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/DeleteResponse.java +++ b/sdnr/wt/common/src/main/java/org/onap/ccsdk/features/sdnr/wt/common/database/responses/DeleteResponse.java @@ -30,13 +30,15 @@ public class DeleteResponse extends BaseResponse { public DeleteResponse(Response response) { super(response); - int code = response.getStatusLine().getStatusCode(); - if (code < 210) { + if (this.isResponseSucceeded()) { JSONObject o = this.getJson(response); if (o != null) { this.isDeleted = "deleted".equals(o.getString("result")); } + else { + this.isDeleted=false; + } } else { this.isDeleted=false; -- cgit 1.2.3-korg