From 7e5a7d1d8cd74e42c8549e02724ec2df589e4117 Mon Sep 17 00:00:00 2001 From: Pawel Date: Fri, 20 Dec 2019 09:56:29 +0100 Subject: updated jackson-databind and changed json schema validator library Issue-ID: DCAEGEN2-1825 Signed-off-by: Pawel Change-Id: Ia4c9c6286adcf84631a58f9d5bfef124fed1cee6 --- pom.xml | 19 +++---- .../java/org/onap/dcae/ApplicationSettings.java | 50 ++++--------------- .../java/org/onap/dcae/JSonSchemasSupplier.java | 58 ++++++++++++++++++++++ .../java/org/onap/dcae/restapi/EventValidator.java | 29 ++--------- .../org/onap/dcae/restapi/SchemaValidator.java | 58 ++++++++++++++++++++++ .../org/onap/dcae/ApplicationSettingsTest.java | 42 +++++++++++----- .../org/onap/dcae/restapi/EventValidatorTest.java | 18 +++---- version.properties | 2 +- 8 files changed, 178 insertions(+), 98 deletions(-) create mode 100644 src/main/java/org/onap/dcae/JSonSchemasSupplier.java create mode 100644 src/main/java/org/onap/dcae/restapi/SchemaValidator.java diff --git a/pom.xml b/pom.xml index f36068fe..5dfcaf48 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ org.onap.dcaegen2.collectors.ves VESCollector - 1.5.2-SNAPSHOT + 1.5.3-SNAPSHOT dcaegen2-collectors-ves VESCollector @@ -281,7 +281,7 @@ org.springframework.boot spring-boot-dependencies - 2.1.0.RELEASE + 2.2.2.RELEASE pom import @@ -295,14 +295,15 @@ 1.1.1 - com.github.fge + com.networknt json-schema-validator - 2.2.6 - - - com.github.fge - json-schema-core - 1.2.5 + 1.0.29 + + + com.fasterxml.jackson.core + jackson-databind + + com.google.code.gson diff --git a/src/main/java/org/onap/dcae/ApplicationSettings.java b/src/main/java/org/onap/dcae/ApplicationSettings.java index 5164f878..1e9ae698 100644 --- a/src/main/java/org/onap/dcae/ApplicationSettings.java +++ b/src/main/java/org/onap/dcae/ApplicationSettings.java @@ -3,7 +3,7 @@ * PROJECT * ================================================================================ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. - * Copyright (C) 2018 Nokia. All rights reserved.s + * Copyright (C) 2018 - 2019 Nokia. All rights reserved.s * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,31 +21,21 @@ package org.onap.dcae; -import static io.vavr.API.Tuple; -import static java.lang.String.format; -import static java.nio.file.Files.readAllBytes; - -import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; import com.google.common.annotations.VisibleForTesting; +import com.networknt.schema.JsonSchema; import io.vavr.Function1; -import io.vavr.Tuple2; import io.vavr.collection.HashMap; import io.vavr.collection.List; import io.vavr.collection.Map; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import javax.annotation.Nullable; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.json.JSONObject; import org.onap.dcae.common.configuration.AuthMethodType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; +import java.nio.file.Path; +import java.nio.file.Paths; +import static java.lang.String.format; /** * Abstraction over application configuration. @@ -58,6 +48,7 @@ public class ApplicationSettings { private final String appInvocationDir; private final String configurationFileLocation; private final PropertiesConfiguration properties = new PropertiesConfiguration(); + private final JSonSchemasSupplier jSonSchemasSupplier = new JSonSchemasSupplier(); private final Map loadedJsonSchemas; public ApplicationSettings(String[] args, Function1> argsParser) { @@ -71,7 +62,9 @@ public class ApplicationSettings { configurationFileLocation = findOutConfigurationFileLocation(parsedArgs); loadPropertiesFromFile(); parsedArgs.filterKeys(k -> !"c".equals(k)).forEach(this::addOrUpdate); - loadedJsonSchemas = loadJsonSchemas(); + String collectorSchemaFile = properties.getString("collector.schema.file", + format("{\"%s\":\"etc/CommonEventFormat_28.4.1.json\"}", FALLBACK_VES_VERSION)); + loadedJsonSchemas = jSonSchemasSupplier.loadJsonSchemas(collectorSchemaFile); } public void reloadProperties() { @@ -187,24 +180,6 @@ public class ApplicationSettings { return prependWithUserDirOnRelative(parsedArgs.get("c").getOrElse("etc/collector.properties")); } - private Map loadJsonSchemas() { - return jsonSchema().toMap().entrySet().stream() - .map(this::readSchemaForVersion) - .collect(HashMap.collector()); - } - - private Tuple2 readSchemaForVersion(java.util.Map.Entry versionToFilePath) { - try { - String schemaContent = new String( - readAllBytes(Paths.get(versionToFilePath.getValue().toString()))); - JsonNode schemaNode = JsonLoader.fromString(schemaContent); - JsonSchema schema = JsonSchemaFactory.byDefault().getJsonSchema(schemaNode); - return Tuple(versionToFilePath.getKey(), schema); - } catch (IOException | ProcessingException e) { - throw new ApplicationException("Could not read schema from path: " + versionToFilePath.getValue(), e); - } - } - private Map prepareUsersMap(@Nullable String allowedUsers) { return allowedUsers == null ? HashMap.empty() : List.of(allowedUsers.split("\\|")) @@ -212,11 +187,6 @@ public class ApplicationSettings { .toMap(t-> t[0].trim(), t -> t[1].trim()); } - private JSONObject jsonSchema() { - return new JSONObject(properties.getString("collector.schema.file", - format("{\"%s\":\"etc/CommonEventFormat_28.4.1.json\"}", FALLBACK_VES_VERSION))); - } - private Map convertDMaaPStreamsPropertyToMap(String streamIdsProperty) { java.util.HashMap domainToStreamIdsMapping = new java.util.HashMap<>(); String[] topics = streamIdsProperty.split("\\|"); diff --git a/src/main/java/org/onap/dcae/JSonSchemasSupplier.java b/src/main/java/org/onap/dcae/JSonSchemasSupplier.java new file mode 100644 index 00000000..d7cd5663 --- /dev/null +++ b/src/main/java/org/onap/dcae/JSonSchemasSupplier.java @@ -0,0 +1,58 @@ +/* + * ============LICENSE_START======================================================= + * PROJECT + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2019 Nokia. All rights reserved.s + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.dcae; + +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import io.vavr.Tuple2; +import io.vavr.collection.HashMap; +import io.vavr.collection.Map; +import org.json.JSONObject; + +import java.io.IOException; +import java.nio.file.Paths; + +import static io.vavr.API.Tuple; +import static java.nio.file.Files.readAllBytes; + +class JSonSchemasSupplier { + + public Map loadJsonSchemas(String collectorSchemaFile) { + JSONObject jsonObject = new JSONObject(collectorSchemaFile); + return jsonObject.toMap().entrySet().stream() + .map(JSonSchemasSupplier::readSchemaForVersion) + .collect(HashMap.collector()); + } + + + private static Tuple2 readSchemaForVersion(java.util.Map.Entry versionToFilePath) { + try { + String schemaContent = new String( + readAllBytes(Paths.get(versionToFilePath.getValue().toString()))); + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(); + JsonSchema schema = factory.getSchema(schemaContent); + + return Tuple(versionToFilePath.getKey(), schema); + } catch (IOException e) { + throw new ApplicationException("Could not read schema from path: " + versionToFilePath.getValue(), e); + } + } +} diff --git a/src/main/java/org/onap/dcae/restapi/EventValidator.java b/src/main/java/org/onap/dcae/restapi/EventValidator.java index f119b507..009247c0 100644 --- a/src/main/java/org/onap/dcae/restapi/EventValidator.java +++ b/src/main/java/org/onap/dcae/restapi/EventValidator.java @@ -20,23 +20,15 @@ */ package org.onap.dcae.restapi; -import static java.util.stream.StreamSupport.stream; - -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import java.util.Optional; import org.json.JSONObject; -import org.onap.dcae.ApplicationException; import org.onap.dcae.ApplicationSettings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.ResponseEntity; - +import java.util.Optional; public class EventValidator { - private static final Logger log = LoggerFactory.getLogger(EventValidator.class); - + private final SchemaValidator schemaValidator = new SchemaValidator(); private ApplicationSettings applicationSettings; public EventValidator(ApplicationSettings applicationSettings) { @@ -46,7 +38,7 @@ public class EventValidator { public Optional> validate(JSONObject jsonObject, String type, String version){ if (applicationSettings.jsonSchemaValidationEnabled()) { if (jsonObject.has(type)) { - if (!conformsToSchema(jsonObject, version)) { + if (!schemaValidator.conformsToSchema(jsonObject, applicationSettings.jsonSchema(version))) { return errorResponse(ApiException.SCHEMA_VALIDATION_FAILED); } } else { @@ -56,21 +48,6 @@ public class EventValidator { return Optional.empty(); } - private boolean conformsToSchema(JSONObject payload, String version) { - try { - JsonSchema schema = applicationSettings.jsonSchema(version); - ProcessingReport report = schema.validate(JsonLoader.fromString(payload.toString())); - if (report.isSuccess()) { - return true; - } - log.warn("Schema validation failed for event: " + payload); - stream(report.spliterator(), false).forEach(e -> log.warn(e.getMessage())); - return false; - } catch (Exception e) { - throw new ApplicationException("Unable to validate against schema", e); - } - } - private Optional> errorResponse(ApiException noServerResources) { return Optional.of(ResponseEntity.status(noServerResources.httpStatusCode) .body(noServerResources.toJSON().toString())); diff --git a/src/main/java/org/onap/dcae/restapi/SchemaValidator.java b/src/main/java/org/onap/dcae/restapi/SchemaValidator.java new file mode 100644 index 00000000..94638071 --- /dev/null +++ b/src/main/java/org/onap/dcae/restapi/SchemaValidator.java @@ -0,0 +1,58 @@ +/* + * ============LICENSE_START======================================================= + * PROJECT + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2019 Nokia. All rights reserved.s + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.dcae.restapi; + + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.ValidationMessage; +import org.json.JSONObject; +import org.onap.dcae.ApplicationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Set; + +class SchemaValidator { + public static final Logger log = LoggerFactory.getLogger(SchemaValidator.class); + + public boolean conformsToSchema(JSONObject payload, JsonSchema schema) { + try { + ObjectMapper mapper = new ObjectMapper(); + + String content = payload.toString(); + JsonNode node = mapper.readTree(content); + Set messageSet = schema.validate(node); + + if (messageSet.isEmpty()) { + return true; + } + + log.warn("Schema validation failed for event: " + payload); + messageSet.stream().forEach(it->log.warn(it.getMessage()) ); + + return false; + } catch (Exception e) { + throw new ApplicationException("Unable to validate against schema", e); + } + } +} diff --git a/src/test/java/org/onap/dcae/ApplicationSettingsTest.java b/src/test/java/org/onap/dcae/ApplicationSettingsTest.java index 6b0023f8..c4ba586e 100644 --- a/src/test/java/org/onap/dcae/ApplicationSettingsTest.java +++ b/src/test/java/org/onap/dcae/ApplicationSettingsTest.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * org.onap.dcaegen2.collectors.ves * ================================================================================ - * Copyright (C) 2018 Nokia. All rights reserved. + * Copyright (C) 2018 - 2019 Nokia. All rights reserved. * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,8 +32,7 @@ import static org.onap.dcae.TestingUtilities.createTemporaryFile; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.main.JsonSchema; +import com.networknt.schema.JsonSchema; import io.vavr.collection.HashMap; import io.vavr.collection.Map; import java.io.File; @@ -47,6 +46,13 @@ import org.junit.Test; public class ApplicationSettingsTest { + private static final String SAMPLE_JSON_SCHEMA = "{" + + " \"type\": \"object\"," + + " \"properties\": {" + + " \"state\": { \"type\": \"string\" }" + + " }" + + "}"; + @Test public void shouldMakeApplicationSettingsOutOfCLIArguments() { // given @@ -254,15 +260,9 @@ public class ApplicationSettingsTest { } @Test - public void shouldReturnJSONSchema() throws IOException, ProcessingException { + public void shouldReportValidateJSONSchemaErrorWhenJsonContainsIntegerValueNotString() throws IOException { // when - String sampleJsonSchema = "{" - + " \"type\": \"object\"," - + " \"properties\": {" - + " \"state\": { \"type\": \"string\" }" - + " }" - + "}"; - Path temporarySchemaFile = createTemporaryFile(sampleJsonSchema); + Path temporarySchemaFile = createTemporaryFile(SAMPLE_JSON_SCHEMA); // when JsonSchema schema = fromTemporaryConfiguration( @@ -271,9 +271,25 @@ public class ApplicationSettingsTest { // then JsonNode incorrectTestObject = new ObjectMapper().readTree("{ \"state\": 1 }"); + + assertFalse(schema.validate(incorrectTestObject).isEmpty()); + + } + + @Test + public void shouldDoNotReportAnyValidateJSONSchemaError() throws IOException { + // when + Path temporarySchemaFile = createTemporaryFile(SAMPLE_JSON_SCHEMA); + + // when + JsonSchema schema = fromTemporaryConfiguration( + String.format("collector.schema.file={\"v1\": \"%s\"}", temporarySchemaFile)) + .jsonSchema("v1"); + + // then JsonNode correctTestObject = new ObjectMapper().readTree("{ \"state\": \"hi\" }"); - assertFalse(schema.validate(incorrectTestObject).isSuccess()); - assertTrue(schema.validate(correctTestObject).isSuccess()); + ; + assertTrue(schema.validate(correctTestObject).isEmpty()); } @Test diff --git a/src/test/java/org/onap/dcae/restapi/EventValidatorTest.java b/src/test/java/org/onap/dcae/restapi/EventValidatorTest.java index 5bacd31c..4ac3c487 100644 --- a/src/test/java/org/onap/dcae/restapi/EventValidatorTest.java +++ b/src/test/java/org/onap/dcae/restapi/EventValidatorTest.java @@ -20,9 +20,8 @@ package org.onap.dcae.restapi; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.main.JsonSchemaFactory; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; import org.json.JSONObject; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -84,7 +83,7 @@ public class EventValidatorTest { } @Test - public void shouldReturnSchemaValidationFailedErrorOnInvalidJsonObjectSchema() throws ProcessingException, IOException { + public void shouldReturnSchemaValidationFailedErrorOnInvalidJsonObjectSchema() throws IOException { //given String schemaRejectingEverything = "{\"not\":{}}"; mockJsonSchema(schemaRejectingEverything); @@ -98,7 +97,7 @@ public class EventValidatorTest { } @Test - public void shouldReturnEmptyOptionalOnValidJsonObjectSchema() throws ProcessingException, IOException { + public void shouldReturnEmptyOptionalOnValidJsonObjectSchema() throws IOException { //given String schemaAcceptingEverything = "{}"; mockJsonSchema(schemaAcceptingEverything); @@ -111,10 +110,11 @@ public class EventValidatorTest { assertEquals(Optional.empty(), result); } - private void mockJsonSchema(String jsonSchema) throws IOException, ProcessingException { - when(settings.jsonSchema(any())).thenReturn( - JsonSchemaFactory.byDefault() - .getJsonSchema(JsonLoader.fromString(jsonSchema))); + private void mockJsonSchema(String jsonSchemaContent) { + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(); + + JsonSchema schema = factory.getSchema(jsonSchemaContent); + when(settings.jsonSchema(any())).thenReturn(schema); } private Optional> generateResponseOptional(ApiException schemaValidationFailed) { diff --git a/version.properties b/version.properties index 3f9d877d..d8977634 100644 --- a/version.properties +++ b/version.properties @@ -1,6 +1,6 @@ major=1 minor=5 -patch=2 +patch=3 base_version=${major}.${minor}.${patch} release_version=${base_version} snapshot_version=${base_version}-SNAPSHOT -- cgit 1.2.3-korg