Add resiliency test for the GraphChecker

- introduce IntegrationTest class for Junit 5 based tests that use the spring context
- add resiliency test to test GraphChecker behaviour for interrupted network connections to cassandra
- leverage testcontainers to run against a cassandra container and toxiproxy

Issue-ID: AAI-4070
Change-Id: I2485b5240950ded6201942459f6fcdb4971e43d9
Signed-off-by: Fiete Ostkamp <Fiete.Ostkamp@telekom.de>

9 files changed, 268 insertions, 35 deletions

diff --git a/aai-core/pom.xml b/aai-core/pom.xml
index 8adda4b8..29b8cf90 100644
--- a/aai-core/pom.xml
+++ b/aai-core/pom.xml
@@ -33,8 +33,6 @@ limitations under the License.
 	<name>aai-core</name>
 	<packaging>jar</packaging>
 	<properties>
-		<springframework.version>4.3.24.RELEASE</springframework.version>
-
 		<jacoco.line.coverage.limit>0.50</jacoco.line.coverage.limit>
 		<mockito.core.version>3.4.0</mockito.core.version>
 		<!-- Start of Default ONAP Schema Properties -->
@@ -60,7 +58,6 @@ limitations under the License.
 		<profile>
 			<id>onap</id>
 			<properties>
-				<springframework.version>4.3.24.RELEASE</springframework.version>
 				<aai.release>onap</aai.release>
 				<schema.configuration.location>N/A</schema.configuration.location>
 				<schema.nodes.location>aai-schema/src/main/resources/${aai.release}/oxm</schema.nodes.location>
@@ -190,6 +187,11 @@ limitations under the License.
 			<scope>test</scope>
 		</dependency>
 		<dependency>
+			<groupId>org.janusgraph</groupId>
+			<artifactId>janusgraph-cql</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
 			<groupId>com.fasterxml.jackson.jaxrs</groupId>
 			<artifactId>jackson-jaxrs-json-provider</artifactId>
 		</dependency>
@@ -374,6 +376,36 @@ limitations under the License.
 			<artifactId>lombok</artifactId>
 			<scope>provided</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.testcontainers</groupId>
+			<artifactId>testcontainers</artifactId>
+			<version>${testcontainers.version}</version>
+			<scope>test</scope>
+			<exclusions>
+					<exclusion>
+							<groupId>junit</groupId>
+							<artifactId>junit</artifactId>
+					</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.testcontainers</groupId>
+			<artifactId>junit-jupiter</artifactId>
+			<version>${testcontainers.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+				<groupId>org.testcontainers</groupId>
+				<artifactId>cassandra</artifactId>
+				<version>${testcontainers.version}</version>
+				<scope>test</scope>
+		</dependency>
+		<dependency>
+				<groupId>org.testcontainers</groupId>
+				<artifactId>toxiproxy</artifactId>
+				<version>${testcontainers.version}</version>
+				<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 	<!-- Plugins and repositories -->
diff --git a/aai-core/src/main/java/org/onap/aai/config/GraphConfig.java b/aai-core/src/main/java/org/onap/aai/config/GraphConfig.java
index 352821f5..85feabb5 100644
--- a/aai-core/src/main/java/org/onap/aai/config/GraphConfig.java
+++ b/aai-core/src/main/java/org/onap/aai/config/GraphConfig.java
@@ -22,8 +22,6 @@ package org.onap.aai.config;
 
 import java.io.FileNotFoundException;
 
-import javax.annotation.PreDestroy;
-
 import org.apache.commons.configuration2.ex.ConfigurationException;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.structure.Graph;
diff --git a/aai-core/src/main/java/org/onap/aai/util/GraphChecker.java b/aai-core/src/main/java/org/onap/aai/util/GraphChecker.java
index 43caebc5..35cc832f 100644
--- a/aai-core/src/main/java/org/onap/aai/util/GraphChecker.java
+++ b/aai-core/src/main/java/org/onap/aai/util/GraphChecker.java
@@ -42,7 +42,6 @@ public class GraphChecker {
      *         <li>false, if database is NOT available</li>
      */
     public boolean isAaiGraphDbAvailable() {
-        boolean dbAvailable;
         JanusGraphTransaction transaction = null;
         try {
             // disable caching and other features that are not needed for this check
@@ -53,41 +52,20 @@ public class GraphChecker {
                 .vertexCacheSize(0)
                 .skipDBCacheRead()
                 .start();
-            dbAvailable = transaction.traversal().V().limit(1).hasNext();
-        } catch (JanusGraphException e) {
-            String message = "Database is not available (after JanusGraph exception)";
-            ErrorLogHelper.logError("500", message + ": " + e.getMessage());
-            log.error(message, e);
-            dbAvailable = false;
-        } catch (Error e) {
-            // Following error occurs when aai resources is starting:
-            // - UnsatisfiedLinkError (for org.onap.aai.dbmap.AAIGraph$Helper instantiation)
-            // Following errors are raised when aai resources is starting and cassandra is not
-            // running:
-            // - ExceptionInInitializerError
-            // - NoClassDefFoundError (definition for org.onap.aai.dbmap.AAIGraph$Helper is not
-            // found)
-            String message = "Database is not available (after error)";
-            ErrorLogHelper.logError("500", message + ": " + e.getMessage());
-            log.error(message, e);
-            dbAvailable = false;
-        } catch (Exception e) {
-            String message = "Database availability can not be determined";
-            ErrorLogHelper.logError("500", message + ": " + e.getMessage());
-            log.error(message, e);
-            dbAvailable = false;
+            transaction.traversal().V().limit(1).hasNext(); // if this is not throwing an exception, the database is available
+            return true;
+        } catch (Throwable e) {
+            log.error("Database is not available: ", e);
+            return false;
         } finally {
             if (transaction != null && !transaction.isClosed()) {
                 // check if transaction is open then closed instead of flag
                 try {
                     transaction.rollback();
                 } catch (Exception e) {
-                    String message = "Exception occurred while closing transaction";
-                    log.error(message, e);
-                    ErrorLogHelper.logError("500", message + ": " + e.getMessage());
+                    log.error("Exception occurred while closing transaction", e);
                 }
             }
         }
-        return dbAvailable;
     }
 }
diff --git a/aai-core/src/test/java/org/onap/aai/AAISetup.java b/aai-core/src/test/java/org/onap/aai/AAISetup.java
index 5273c6e7..e280d889 100644
--- a/aai-core/src/test/java/org/onap/aai/AAISetup.java
+++ b/aai-core/src/test/java/org/onap/aai/AAISetup.java
@@ -53,7 +53,7 @@ import org.springframework.test.context.web.WebAppConfiguration;
         classes = {ConfigConfiguration.class, AAIConfigTranslator.class, EdgeIngestor.class, EdgeSerializer.class,
                 NodeIngestor.class, SpringContextAware.class, IntrospectionConfig.class, RestBeanConfig.class,
                 XmlFormatTransformerConfiguration.class, ValidationService.class, ValidationConfiguration.class,
-                KafkaConfig.class, LoaderFactory.class, NotificationService.class, KafkaConfig.class})
+                KafkaConfig.class, LoaderFactory.class, NotificationService.class})
 @TestPropertySource(
         properties = {"schema.uri.base.path = /aai", "schema.xsd.maxoccurs = 5000", "schema.translator.list=config",
                 "schema.nodes.location=src/test/resources/onap/oxm",
diff --git a/aai-core/src/test/java/org/onap/aai/IntegrationTest.java b/aai-core/src/test/java/org/onap/aai/IntegrationTest.java
new file mode 100644
index 00000000..9ee5fce5
--- /dev/null
+++ b/aai-core/src/test/java/org/onap/aai/IntegrationTest.java
@@ -0,0 +1,61 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2024 Deutsche Telekom. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.aai;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.onap.aai.config.ConfigConfiguration;
+import org.onap.aai.config.GraphConfig;
+import org.onap.aai.config.IntrospectionConfig;
+import org.onap.aai.config.KafkaConfig;
+import org.onap.aai.config.RestBeanConfig;
+import org.onap.aai.config.SpringContextAware;
+import org.onap.aai.config.XmlFormatTransformerConfiguration;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.nodes.NodeIngestor;
+import org.onap.aai.prevalidation.ValidationConfiguration;
+import org.onap.aai.prevalidation.ValidationService;
+import org.onap.aai.rest.notification.NotificationService;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.setup.AAIConfigTranslator;
+import org.onap.aai.util.GraphChecker;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+@ExtendWith(SpringExtension.class)
+@ContextConfiguration(
+        classes = {ConfigConfiguration.class, AAIConfigTranslator.class, EdgeIngestor.class, EdgeSerializer.class,
+                NodeIngestor.class, SpringContextAware.class, IntrospectionConfig.class, RestBeanConfig.class,
+                XmlFormatTransformerConfiguration.class, ValidationService.class, ValidationConfiguration.class,
+                KafkaConfig.class, LoaderFactory.class, NotificationService.class})
+@TestPropertySource(
+        value = "classpath:/application.properties",
+        properties = {
+                "schema.uri.base.path = /aai", "schema.xsd.maxoccurs = 5000",
+                "schema.translator.list=config",
+                "schema.nodes.location=src/test/resources/onap/oxm",
+                "schema.edges.location=src/test/resources/onap/dbedgerules",
+                "aai.notifications.enabled=false","classpath:/application.properties",
+        })
+public class IntegrationTest {
+
+}
diff --git a/aai-core/src/test/java/org/onap/aai/JanusgraphCassandraConfiguration.java b/aai-core/src/test/java/org/onap/aai/JanusgraphCassandraConfiguration.java
new file mode 100644
index 00000000..08c99170
--- /dev/null
+++ b/aai-core/src/test/java/org/onap/aai/JanusgraphCassandraConfiguration.java
@@ -0,0 +1,34 @@
+package org.onap.aai;
+
+import java.io.FileNotFoundException;
+
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
+import org.janusgraph.core.JanusGraphProperty;
+import org.janusgraph.core.schema.JanusGraphConfiguration;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+
+@TestConfiguration
+public class JanusgraphCassandraConfiguration {
+
+  @Value("${testcontainers.cassandra.host}")
+  String cassandraHost;
+
+  @Value("${testcontainers.cassandra.port}")
+  int cassandraPort;
+
+  @Bean
+  public org.apache.commons.configuration2.Configuration getGraphProperties()
+      throws FileNotFoundException, ConfigurationException {
+
+    Configuration janusgraphConfiguration = new PropertiesConfiguration();
+    janusgraphConfiguration.addProperty("storage.backend", "cql");
+    janusgraphConfiguration.addProperty("storage.hostname", cassandraHost);
+    janusgraphConfiguration.addProperty("storage.port", cassandraPort);
+    return janusgraphConfiguration;
+  }
+}
diff --git a/aai-core/src/test/java/org/onap/aai/util/GraphCheckerResiliencyTest.java b/aai-core/src/test/java/org/onap/aai/util/GraphCheckerResiliencyTest.java
new file mode 100644
index 00000000..03662a93
--- /dev/null
+++ b/aai-core/src/test/java/org/onap/aai/util/GraphCheckerResiliencyTest.java
@@ -0,0 +1,120 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2024 Deutsche Telekom. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.aai.util;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.onap.aai.IntegrationTest;
+import org.onap.aai.JanusgraphCassandraConfiguration;
+import org.onap.aai.config.GraphConfig;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Import;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.DynamicPropertyRegistry;
+import org.springframework.test.context.DynamicPropertySource;
+import org.testcontainers.cassandra.CassandraContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.ToxiproxyContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import eu.rekawek.toxiproxy.Proxy;
+import eu.rekawek.toxiproxy.ToxiproxyClient;
+import eu.rekawek.toxiproxy.model.ToxicDirection;
+
+@Testcontainers
+@Import(JanusgraphCassandraConfiguration.class)
+@ContextConfiguration(classes = {
+  GraphConfig.class, GraphChecker.class
+})
+public class GraphCheckerResiliencyTest extends IntegrationTest {
+
+  private static final Network network = Network.newNetwork();
+
+  @Container
+  private static final CassandraContainer cassandraContainer = new CassandraContainer("cassandra:4.0.5")
+      .withExposedPorts(9042)
+      .withNetwork(network)
+      .withNetworkAliases("cassandra");
+
+
+  @Container
+  private static final ToxiproxyContainer toxiproxy = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:2.5.0")
+      .withNetwork(network);
+
+  private static Proxy cassandraProxy;
+
+  @Autowired
+  GraphChecker graphChecker;
+
+  @Autowired
+  GraphTraversalSource g;
+
+  @DynamicPropertySource
+  static void setProperties(DynamicPropertyRegistry registry) throws IOException {
+    registry.add("testcontainers.cassandra.host", () -> toxiproxy.getHost());
+    registry.add("testcontainers.cassandra.port", () -> toxiproxy.getMappedPort(8666));
+
+    var toxiproxyClient = new ToxiproxyClient(toxiproxy.getHost(), toxiproxy.getControlPort());
+
+    cassandraProxy = toxiproxyClient.createProxy("cassandra", "0.0.0.0:8666", "cassandra:9042");
+
+  }
+
+  @BeforeEach
+  void resetProxy() throws IOException {
+    for(var toxic: cassandraProxy.toxics().getAll()) {
+      toxic.remove();
+    }
+  }
+
+  @Test
+  public void test() {
+    boolean available = graphChecker.isAaiGraphDbAvailable();
+    assertTrue(available);
+  }
+
+  @Test
+  public void testConnectionFailure() throws IOException {
+    assertTrue(graphChecker.isAaiGraphDbAvailable());
+
+    cassandraProxy.toxics().bandwidth("no-connection-up", ToxicDirection.UPSTREAM, 0);
+    cassandraProxy.toxics().bandwidth("no-connection-down", ToxicDirection.DOWNSTREAM, 0);
+    assertFalse(graphChecker.isAaiGraphDbAvailable());
+
+    // boolean available = graphChecker.isAaiGraphDbAvailable();
+    // // g.addV().property("foo","bar").next();
+    // // boolean exists = g.V().has("foo","bar").hasNext();
+    // assertFalse(available);
+  }
+
+  @AfterAll
+  public static void tearDown() {
+    cassandraContainer.stop();
+  }
+}
diff --git a/aai-core/src/test/java/org/onap/aai/util/GraphCheckerTest.java b/aai-core/src/test/java/org/onap/aai/util/GraphCheckerTest.java
index 03634c8c..524a2704 100644
--- a/aai-core/src/test/java/org/onap/aai/util/GraphCheckerTest.java
+++ b/aai-core/src/test/java/org/onap/aai/util/GraphCheckerTest.java
@@ -8,6 +8,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphException;
 import org.janusgraph.core.JanusGraphTransaction;
 import org.janusgraph.core.TransactionBuilder;
 import org.junit.Before;
@@ -47,10 +48,18 @@ public class GraphCheckerTest extends AAISetup {
     boolean available = graphChecker.isAaiGraphDbAvailable();
     assertTrue(available);
   }
+
   @Test
-  public void thatAvailabilityCanBeFalse() {
+  public void thatEmptyDBQueryIsInterpretedAsAvailable() {
     when(traversal.hasNext()).thenReturn(false);
     boolean available = graphChecker.isAaiGraphDbAvailable();
+    assertTrue(available);
+  }
+
+  @Test
+  public void thatExceptionWillReturnAvailabilityFalse() {
+    when(traversal.hasNext()).thenThrow(JanusGraphException.class);
+    boolean available = graphChecker.isAaiGraphDbAvailable();
     assertFalse(available);
   }
 
diff --git a/aai-parent/pom.xml b/aai-parent/pom.xml
index ff2b4232..e45c009f 100644
--- a/aai-parent/pom.xml
+++ b/aai-parent/pom.xml
@@ -102,6 +102,7 @@ limitations under the License.
     <reflections.version>0.9.10</reflections.version>
     <snakeyaml.version>1.29</snakeyaml.version>
     <lombok.version>1.18.34</lombok.version>
+    <testcontainers.version>1.20.4</testcontainers.version>
 
     <javax.servlet.version>3.1.0</javax.servlet.version>
     <javax.annotation.version>1.2</javax.annotation.version>