diff options
46 files changed, 3700 insertions, 1075 deletions
diff --git a/mdbc-benchmark/pom.xml b/mdbc-benchmark/pom.xml index 84b7e56..a12e4a2 100644 --- a/mdbc-benchmark/pom.xml +++ b/mdbc-benchmark/pom.xml @@ -1,4 +1,3 @@ -<?xml version="1.0" encoding="UTF-8"?> <!-- Copyright (c) 2014, Oracle America, Inc. All rights reserved. @@ -30,72 +29,171 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <parent> - <artifactId>mdbc</artifactId> - <groupId>org.onap.music.mdbc</groupId> - <version>0.0.1-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>mdbc</artifactId> + <groupId>org.onap.music.mdbc</groupId> + <version>0.0.1-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> - <artifactId>benchmark</artifactId> - <packaging>jar</packaging> + <artifactId>mdbc-benchmark</artifactId> + <version>0.0.1-SNAPSHOT</version> + <name>mdbc-benchmark</name> + <description>MDBC Benchmark</description> + <packaging>jar</packaging> - <name>JMH benchmark sample: Java</name> + <dependencies> + <dependency> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + <version>20160810</version> + </dependency> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-core</artifactId> + <version>${jmh.version}</version> + </dependency> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-generator-annprocess</artifactId> + <version>${jmh.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.att.eelf</groupId> + <artifactId>eelf-core</artifactId> + <version>1.0.0</version> + </dependency> + <dependency> + <groupId>org.apache.calcite.avatica</groupId> + <artifactId>avatica-server</artifactId> + <version>1.12.0</version> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.8.5</version> + </dependency> + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <version>1.1.7</version> + </dependency> + <dependency> + <groupId>postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>9.1-901-1.jdbc4</version> + </dependency> + <dependency> + <groupId>com.datastax.cassandra</groupId> + <artifactId>cassandra-driver-core</artifactId> + <version>3.3.0</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>19.0</version> + </dependency> + <dependency> + <!-- The oparent dependency managed version 9.4.12.v20180830 will not work --> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <version>9.2.19.v20160908</version> + </dependency> + <dependency> + <!-- The oparent dependency managed version 9.4.12.v20180830 will not work --> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>9.2.19.v20160908</version> + </dependency> + <dependency> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-security</artifactId> + <version>9.2.19.v20160908</version> + </dependency> + <dependency> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + <version>9.2.19.v20160908</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.7</version> + </dependency> + <!-- These two dependencies pull in optional libraries for Cassandra --> + <dependency> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + <version>1.3.0</version> + </dependency> + <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <version>1.1.2.6</version> + </dependency> + <dependency> + <groupId>com.github.jsqlparser</groupId> + <artifactId>jsqlparser</artifactId> + <version>1.1</version> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-core</artifactId> + <version>1.2.3</version> + </dependency> - <!-- - This is the demo/sample template build script for building Java benchmarks with JMH. - Edit as needed. - --> + <!-- https://mvnrepository.com/artifact/javax.servlet/servlet-api --> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.5</version> + <scope>provided</scope> + </dependency> + <!-- https://mvnrepository.com/artifact/org.apache.calcite.avatica/avatica-server --> + <dependency> + <groupId>org.apache.calcite.avatica</groupId> + <artifactId>avatica-server</artifactId> + <version>1.12.0</version> + </dependency> + <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-plus --> + <dependency> + <groupId>org.apache.calcite</groupId> + <artifactId>calcite-plus</artifactId> + <version>1.12.0</version> + </dependency> + <dependency> + <groupId>com.beust</groupId> + <artifactId>jcommander</artifactId> + <version>1.72</version> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.8.5</version> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-handler</artifactId> + <version>4.1.30.Final</version> + </dependency> + <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java --> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + <version>3.6.1</version> + </dependency> + </dependencies> - <dependencies> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <version>20160810</version> - </dependency> - <dependency> - <groupId>org.openjdk.jmh</groupId> - <artifactId>jmh-core</artifactId> - <version>${jmh.version}</version> - </dependency> - <dependency> - <groupId>org.openjdk.jmh</groupId> - <artifactId>jmh-generator-annprocess</artifactId> - <version>${jmh.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.att.eelf</groupId> - <artifactId>eelf-core</artifactId> - <version>1.0.0</version> - </dependency> - <dependency> - <groupId>org.apache.calcite.avatica</groupId> - <artifactId>avatica-server</artifactId> - <version>1.12.0</version> - </dependency> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <version>2.8.5</version> - </dependency> - <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - <version>1.1.7</version> - </dependency> - <dependency> - <groupId>postgresql</groupId> - <artifactId>postgresql</artifactId> - <version>9.1-901-1.jdbc4</version> - </dependency> - </dependencies> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - - <!-- + <!-- JMH version to use with this project. --> <jmh.version>1.21</jmh.version> @@ -108,98 +206,100 @@ THE POSSIBILITY OF SUCH DAMAGE. <!-- Name of the benchmark Uber-JAR to generate. --> - <uberjar.name>benchmarks</uberjar.name> + <!-- + Name of the benchmark Uber-JAR to generate. + --> + <uberjar.name>benchmarks</uberjar.name> </properties> <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.1</version> - <configuration> - <compilerVersion>${javac.target}</compilerVersion> - <source>${javac.target}</source> - <target>${javac.target}</target> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>2.2</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <finalName>${uberjar.name}</finalName> - <transformers> - <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>org.openjdk.jmh.Main</mainClass> - </transformer> - </transformers> - <filters> - <filter> - <!-- - Shading signed JARs will fail without this. - http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar - --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - <pluginManagement> - <plugins> - <plugin> - <artifactId>maven-clean-plugin</artifactId> - <version>2.5</version> - </plugin> - <plugin> - <artifactId>maven-deploy-plugin</artifactId> - <version>2.8.1</version> - </plugin> - <plugin> - <artifactId>maven-install-plugin</artifactId> - <version>2.5.1</version> - </plugin> - <plugin> - <artifactId>maven-jar-plugin</artifactId> - <version>2.4</version> - </plugin> - <plugin> - <artifactId>maven-javadoc-plugin</artifactId> - <version>2.9.1</version> - </plugin> - <plugin> - <artifactId>maven-resources-plugin</artifactId> - <version>2.6</version> - </plugin> - <plugin> - <artifactId>maven-site-plugin</artifactId> - <version>3.3</version> - </plugin> - <plugin> - <artifactId>maven-source-plugin</artifactId> - <version>2.2.1</version> - </plugin> - <plugin> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.17</version> - </plugin> - </plugins> - </pluginManagement> - </build> - + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.1</version> + <configuration> + <compilerVersion>${javac.target}</compilerVersion> + <source>${javac.target}</source> + <target>${javac.target}</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.2</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <finalName>${uberjar.name}</finalName> + <transformers> + <transformer + implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.openjdk.jmh.Main</mainClass> + </transformer> + </transformers> + <filters> + <filter> + <!-- + Shading signed JARs will fail without this. + http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar + --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + <pluginManagement> + <plugins> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <version>2.5</version> + </plugin> + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8.1</version> + </plugin> + <plugin> + <artifactId>maven-install-plugin</artifactId> + <version>2.5.1</version> + </plugin> + <plugin> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + </plugin> + <plugin> + <artifactId>maven-javadoc-plugin</artifactId> + <version>2.9.1</version> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + </plugin> + <plugin> + <artifactId>maven-site-plugin</artifactId> + <version>3.3</version> + </plugin> + <plugin> + <artifactId>maven-source-plugin</artifactId> + <version>2.2.1</version> + </plugin> + <plugin> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.17</version> + </plugin> + </plugins> + </pluginManagement> +</build> </project> diff --git a/mdbc-benchmark/src/main/java/org/onap/music/mdbc/BenchmarkUtils.java b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/BenchmarkUtils.java new file mode 100644 index 0000000..06f2a95 --- /dev/null +++ b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/BenchmarkUtils.java @@ -0,0 +1,245 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ +package org.onap.music.mdbc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +public class BenchmarkUtils { + public static final String driver = "org.apache.calcite.avatica.remote.Driver"; + public static final String mariaDriver = "org.mariadb.jdbc.Driver"; + public static final String postgresDriver = "org.postgresql.Driver"; + public static String TABLE; + public static String updateBuilder; + + public enum ExecutionType { + MARIA_DB, COCKROACH_DB, METRIC,POSTGRES + } + + public static void SetupTable(String table){ + TABLE=table; + updateBuilder = new StringBuilder() + .append("UPDATE ") + .append(table) + .append(" SET Counter = Counter + 1,") + .append("City = 'Sandy Springs'") + .append(";").toString(); + } + + public static void setupCreateTables(Connection connection){ + createTable(connection); + try { + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static void setupCreateRows(Connection testConnection, int rows){ + //Empty database + boolean cleanResult = BenchmarkUtils.cleanTable(testConnection); + //Add new lines + BenchmarkUtils.addRowsToTable(Integer.valueOf(rows),testConnection); + + //Commit + try { + testConnection.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static Connection getConnection(String ip, ExecutionType type, String user, String password) { + final String connectionUrl = "jdbc:avatica:remote:url=http://" + ip + ":30000;serialization=protobuf"; + final String mariaConnectionUrl = "jdbc:mariadb://" + ip + ":3306/test"; + final String cockroachUrl = "jdbc:postgresql://" + ip + ":26257/test"; + final String postgresUrl = "jdbc:postgresql://" + ip + ":5432/test"; + + try { + switch (type) { + case MARIA_DB: + Class.forName(mariaDriver); + break; + case METRIC: + Class.forName(driver); + break; + case COCKROACH_DB: + case POSTGRES: + Class.forName(postgresDriver); + break; + } + } catch (ClassNotFoundException e) { + e.printStackTrace(); + System.exit(1); + } + Connection connection = null; + try { + if (type == ExecutionType.METRIC) { + connection = DriverManager.getConnection(connectionUrl); + } else { + Properties connectionProps = new Properties(); + connectionProps.put("user", user); + if (type == ExecutionType.COCKROACH_DB) { + connectionProps.setProperty("sslmode", "disable"); + } else { + connectionProps.put("password", password); + } + final String url = (type == ExecutionType.MARIA_DB) ? mariaConnectionUrl : + (type == ExecutionType.COCKROACH_DB)?cockroachUrl:postgresUrl; + connection = DriverManager.getConnection(url, connectionProps); + } + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + connection.setAutoCommit(false); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + return connection; + } + + public static void createTable(Connection connection) { + final String sql = "CREATE TABLE IF NOT EXISTS "+TABLE+" (\n" + + " PersonID int,\n" + + " Counter int,\n" + + " LastName varchar(255),\n" + + " FirstName varchar(255),\n" + + " Address varchar(255),\n" + + " City varchar(255),\n" + + " PRIMARY KEY(PersonID)"+ + ");"; + + Statement stmt = null; + try { + stmt = connection.createStatement(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + + Boolean execute = null; + try { + execute = stmt.execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + + try { + connection.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + + try { + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static boolean cleanTable(Connection testConnection) { + String cleanCmd = "DELETE FROM "+TABLE+";"; + Statement stmt = null; + try { + stmt = testConnection.createStatement(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + + Boolean execute = null; + try { + execute = stmt.execute(cleanCmd); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + testConnection.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + return execute; + } + + public static void addRowsToTable(int totalNumberOfRows, Connection testConnection) { + for (int i = 0; i < totalNumberOfRows; i++) { + final StringBuilder insertSQLBuilder = new StringBuilder() + .append("INSERT INTO "+TABLE+" VALUES (") + .append(i) + .append(", ") + .append(0) + .append(", '") + .append("Last-") + .append(i) + .append("', '") + .append("First-") + .append(i) + .append("', 'KACB', 'ATLANTA');"); + Statement stmt = null; + try { + stmt = testConnection.createStatement(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + + Boolean execute = null; + try { + execute = stmt.execute(insertSQLBuilder.toString()); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + try { + testConnection.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + +} diff --git a/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricBenchmark.java b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricBenchmark.java index 1fb584c..19db6f4 100644 --- a/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricBenchmark.java +++ b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricBenchmark.java @@ -2,7 +2,7 @@ * ============LICENSE_START==================================================== * org.onap.music.mdbc * ============================================================================= - * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. * ============================================================================= * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,40 +19,39 @@ */ package org.onap.music.mdbc; +import static org.onap.music.mdbc.BenchmarkUtils.setupCreateRows; +import static org.onap.music.mdbc.BenchmarkUtils.setupCreateTables; + +import org.onap.music.mdbc.BenchmarkUtils.ExecutionType; import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.Properties; import java.util.concurrent.TimeUnit; @BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class MetricBenchmark { - public StringBuilder updateBuilder = new StringBuilder() - .append("UPDATE PERSONS ") - .append("SET Counter = Counter + 1,") - .append("City = 'Sandy Springs'") - .append(";"); - public String update = updateBuilder.toString(); public static void main(String[] args) throws RunnerException { Options opt = new OptionsBuilder() .include(MetricBenchmark.class.getSimpleName()) .param("type", ExecutionType.METRIC.name()) + .forks(0) + .threads(1) .build(); new Runner(opt).run(); } @Benchmark - public boolean testMethod(MyState state) { + public boolean testMethod(MyState state, Blackhole blackhole) { Statement stmt = null; try { stmt = state.testConnection.createStatement(); @@ -63,7 +62,7 @@ public class MetricBenchmark { Boolean execute = null; try { - execute = stmt.execute(update); + execute = stmt.execute(state.update); } catch (SQLException e) { e.printStackTrace(); System.exit(1); @@ -81,15 +80,14 @@ public class MetricBenchmark { e.printStackTrace(); System.exit(1); } + blackhole.consume(execute); return execute; } - public static enum ExecutionType { - MARIA_DB, COCKROACH_DB, METRIC - } @State(Scope.Benchmark) public static class MyState { + public String update; public final String driver = "org.apache.calcite.avatica.remote.Driver"; public final String mariaDriver = "org.mariadb.jdbc.Driver"; public final String cockroachDriver = "org.postgresql.Driver"; @@ -97,205 +95,34 @@ public class MetricBenchmark { final String password = "metriccluster"; @Param({"104.209.240.219"}) public String ip; + @Param({"PERSONS"}) + public String table; @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) public int rows; - @Param({"MARIA_DB", "COCKROACH_DB", "METRIC"}) + @Param({"MARIA_DB", "COCKROACH_DB", "METRIC", "POSTGRES"}) public ExecutionType type; public Connection testConnection; private Connection createConnection() { - final String connectionUrl = "jdbc:avatica:remote:url=http://" + ip + ":30000;serialization=protobuf"; - final String mariaConnectionUrl = "jdbc:mariadb://" + ip + ":3306/test"; - final String cockroachUrl = "jdbc:postgresql://" + ip + ":26257/test"; - - try { - switch (type) { - case MARIA_DB: - Class.forName(this.mariaDriver); - break; - case METRIC: - Class.forName(this.driver); - break; - case COCKROACH_DB: - Class.forName(this.cockroachDriver); - break; - } - } catch (ClassNotFoundException e) { - e.printStackTrace(); - System.exit(1); - } - Connection connection = null; - try { - if (type == ExecutionType.METRIC) { - connection = DriverManager.getConnection(connectionUrl); - } else { - Properties connectionProps = new Properties(); - connectionProps.put("user", user); - if(type == ExecutionType.COCKROACH_DB){ - connectionProps.setProperty("sslmode", "disable"); - } - else{ - connectionProps.put("password", password); - } - final String url = (type == ExecutionType.MARIA_DB) ? mariaConnectionUrl : cockroachUrl; - connection = DriverManager.getConnection(url, connectionProps); - } - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - try { - connection.setAutoCommit(false); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - return connection; + return BenchmarkUtils.getConnection(ip,type,user,password); } - private void createTable(Connection connection) { - final String sql = "CREATE TABLE IF NOT EXISTS PERSONS (\n" + - " PersonID int,\n" + - " Counter int,\n" + - " LastName varchar(255),\n" + - " FirstName varchar(255),\n" + - " Address varchar(255),\n" + - " City varchar(255)\n" + - ");"; - - Statement stmt = null; - try { - stmt = connection.createStatement(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - - Boolean execute = null; - try { - execute = stmt.execute(sql); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - - try { - connection.commit(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - } - - private boolean cleanTable() { - String cleanCmd = "DELETE FROM PERSONS;"; - Statement stmt = null; - try { - stmt = testConnection.createStatement(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - - Boolean execute = null; - try { - execute = stmt.execute(cleanCmd); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - try { - testConnection.commit(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - return execute; - } - - private void addRowsToTable(int totalNumberOfRows) { - for (int i = 0; i < totalNumberOfRows; i++) { - final StringBuilder insertSQLBuilder = new StringBuilder() - .append("INSERT INTO PERSONS VALUES (") - .append(i) - .append(", ") - .append(0) - .append(", '") - .append("Last-") - .append(i) - .append("', '") - .append("First-") - .append(i) - .append("', 'KACB', 'ATLANTA');"); - Statement stmt = null; - try { - stmt = testConnection.createStatement(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - - Boolean execute = null; - try { - execute = stmt.execute(insertSQLBuilder.toString()); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - } - try { - testConnection.commit(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } + @Setup(Level.Trial) + public void doTrialSetup(){ + BenchmarkUtils.SetupTable(table); + update = BenchmarkUtils.updateBuilder; } @Setup(Level.Iteration) public void doSetup() { Connection connection = createConnection(); - createTable(connection); - try { - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } + setupCreateTables(connection); //Setup connection testConnection = createConnection(); - //Empty database - boolean cleanResult = cleanTable(); - //Add new lines - addRowsToTable(Integer.valueOf(rows)); + setupCreateRows(testConnection,rows); - //Commit - try { - testConnection.commit(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } } @TearDown(Level.Iteration) diff --git a/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricCommitBenchmark.java b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricCommitBenchmark.java new file mode 100644 index 0000000..803a162 --- /dev/null +++ b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricCommitBenchmark.java @@ -0,0 +1,141 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ +package org.onap.music.mdbc; + +import static org.onap.music.mdbc.BenchmarkUtils.setupCreateRows; +import static org.onap.music.mdbc.BenchmarkUtils.setupCreateTables; + +import org.onap.music.mdbc.BenchmarkUtils.ExecutionType; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricCommitBenchmark { + + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricBenchmark.class.getSimpleName()) + .param("type", ExecutionType.METRIC.name()) + .forks(1) + .threads(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void testMethod(MyState state) { + try { + state.testConnection.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + + @State(Scope.Benchmark) + public static class MyState { + + String update; + final String user = "root"; + final String password = "metriccluster"; + @Param({"104.209.240.219"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + @Param({"MARIA_DB", "COCKROACH_DB", "METRIC","POSTGRES"}) + public ExecutionType type; + @Param({"PERSONS"}) + public String table; + + public Connection testConnection; + + public Statement stmt; + + private Connection createConnection() { + return BenchmarkUtils.getConnection(ip,type,user,password); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + BenchmarkUtils.SetupTable(table); + update=BenchmarkUtils.updateBuilder; + } + + @Setup(Level.Invocation) + public void doInvocationSetup(){ + try { + stmt = testConnection.createStatement(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + Boolean execute = null; + try { + execute = stmt.execute(update); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @TearDown(Level.Invocation) + public void doInvocationTearDown(){ + try { + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @Setup(Level.Iteration) + public void doSetup() { + Connection connection = createConnection(); + setupCreateTables(connection); + //Setup connection + testConnection = createConnection(); + setupCreateRows(testConnection,rows); + } + + @TearDown(Level.Iteration) + public void doTearDown() { + System.out.println("Do TearDown"); + try { + testConnection.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + } +} diff --git a/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricStatementBenchmark.java b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricStatementBenchmark.java new file mode 100644 index 0000000..7dc950a --- /dev/null +++ b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricStatementBenchmark.java @@ -0,0 +1,144 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ +package org.onap.music.mdbc; + +import static org.onap.music.mdbc.BenchmarkUtils.setupCreateRows; +import static org.onap.music.mdbc.BenchmarkUtils.setupCreateTables; + +import org.onap.music.mdbc.BenchmarkUtils.ExecutionType; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricStatementBenchmark { + + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricBenchmark.class.getSimpleName()) + .param("type", ExecutionType.METRIC.name()) + .forks(1) + .threads(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public boolean testMethod(MyState state, Blackhole blackhole) { + Boolean execute = null; + try { + execute = state.stmt.execute(state.update); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + blackhole.consume(execute); + return execute; + } + + + @State(Scope.Benchmark) + public static class MyState { + + public String update; + final String user = "root"; + final String password = "metriccluster"; + @Param({"104.209.240.219"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + @Param({"MARIA_DB", "COCKROACH_DB", "METRIC","POSTGRES"}) + public ExecutionType type; + @Param({"PERSONS"}) + public String table; + + public Connection testConnection; + + public Statement stmt; + + private Connection createConnection() { + return BenchmarkUtils.getConnection(ip,type,user,password); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + BenchmarkUtils.SetupTable(table); + update=BenchmarkUtils.updateBuilder; + } + + @Setup(Level.Invocation) + public void doInvocationSetup(){ + try { + stmt = testConnection.createStatement(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @TearDown(Level.Invocation) + public void doInvocationTearDown(){ + try { + //TODO: check if state need to be consumed by blackhole to guarantee execution + testConnection.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @Setup(Level.Iteration) + public void doSetup() { + Connection connection = createConnection(); + setupCreateTables(connection); + //Setup connection + testConnection = createConnection(); + setupCreateRows(testConnection,rows); + } + + @TearDown(Level.Iteration) + public void doTearDown() { + System.out.println("Do TearDown"); + try { + testConnection.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + } +} diff --git a/mdbc-benchmark/src/main/resources/logback.xml b/mdbc-benchmark/src/main/resources/logback.xml index e4d0030..4215681 100755 --- a/mdbc-benchmark/src/main/resources/logback.xml +++ b/mdbc-benchmark/src/main/resources/logback.xml @@ -37,7 +37,7 @@ --> -<configuration scan="true" scanPeriod="3 seconds" debug="true"> +<configuration scan="false" scanPeriod="3 seconds" debug="true"> <!-- Logback files for the mdbc Driver "mdbc" are created in directory ${catalina.base}/logs/mdbc; diff --git a/mdbc-internal-benchmark/pom.xml b/mdbc-internal-benchmark/pom.xml new file mode 100644 index 0000000..b356638 --- /dev/null +++ b/mdbc-internal-benchmark/pom.xml @@ -0,0 +1,193 @@ +<!-- +Copyright (c) 2014, Oracle America, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Oracle nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>mdbc</artifactId> + <groupId>org.onap.music.mdbc</groupId> + <version>0.0.1-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>mdbc-own-benchmark</artifactId> + <packaging>jar</packaging> + + <name>JMH benchmark sample: Java</name> + + <!-- + This is the demo/sample template build script for building Java benchmarks with JMH. + Edit as needed. + --> + + <prerequisites> + <maven>3.0</maven> + </prerequisites> + + <dependencies> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-core</artifactId> + <version>${jmh.version}</version> + </dependency> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-generator-annprocess</artifactId> + <version>${jmh.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <version>1.1.7</version> + </dependency> + <dependency> + <groupId>org.onap.music.mdbc</groupId> + <artifactId>mdbc-server</artifactId> + <version>0.0.1-SNAPSHOT</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.onap.music</groupId> + <artifactId>dev-MUSIC-cassandra</artifactId> + <version>3.2.1</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.datastax.cassandra</groupId> + <artifactId>cassandra-driver-core</artifactId> + <version>3.3.0</version> + </dependency> + </dependencies> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <jmh.version>1.9.3</jmh.version> + <javac.target>1.6</javac.target> + <uberjar.name>benchmarks</uberjar.name> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.1</version> + <configuration> + <compilerVersion>${javac.target}</compilerVersion> + <source>7</source> + <target>7</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.2</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <finalName>${uberjar.name}</finalName> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.openjdk.jmh.Main</mainClass> + </transformer> + </transformers> + <filters> + <filter> + <!-- + Shading signed JARs will fail without this. + http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar + --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + <pluginManagement> + <plugins> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <version>2.5</version> + </plugin> + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8.1</version> + </plugin> + <plugin> + <artifactId>maven-install-plugin</artifactId> + <version>2.5.1</version> + </plugin> + <plugin> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + </plugin> + <plugin> + <artifactId>maven-javadoc-plugin</artifactId> + <version>2.9.1</version> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + </plugin> + <plugin> + <artifactId>maven-site-plugin</artifactId> + <version>3.3</version> + </plugin> + <plugin> + <artifactId>maven-source-plugin</artifactId> + <version>2.2.1</version> + </plugin> + <plugin> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.17</version> + </plugin> + </plugins> + </pluginManagement> + </build> + +</project> diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricAddTxDigestBenchmark.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricAddTxDigestBenchmark.java new file mode 100644 index 0000000..d15c3e6 --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricAddTxDigestBenchmark.java @@ -0,0 +1,143 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.onap.music.mdbc.tables.MusicTxDigestId; +import org.onap.music.mdbc.tables.StagingTable; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricAddTxDigestBenchmark { + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricAddTxDigestBenchmark.class.getSimpleName()) + .threads(1) + .forks(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void testMethod(MyState state, Blackhole blackhole) { + OwnUtils.addTransactionDigest(state.copy, state.copyTxDigestId, + (MusicMixin) state.getManager().getMusicInterface()); + } + + @State(Scope.Benchmark) + public static class MyState { + public MusicTxDigestId musicTxDigestId,copyTxDigestId; + public StagingTable copy,current; + final String user = OwnUtils.SQL_USER; + final String password = OwnUtils.SQL_PASSWORD; + public final Range range = new Range(OwnUtils.TABLE); + @Param({"127.0.0.1"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + private MusicInterface musicMixin = null; + private MdbcConnection conn; + private DBInterface dbMixin; + private MdbcServerLogic meta; + private String id; + + private void setupServer(){ + meta = OwnUtils.setupServer(user, password); + } + + private StateManager getManager(){ + return meta.getStateManager(); + } + + private void assignManager() { + StateManager manager = getManager(); + musicMixin=manager.getMusicInterface(); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + OwnUtils.dropAll(ip); + setupServer(); + assignManager(); + OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1); + id = UUID.randomUUID().toString(); + conn = (MdbcConnection) getManager().getConnection(id); + try { + Statement stmt = conn.createStatement(); + stmt.execute(OwnUtils.UPDATE); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest()); + final DatabasePartition partition = conn.getPartition(); + OwnUtils.appendToRedo((MusicMixin)getManager().getMusicInterface(),partition.getMRIIndex(),partition.getLockId(), + musicTxDigestId); + current=conn.getTransactionDigest(); + } + + @TearDown(Level.Trial) + public void doTrialTearDown(){ + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + meta=null; + } + + @Setup(Level.Invocation) + public void doInvocationSetup(){ + try { + copy = new StagingTable(current); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + System.exit(1); + } + copyTxDigestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1); + } + } +} diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricCommitBenchmark.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricCommitBenchmark.java new file mode 100644 index 0000000..7461457 --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricCommitBenchmark.java @@ -0,0 +1,137 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc; + +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricCommitBenchmark { + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricCommitBenchmark.class.getSimpleName()) + .threads(1) + .forks(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void testMethod(MyState state, Blackhole blackhole) { + try { + state.conn.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @State(Scope.Benchmark) + public static class MyState { + final String user = OwnUtils.SQL_USER; + final String password = OwnUtils.SQL_PASSWORD; + public final Range range = new Range(OwnUtils.TABLE); + @Param({"104.209.240.219"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + private MusicInterface musicMixin = null; + private MdbcConnection conn; + private DBInterface dbMixin; + private MdbcServerLogic meta; + private String id; + + private void setupServer(){ + meta = OwnUtils.setupServer(user, password); + } + + private StateManager getManager(){ + return meta.getStateManager(); + } + + private void assignManager() { + StateManager manager = getManager(); + musicMixin=manager.getMusicInterface(); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + OwnUtils.dropAll(ip); + setupServer(); + assignManager(); + OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1); + id = UUID.randomUUID().toString(); + conn = (MdbcConnection) getManager().getConnection(id); + + } + + @TearDown(Level.Trial) + public void doTrialTearDown(){ + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + meta=null; + } + + @Setup(Level.Invocation) + public void doInvocationSetup(){ + Statement stmt = null; + try { + stmt = conn.createStatement(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + stmt.execute(OwnUtils.UPDATE); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + } +} diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricIsolatedMixinCommBenchmark.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricIsolatedMixinCommBenchmark.java new file mode 100644 index 0000000..c789ca7 --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricIsolatedMixinCommBenchmark.java @@ -0,0 +1,151 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc; + +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.onap.music.exceptions.MDBCServiceException; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.onap.music.mdbc.tables.MusicTxDigestId; +import org.onap.music.mdbc.tables.StagingTable; +import org.onap.music.mdbc.tables.TxCommitProgress; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricIsolatedMixinCommBenchmark { + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricIsolatedMixinCommBenchmark.class.getSimpleName()) + .threads(1) + .forks(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void testMethod(MyState state, Blackhole blackhole) { + try { + state.musicMixin.commitLog(state.conn.getPartition(),null,state.copy, + state.copyTxDigestId.transactionId.toString(),state.progressKeeper); + } catch (MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @State(Scope.Benchmark) + public static class MyState { + public TxCommitProgress progressKeeper; + public MusicTxDigestId musicTxDigestId,copyTxDigestId; + final String user = OwnUtils.SQL_USER; + final String password = OwnUtils.SQL_PASSWORD; + public final Range range = new Range(OwnUtils.TABLE); + @Param({"104.209.240.219"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + private MusicInterface musicMixin = null; + private MdbcConnection conn; + private DBInterface dbMixin; + private MdbcServerLogic meta; + private String id; + public StagingTable copy,current; + + private void setupServer(){ + meta = OwnUtils.setupServer(user, password); + } + + private StateManager getManager(){ + return meta.getStateManager(); + } + + private void assignManager() { + StateManager manager = getManager(); + musicMixin=manager.getMusicInterface(); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + OwnUtils.dropAll(ip); + setupServer(); + assignManager(); + OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1); + id = UUID.randomUUID().toString(); + conn = (MdbcConnection) getManager().getConnection(id); + Statement stmt = null; + try { + stmt = conn.createStatement(); + stmt.execute(OwnUtils.UPDATE); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest()); + current=conn.getTransactionDigest(); + progressKeeper = new TxCommitProgress(); + } + + @TearDown(Level.Trial) + public void doTrialTearDown(){ + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + meta=null; + } + + @Setup(Level.Invocation) + public void doInvocationSetup(){ + try { + copy = new StagingTable(current); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + System.exit(1); + } + copyTxDigestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1); + progressKeeper.createNewTransactionTracker(copyTxDigestId.transactionId.toString(),conn); + } + + } +} diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricOwnBenchmark.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricOwnBenchmark.java new file mode 100644 index 0000000..42665b1 --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricOwnBenchmark.java @@ -0,0 +1,142 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc; + +import java.sql.SQLException; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.onap.music.exceptions.MDBCServiceException; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricOwnBenchmark{ + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricOwnBenchmark.class.getSimpleName()) + .threads(1) + .forks(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void testMethod(MyState state, Blackhole blackhole) { + try { + state.conn.preStatementHook(state.update); + } catch (MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @State(Scope.Benchmark) + public static class MyState { + public String update="SELECT * FROM PERSONS;"; + final String user = OwnUtils.SQL_USER; + final String password = OwnUtils.SQL_PASSWORD; + public final Range range = new Range(OwnUtils.TABLE); + @Param({"104.209.240.219"}) + public String ip; + //Rows per transaction (e.g. size of each tx digest) + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + //Transaction before each ownership transition (e.g. size of redo log) + @Param({"1","10","100"}) + public int updates; + //Number of ownership transitions before measurement (e.g. number of mri rows" + @Param({"1","10","100"}) + public int transitions; + private MusicInterface musicMixin = null; + private MdbcConnection conn; + private DBInterface dbMixin; + private MdbcServerLogic meta; + private String id; + + private void setupServer(){ + meta = OwnUtils.setupServer(user, password); + } + + private StateManager getManager(){ + return meta.getStateManager(); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + OwnUtils.dropAll(ip); + setupServer(); + StateManager manager = getManager(); + musicMixin=manager.getMusicInterface(); + OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows, updates, transitions); + meta=null; + } + + @TearDown(Level.Trial) + public void doTrialTearDown(){ + meta=null; + } + + @Setup(Level.Invocation) + public void doInvocationSetup(){ + setupServer(); + id = UUID.randomUUID().toString(); + conn = (MdbcConnection) getManager().getConnection(id); + try { + OwnUtils.dropAndCreateTable(conn.getDBInterface()); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @TearDown(Level.Invocation) + public void doInvocationTearDown(){ + OwnUtils.deleteLastMriRow((MusicMixin) musicMixin); + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + getManager().closeConnection(id); + } + + } +} diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricPeekLockBenchmark.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricPeekLockBenchmark.java new file mode 100644 index 0000000..1b56c6b --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricPeekLockBenchmark.java @@ -0,0 +1,151 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc; + +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.ExecutionInfo; +import com.datastax.driver.core.QueryTrace; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.SimpleStatement; +import org.onap.music.datastore.MusicDataStore; +import org.onap.music.datastore.MusicDataStoreHandle; +import org.onap.music.datastore.PreparedQueryObject; +import org.onap.music.exceptions.MusicQueryException; +import org.onap.music.exceptions.MusicServiceException; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.onap.music.mdbc.tables.MusicRangeInformationRow; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricPeekLockBenchmark { + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricPeekLockBenchmark.class.getSimpleName()) + .threads(1) + .forks(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void testMethod(MyState state, Blackhole blackhole) { + String table = state.table_prepend_name+OwnUtils.MRI_TABLE_NAME; + String selectQuery = "select * from "+OwnUtils.KEYSPACE+"."+table+" where key='"+state.key+"' LIMIT 1;"; + PreparedQueryObject queryObject = new PreparedQueryObject(); + queryObject.appendQueryString(selectQuery); + ResultSet results=null; + SimpleStatement statement = new SimpleStatement(queryObject.getQuery(), queryObject.getValues().toArray()); + statement.setConsistencyLevel(ConsistencyLevel.ONE); + statement.enableTracing(); + results = state.dsHandle.getSession().execute(statement); + blackhole.consume(results); + } + + public void printInfo(ResultSet results){ + ExecutionInfo executionInfo = results.getExecutionInfo(); + System.out.println(executionInfo.getQueriedHost().getAddress()); + System.out.println(executionInfo.getQueriedHost().getDatacenter()); + System.out.println(executionInfo.getQueriedHost().getRack()); + final QueryTrace trace = executionInfo.getQueryTrace(); + System.out.printf( + "'%s' to %s took %dμs%n", + trace.getRequestType(), trace.getCoordinator(), trace.getDurationMicros()); + for (QueryTrace.Event event : trace.getEvents()) { + System.out.printf( + " %d - %s - %s%n", + event.getSourceElapsedMicros(), event.getSource(), event.getDescription()); + } + } + + @State(Scope.Benchmark) + public static class MyState { + public MusicDataStore dsHandle; + public String key; + private String table_prepend_name = "lockQ_"; + final String user = OwnUtils.SQL_USER; + final String password = OwnUtils.SQL_PASSWORD; + public final Range range = new Range(OwnUtils.TABLE); + @Param({"104.209.240.219"}) + public String ip; + private MusicInterface musicMixin = null; + private MdbcConnection conn; + private DBInterface dbMixin; + private MdbcServerLogic meta; + private String id; + public MusicRangeInformationRow lastRow; + + private void setupServer(){ + meta = OwnUtils.setupServer(user, password); + } + + private StateManager getManager(){ + return meta.getStateManager(); + } + + private void assignManager() { + StateManager manager = getManager(); + musicMixin=manager.getMusicInterface(); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + OwnUtils.dropAll(ip); + setupServer(); + try { + dsHandle=MusicDataStoreHandle.getDSHandle(); + } catch (MusicServiceException e) { + e.printStackTrace(); + System.exit(1); + } + assignManager(); + OwnUtils.initMri((MusicMixin) musicMixin,range,meta, 1,1,1); + id = UUID.randomUUID().toString(); + conn = (MdbcConnection) getManager().getConnection(id); + lastRow = OwnUtils.getLastRow((MusicMixin) musicMixin); + key = lastRow.getPartitionIndex().toString(); + } + + @TearDown(Level.Trial) + public void doTrialTearDown(){ + try { + conn.close(); + } catch (SQLException e) { + System.exit(1); + } + meta=null; + } + } +} diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricRedoLogBenchmark.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricRedoLogBenchmark.java new file mode 100644 index 0000000..7e227fd --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricRedoLogBenchmark.java @@ -0,0 +1,130 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.onap.music.mdbc.tables.MusicTxDigestId; +import org.onap.music.mdbc.tables.StagingTable; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricRedoLogBenchmark { + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricRedoLogBenchmark.class.getSimpleName()) + .threads(1) + .forks(1) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void testMethod(MyState state, Blackhole blackhole) { + final DatabasePartition partition = state.conn.getPartition(); + OwnUtils.appendToRedo((MusicMixin)state.getManager().getMusicInterface(),partition.getMRIIndex(), + partition.getLockId(),state.musicTxDigestId); + } + + @State(Scope.Benchmark) + public static class MyState { + public MusicTxDigestId musicTxDigestId; + final String user = OwnUtils.SQL_USER; + final String password = OwnUtils.SQL_PASSWORD; + public final Range range = new Range(OwnUtils.TABLE); + @Param({"104.209.240.219"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + private MusicInterface musicMixin = null; + private MdbcConnection conn; + private DBInterface dbMixin; + private MdbcServerLogic meta; + private String id; + + private void setupServer(){ + meta = OwnUtils.setupServer(user, password); + } + + private StateManager getManager(){ + return meta.getStateManager(); + } + + private void assignManager() { + StateManager manager = getManager(); + musicMixin=manager.getMusicInterface(); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + OwnUtils.dropAll(ip); + setupServer(); + assignManager(); + OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1); + id = UUID.randomUUID().toString(); + conn = (MdbcConnection) getManager().getConnection(id); + Statement stmt = null; + try { + stmt = conn.createStatement(); + stmt.execute(OwnUtils.UPDATE); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest()); + } + + @TearDown(Level.Trial) + public void doTrialTearDown(){ + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + meta=null; + } + } +} diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricThreadJoinBenchmark.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricThreadJoinBenchmark.java new file mode 100644 index 0000000..70ba10a --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/MetricThreadJoinBenchmark.java @@ -0,0 +1,197 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc; + +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.onap.music.exceptions.MDBCServiceException; +import org.onap.music.exceptions.MusicLockingException; +import org.onap.music.logging.EELFLoggerDelegate; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.onap.music.mdbc.tables.MusicTxDigestId; +import org.onap.music.mdbc.tables.StagingTable; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricThreadJoinBenchmark { + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricThreadJoinBenchmark.class.getSimpleName()) + .threads(1) + .forks(1) + .build(); + new Runner(opt).run(); + } + + public void testMethod1(final MyState state) { + final String lockId = state.conn.getPartition().getLockId(); + final UUID MRIIndex = state.conn.getPartition().getMRIIndex(); + Thread t1=null; + Thread t2=null; + if(state.runTxDigest) { + final Runnable insertDigestCallable = new Runnable() { + @Override + public void run() { + OwnUtils.hardcodedAddtransaction(110); + } + }; + t1 = new Thread(insertDigestCallable); + t1.start(); + } + if(state.runRedo) { + final Runnable appendCallable = new Runnable() { + @Override + public void run() { + OwnUtils.hardcodedAppendToRedo(MRIIndex,lockId); + } + }; + t2 = new Thread(appendCallable); + t2.start(); + } + + try { + if(state.runTxDigest) { + t1.join(); + } + if(state.runRedo) { + t2.join(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + System.exit(1); + } + } + + @Benchmark + public void testMethod2(final MyState state) { + final String lockId = state.conn.getPartition().getLockId(); + final UUID MRIIndex = state.conn.getPartition().getMRIIndex(); + OwnUtils.hardcodedAddtransaction(110); + OwnUtils.hardcodedAppendToRedo(MRIIndex,lockId); + } + + @State(Scope.Benchmark) + public static class MyState { + public MusicTxDigestId musicTxDigestId,copyTxDigestId; + public StagingTable copy,current; + final String user = OwnUtils.SQL_USER; + final String password = OwnUtils.SQL_PASSWORD; + public final Range range = new Range(OwnUtils.TABLE); + @Param({"127.0.0.1"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + @Param({"true","false"}) + public boolean runRedo; + @Param({"true","false"}) + public boolean runTxDigest; + private MusicInterface musicMixin = null; + private MdbcConnection conn; + private DBInterface dbMixin; + private MdbcServerLogic meta; + private String id; + public ExecutorService commitExecutorThreads; + + private void setupServer(){ + meta = OwnUtils.setupServer(user, password); + } + + private StateManager getManager(){ + return meta.getStateManager(); + } + + private void assignManager() { + StateManager manager = getManager(); + musicMixin=manager.getMusicInterface(); + } + + @Setup(Level.Trial) + public void doTrialSetup(){ + commitExecutorThreads = Executors.newFixedThreadPool(4); + OwnUtils.dropAll(ip); + setupServer(); + assignManager(); + OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,0,0); + id = UUID.randomUUID().toString(); + conn = (MdbcConnection) getManager().getConnection(id); + try { + Statement stmt = conn.createStatement(); + stmt.execute(OwnUtils.UPDATE); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest()); + current=conn.getTransactionDigest(); + } + + @TearDown(Level.Trial) + public void doTrialTearDown(){ + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + meta=null; + commitExecutorThreads.shutdown(); + } + + @Setup(Level.Invocation) + public void doInvocationSetup(){ + try { + copy = new StagingTable(current); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + System.exit(1); + } + copyTxDigestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1); + } + } +} diff --git a/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/OwnUtils.java b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/OwnUtils.java new file mode 100644 index 0000000..980408b --- /dev/null +++ b/mdbc-internal-benchmark/src/main/java/org/onap/music/mdbc/OwnUtils.java @@ -0,0 +1,383 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ +package org.onap.music.mdbc; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Cluster.Builder; +import com.datastax.driver.core.Session; + +import java.nio.ByteBuffer; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import com.datastax.driver.core.SocketOptions; +import org.onap.music.datastore.PreparedQueryObject; +import org.onap.music.exceptions.MDBCServiceException; +import org.onap.music.exceptions.MusicLockingException; +import org.onap.music.exceptions.MusicServiceException; +import org.onap.music.lockingservice.cassandra.MusicLockState; +import org.onap.music.main.MusicCore; +import org.onap.music.main.ResultType; +import org.onap.music.main.ReturnType; +import org.onap.music.mdbc.configurations.NodeConfiguration; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; +import org.onap.music.mdbc.mixins.MusicMixin; +import org.onap.music.mdbc.tables.MriRowComparator; +import org.onap.music.mdbc.tables.MusicRangeInformationRow; +import org.onap.music.mdbc.tables.MusicTxDigestId; +import org.onap.music.mdbc.tables.StagingTable; + +public class OwnUtils { + + public static final int SQL_PORT= 3306; + + public static final String SQL_USER = "root"; + public static final String SQL_PASSWORD = "metriccluster"; + public static final String CASSANDRA_USER = "metric"; + public static final String CASSANDRA_PASSWORD = "metriccluster"; + public static final String KEYSPACE ="namespace"; + public static final String MRI_TABLE_NAME = "musicrangeinformation"; + public static final String MTD_TABLE_NAME = "musictxdigest"; + public static final String MDBC_SERVER_NAME = "name"; + public static final String DATABASE = "test"; + public static final String TABLE= "PERSONS"; + public static final int REPLICATION_FACTOR = 3; + public static final String SQL_URL="jdbc:mariadb://localhost:"+OwnUtils.SQL_PORT; + final static String CREATE_TABLE = "CREATE TABLE IF NOT EXISTS "+TABLE+" (\n" + + " PersonID int,\n" + + " Counter int,\n" + + " LastName varchar(255),\n" + + " FirstName varchar(255),\n" + + " Address varchar(255),\n" + + " City varchar(255),\n" + + " PRIMARY KEY(PersonID)"+ + ");"; + + public static final String UPDATE = new StringBuilder() + .append("UPDATE PERSONS ") + .append("SET Counter = Counter + 1,") + .append("City = 'Sandy Springs'") + .append(";").toString(); + public static final String PURGE = "PURGE BINARY LOGS BEFORE '"; + public static final String DROP_TABLE = "DROP TABLE IF EXISTS " + TABLE + ";"; + private static Lock sessionLock = new ReentrantLock(); + private static Boolean sessionReady= false; + private static Cluster cluster; + private static Session session; + + private static String getPurgeCommand(){ + String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()); + return PURGE+timeStamp+"';"; + } + + public static Session getSession(String CASSANDRA_HOST){ + sessionLock.lock(); + try { + if(!sessionReady){ + SocketOptions options = new SocketOptions(); + options.setConnectTimeoutMillis(30000); + options.setReadTimeoutMillis(300000); + options.setTcpNoDelay(true); + final Builder builder = Cluster.builder(); + builder.addContactPoint(CASSANDRA_HOST); + builder.withCredentials(CASSANDRA_USER, CASSANDRA_PASSWORD); + builder.withSocketOptions(options); + cluster = builder.build(); + session=cluster.newSession(); + } + return session; + // access the resource protected by this lock + } finally { + sessionLock.unlock(); + } + } + + public static Connection getConnection() throws ClassNotFoundException, SQLException { + Class.forName("org.mariadb.jdbc.Driver"); + Properties connectionProps = new Properties(); + connectionProps.put("user", SQL_USER); + connectionProps.put("password", SQL_PASSWORD); + Connection connection = DriverManager.getConnection(SQL_URL+"/"+DATABASE, connectionProps); + connection.setAutoCommit(false); + return connection; + } + + public static void purgeBinLogs(Connection conn) throws SQLException{ + final Statement dropStatement = conn.createStatement(); + dropStatement.execute(getPurgeCommand()); + dropStatement.close(); + } + + public static void dropTable(Connection conn) throws SQLException { + final Statement dropStatement = conn.createStatement(); + dropStatement.execute(DROP_TABLE); + dropStatement.close(); + } + + private static void createTable(Connection conn) throws SQLException { + final Statement createStatement = conn.createStatement(); + createStatement.execute(CREATE_TABLE); + createStatement.close(); + } + + public static void dropAndCreateTable(DBInterface dbMixin) throws SQLException { + dbMixin.dropSQLTriggers(TABLE); + dropTable(dbMixin.getSQLConnection()); + createTable(dbMixin.getSQLConnection()); + dbMixin.createSQLTriggers(TABLE); + } + + public static void unlockRow(String keyspace, String mriTableName, DatabasePartition partition) + throws MusicLockingException { + String fullyQualifiedMriKey = keyspace+"."+ mriTableName+"."+partition.getMRIIndex().toString(); + MusicLockState musicLockState = MusicCore.voluntaryReleaseLock(fullyQualifiedMriKey, partition.getLockId()); + } + + public static DatabasePartition createBasicRow(Range range, MusicInterface mixin, String mdbcServerName) + throws MDBCServiceException { + List<Range> ranges = new ArrayList<>(); + ranges.add(range); + final UUID uuid = MDBCUtils.generateTimebasedUniqueKey(); + DatabasePartition dbPartition = new DatabasePartition(ranges,uuid,null); + MusicRangeInformationRow newRow = new MusicRangeInformationRow(uuid,dbPartition, new ArrayList<MusicTxDigestId>(), "", + mdbcServerName, true); + DatabasePartition partition=mixin.createMusicRangeInformation(newRow); + return partition; + } + + public static void initMriTable(MusicMixin musicMixin, Range range) + throws MDBCServiceException, SQLException, MusicLockingException { + final DatabasePartition partition = createBasicRow(range, musicMixin, MDBC_SERVER_NAME); + unlockRow(KEYSPACE,MRI_TABLE_NAME,partition); + } + + public static MusicRangeInformationRow getLastRow(MusicMixin musicMixin){ + List<MusicRangeInformationRow> allMriRows; + try { + allMriRows = musicMixin.getAllMriRows(); + } catch (MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + allMriRows=null;//Just to avoid IDE annoyance + } + Collections.sort(allMriRows, new MriRowComparator()); + MusicRangeInformationRow musicRangeInformationRow = allMriRows.get(allMriRows.size() - 1); + return musicRangeInformationRow; + } + + public static void deleteLastMriRow(MusicMixin musicMixin){ + MusicRangeInformationRow musicRangeInformationRow = getLastRow(musicMixin); + try { + musicMixin.deleteMriRow(musicRangeInformationRow); + } catch (MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + } + } + + private static void changeRows(Connection conn) throws SQLException { + Statement stmt = conn.createStatement(); + Boolean execute = stmt.execute(UPDATE); + conn.commit(); + stmt.close(); + } + + private static void addRowsToTable(int totalNumberOfRows, Connection testConnection) throws SQLException { + for (int i = 0; i < totalNumberOfRows; i++) { + final StringBuilder insertSQLBuilder = new StringBuilder() + .append("INSERT INTO PERSONS VALUES (") + .append(i) + .append(", ") + .append(0) + .append(", '") + .append("Last-") + .append(i) + .append("', '") + .append("First-") + .append(i) + .append("', 'KACB', 'ATLANTA');"); + Statement stmt = testConnection.createStatement(); + Boolean execute = stmt.execute(insertSQLBuilder.toString()); + stmt.close(); + } + testConnection.commit(); + } + + public static void createHistory(MdbcServerLogic meta,int rows, int updates,int transitions) + throws SQLException { + final StateManager stateManager = meta.getStateManager(); + String id = UUID.randomUUID().toString(); + Connection connection = stateManager.getConnection(id); + createTable(connection); + addRowsToTable(rows, connection); + connection.close(); + stateManager.closeConnection(id); + for(int mriRow=0;mriRow<transitions;mriRow++) { + final String finalId = UUID.randomUUID().toString(); + connection = stateManager.getConnection(finalId); + for(int depth=0;depth<updates;depth++){ + changeRows(connection); + } + connection.close(); + stateManager.closeConnection(finalId); + } + } + + public static MdbcServerLogic setupServer(String user, String password){ + MdbcServerLogic meta; + NodeConfiguration config = new NodeConfiguration("","",null,OwnUtils.DATABASE, + OwnUtils.MDBC_SERVER_NAME); + //\TODO Add configuration file with Server Info + Properties connectionProps = new Properties(); + connectionProps.setProperty("user", user); + connectionProps.setProperty("password", password); + connectionProps.setProperty(MusicMixin.KEY_MUSIC_RFACTOR,Integer.toString(OwnUtils.REPLICATION_FACTOR)); + connectionProps.setProperty(MusicMixin.KEY_MUSIC_NAMESPACE,OwnUtils.KEYSPACE); + try { + meta = new MdbcServerLogic(OwnUtils.SQL_URL,connectionProps,config); + } catch (SQLException e) { + e.printStackTrace(); + meta=null; + System.exit(1); + } catch (MDBCServiceException e) { + e.printStackTrace(); + meta=null; + System.exit(1); + } + return meta; + } + + public static void dropAll(String ip){ + Session session=OwnUtils.getSession(ip); + session.execute("DROP KEYSPACE IF EXISTS "+OwnUtils.KEYSPACE+";"); + try { + Connection connection = OwnUtils.getConnection(); + OwnUtils.dropTable(connection); + connection.close(); + } catch (SQLException|ClassNotFoundException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static void initMri(MusicMixin musicMixin,Range range,MdbcServerLogic meta, int rows, int updates, int transitions){ + try { + OwnUtils.initMriTable(musicMixin,range); + } catch (MusicLockingException|SQLException|MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + } + try { + OwnUtils.createHistory(meta, rows, updates, transitions); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static void hardcodedAddtransaction(int size){ + final UUID uuid = MDBCUtils.generateTimebasedUniqueKey(); + ByteBuffer serializedTransactionDigest = ByteBuffer.allocate(size); + for(int i=0;i<size;i++){ + serializedTransactionDigest.put((byte)i); + } + PreparedQueryObject query = new PreparedQueryObject(); + String cql = String.format("INSERT INTO %s.%s (txid,transactiondigest,compressed ) VALUES (?,?,?);",KEYSPACE, + MTD_TABLE_NAME); + query.appendQueryString(cql); + query.addValue(uuid); + query.addValue(serializedTransactionDigest); + query.addValue(false); + //\TODO check if I am not shooting on my own foot + try { + MusicCore.nonKeyRelatedPut(query,"critical"); + } catch (MusicServiceException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static void hardcodedAppendToRedo(UUID mriId, String lockId){ + final UUID uuid = MDBCUtils.generateTimebasedUniqueKey(); + PreparedQueryObject query = new PreparedQueryObject(); + StringBuilder appendBuilder = new StringBuilder(); + appendBuilder.append("UPDATE ") + .append(KEYSPACE) + .append(".") + .append(MRI_TABLE_NAME) + .append(" SET txredolog = txredolog +[('") + .append(MTD_TABLE_NAME) + .append("',") + .append(uuid) + .append(")] WHERE rangeid = ") + .append(mriId) + .append(";"); + query.appendQueryString(appendBuilder.toString()); + ReturnType returnType = MusicCore.criticalPut(KEYSPACE, MRI_TABLE_NAME, mriId.toString(), + query, lockId, null); + //returnType.getExecutionInfo() + if (returnType.getResult().compareTo(ResultType.SUCCESS) != 0) { + System.exit(1); + } + } + + public static void addTransactionDigest(StagingTable transactionDigest, MusicTxDigestId digestId, MusicMixin music){ + try { + music.createAndAddTxDigest(transactionDigest, digestId.transactionId); + } catch (MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static void appendToRedo(MusicMixin music, UUID MRIIndex, String lockId, MusicTxDigestId digestId){ + try { + music.appendToRedoLog(KEYSPACE,MRIIndex,digestId.transactionId,lockId,OwnUtils.MTD_TABLE_NAME, + OwnUtils.MRI_TABLE_NAME); + } catch (MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public static MusicTxDigestId setupCommit(DatabasePartition partition, StagingTable transactionDigest){ + UUID mriIndex = partition.getMRIIndex(); + + if(transactionDigest == null || transactionDigest.isEmpty()) { + System.err.println("Transaction digest is empty"); + System.exit(1); + } + + MusicTxDigestId digestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1); + return digestId; + } +} diff --git a/mdbc-internal-benchmark/src/main/resources/logback.xml b/mdbc-internal-benchmark/src/main/resources/logback.xml new file mode 100755 index 0000000..4215681 --- /dev/null +++ b/mdbc-internal-benchmark/src/main/resources/logback.xml @@ -0,0 +1,369 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ============LICENSE_START========================================== + mdbc + =================================================================== + Copyright © 2017 AT&T Intellectual Property. All rights reserved. + =================================================================== + + Unless otherwise specified, all software contained herein is licensed + under the Apache License, Version 2.0 (the “License”); + you may not use this software except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Unless otherwise specified, all documentation contained herein is licensed + under the Creative Commons License, Attribution 4.0 Intl. (the “License”); + you may not use this documentation except in compliance with the License. + You may obtain a copy of the License at + + https://creativecommons.org/licenses/by/4.0/ + + Unless required by applicable law or agreed to in writing, documentation + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ============LICENSE_END============================================ + + + --> + +<configuration scan="false" scanPeriod="3 seconds" debug="true"> + <!-- + Logback files for the mdbc Driver "mdbc" + are created in directory ${catalina.base}/logs/mdbc; + e.g., apache-tomcat-8.0.35/logs/mdbc/application.log + --> + <!--<jmxConfigurator /> --> + + <!-- specify the component name --> + <property name="catalina.home" value="/var/log/metric/"/> + <property name="componentName" value="mdbc"></property> + + <!-- specify the base path of the log directory --> + <property name="logDirPrefix" value="${catalina.base}/logs"></property> + + <!-- The directories where logs are written --> + <property name="logDirectory" value="${logDirPrefix}/${componentName}"/> + <!-- Can easily relocate debug logs by modifying this path. --> + <property name="debugLogDirectory" value="${logDirPrefix}/${componentName}"/> + + <!-- log file names --> + <property name="generalLogName" value="application"/> + <property name="errorLogName" value="error"/> + <property name="metricsLogName" value="metrics"/> + <property name="auditLogName" value="audit"/> + <property name="debugLogName" value="debug"/> + <!-- + These loggers are not used in code (yet). + <property name="securityLogName" value="security" /> + <property name="policyLogName" value="policy" /> + <property name="performanceLogName" value="performance" /> + <property name="serverLogName" value="server" /> + --> + + <!-- 1610 Logging Fields Format Revisions --> + <property name="auditLoggerPattern" + value="%X{AuditLogBeginTimestamp}|%X{AuditLogEndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{ClientIPAddress}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}| %msg%n"/> + + <property name="metricsLoggerPattern" + value="%X{MetricsLogBeginTimestamp}|%X{MetricsLogEndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{ClientIPAddress}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{TargetVisualEntity}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}| %msg%n"/> + + <property name="errorLoggerPattern" + value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{ClassName}|%X{AlertSeverity}|%X{ErrorCode}|%X{ErrorDescription}| %msg%n"/> + + <property name="defaultLoggerPattern" + value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%X{ClassName}| %msg%n"/> + + <!-- use %class so library logging calls yield their class name --> + <property name="applicationLoggerPattern" + value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%class{36}| %msg%n"/> + + <!-- Example evaluator filter applied against console appender --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>${defaultLoggerPattern}</pattern> + </encoder> + </appender> + + <!-- ============================================================================ --> + <!-- EELF Appenders --> + <!-- ============================================================================ --> + + <!-- The EELFAppender is used to record events to the general application + log --> + + + <appender name="EELF" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${generalLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${applicationLoggerPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <!-- Class name is part of caller data --> + <includeCallerData>true</includeCallerData> + <appender-ref ref="EELF"/> + </appender> + + <!-- EELF Security Appender. This appender is used to record security events + to the security log file. Security events are separate from other loggers + in EELF so that security log records can be captured and managed in a secure + way separate from the other logs. This appender is set to never discard any + events. --> + <!-- + <appender name="EELFSecurity" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${securityLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${securityLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFSecurity" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <discardingThreshold>0</discardingThreshold> + <appender-ref ref="EELFSecurity" /> + </appender> + --> + + <!-- EELF Performance Appender. This appender is used to record performance + records. --> + <!-- + <appender name="EELFPerformance" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${performanceLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${performanceLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <outputPatternAsHeader>true</outputPatternAsHeader> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFPerformance" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFPerformance" /> + </appender> + --> + + <!-- EELF Server Appender. This appender is used to record Server related + logging events. The Server logger and appender are specializations of the + EELF application root logger and appender. This can be used to segregate Server + events from other components, or it can be eliminated to record these events + as part of the application root log. --> + <!-- + <appender name="EELFServer" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${serverLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${serverLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFServer" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFServer" /> + </appender> + --> + + <!-- EELF Policy Appender. This appender is used to record Policy engine + related logging events. The Policy logger and appender are specializations + of the EELF application root logger and appender. This can be used to segregate + Policy engine events from other components, or it can be eliminated to record + these events as part of the application root log. --> + <!-- + <appender name="EELFPolicy" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${policyLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${policyLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFPolicy" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFPolicy" /> + </appender> + --> + + <!-- EELF Audit Appender. This appender is used to record audit engine + related logging events. The audit logger and appender are specializations + of the EELF application root logger and appender. This can be used to segregate + Policy engine events from other components, or it can be eliminated to record + these events as part of the application root log. --> + + <appender name="EELFAudit" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${auditLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${auditLoggerPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFAudit"/> + </appender> + + <appender name="EELFMetrics" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${metricsLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${metricsLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${metricsLoggerPattern}</pattern> + </encoder> + </appender> + + + <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFMetrics"/> + </appender> + + <appender name="EELFError" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${errorLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${errorLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${errorLoggerPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFError" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFError"/> + </appender> + + <appender name="EELFDebug" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${debugLogDirectory}/${debugLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${debugLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${defaultLoggerPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFDebug" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFDebug"/> + </appender> + + + <logger name="com.att.eelf" level="error" additivity="false"> + <appender-ref ref="asyncEELF"/> + </logger> + + <logger name="com.att.eelf" level="error" additivity="false"> + <appender-ref ref="asyncEELFAudit"/> + </logger> + + <logger name="com.att.eelf" level="error" additivity="false"> + <appender-ref ref="asyncEELFDebug"/> + </logger> + + <logger name="com.att.eelf.error" level="error" additivity="false"> + <appender-ref ref="asyncEELFError"/> + </logger> + + <logger name="com.att.eelf.metrics" level="error" additivity="false"> + <appender-ref ref="asyncEELFMetrics"/> + </logger> + + <root level="ERROR"> + <appender-ref ref="asyncEELF"/> + </root> + +</configuration> diff --git a/mdbc-internal-benchmark/src/main/resources/mdbc.properties b/mdbc-internal-benchmark/src/main/resources/mdbc.properties new file mode 100755 index 0000000..73e8f77 --- /dev/null +++ b/mdbc-internal-benchmark/src/main/resources/mdbc.properties @@ -0,0 +1,13 @@ +# +# A list of all Mixins that should be checked by MDBC +# +MIXINS= \ + org.onap.music.mdbc.mixins.MySQLMixin \ + org.onap.music.mdbc.mixins.MusicMixin \ + org.onap.music.mdbc.mixins.Music2Mixin + +DEFAULT_DRIVERS=\ + org.h2.Driver \ + com.mysql.jdbc.Driver + +txdaemonsleeps=15
\ No newline at end of file diff --git a/mdbc-internal-benchmark/src/main/resources/music.properties b/mdbc-internal-benchmark/src/main/resources/music.properties new file mode 100755 index 0000000..ccacd38 --- /dev/null +++ b/mdbc-internal-benchmark/src/main/resources/music.properties @@ -0,0 +1,8 @@ +cassandra.host =\ + 10.0.0.5 +cassandra.user =\ + metric +cassandra.password =\ + metriccluster +zookeeper.host =\ + localhost diff --git a/mdbc-server/pom.xml b/mdbc-server/pom.xml index f69cf2f..cdf7cff 100755 --- a/mdbc-server/pom.xml +++ b/mdbc-server/pom.xml @@ -34,49 +34,49 @@ </parent> <dependencies> - <dependency> - <groupId>org.apache.calcite</groupId> - <artifactId>calcite</artifactId> - <version>1.11.0</version> - <type>pom</type> - </dependency> + <dependency> + <groupId>org.apache.calcite</groupId> + <artifactId>calcite</artifactId> + <version>1.11.0</version> + <type>pom</type> + </dependency> <dependency> <groupId>com.datastax.cassandra</groupId> <artifactId>cassandra-driver-core</artifactId> <version>3.3.0</version> </dependency> <dependency> - <!-- The oparent dependency managed version 26.0-jre will not work --> - <!-- 19.0 is needed by cassandra-driver-core 3.3.0 --> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>19.0</version> + <!-- The oparent dependency managed version 26.0-jre will not work --> + <!-- 19.0 is needed by cassandra-driver-core 3.3.0 --> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>19.0</version> </dependency> <dependency> - <!-- The oparent dependency managed version 9.4.12.v20180830 will not work --> - <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <version>9.2.19.v20160908</version> + <!-- The oparent dependency managed version 9.4.12.v20180830 will not work --> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <version>9.2.19.v20160908</version> </dependency> <dependency> - <!-- The oparent dependency managed version 9.4.12.v20180830 will not work --> - <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <version>9.2.19.v20160908</version> + <!-- The oparent dependency managed version 9.4.12.v20180830 will not work --> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>9.2.19.v20160908</version> </dependency> <dependency> - <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-security</artifactId> - <version>9.2.19.v20160908</version> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-security</artifactId> + <version>9.2.19.v20160908</version> </dependency> <dependency> - <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - <version>9.2.19.v20160908</version> + <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 --> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + <version>9.2.19.v20160908</version> </dependency> <dependency> <groupId>com.h2database</groupId> @@ -89,9 +89,9 @@ <version>20160810</version> </dependency> <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - <version>2.3.0</version> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <version>2.3.0</version> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 --> <dependency> @@ -214,11 +214,11 @@ </configuration> <executions> <execution> - <id>make-assembly</id> <!-- this is used for inheritance merges --> - <phase>package</phase> <!-- bind to the packaging phase --> - <goals> - <goal>single</goal> - </goals> + <id>make-assembly</id> <!-- this is used for inheritance merges --> + <phase>package</phase> <!-- bind to the packaging phase --> + <goals> + <goal>single</goal> + </goals> </execution> <execution> <id>node-configuration</id> @@ -240,6 +240,25 @@ </configuration> </execution> <execution> + <id>cluster-setup</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <archive> + <manifest> + <mainClass>org.onap.music.mdbc.tools.ClusterSetup</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + <finalName>cluster-setup</finalName> + <appendAssemblyId>false</appendAssemblyId> + </configuration> + </execution> + <execution> <id>mdbc-server</id> <phase>package</phase> <goals> diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java index 7377c4f..72b8899 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java @@ -53,7 +53,7 @@ import org.onap.music.mdbc.ownership.Dag; import org.onap.music.mdbc.ownership.DagNode; import org.onap.music.mdbc.ownership.OwnershipAndCheckpoint; import org.onap.music.mdbc.query.QueryProcessor; -import org.onap.music.mdbc.tables.MusicTxDigest; +import org.onap.music.mdbc.tables.MusicTxDigestDaemon; import org.onap.music.mdbc.tables.MusicRangeInformationRow; import org.onap.music.mdbc.tables.StagingTable; import org.onap.music.mdbc.tables.TxCommitProgress; @@ -115,6 +115,10 @@ public class MdbcConnection implements Connection { logger.debug("Mdbc connection created with id: "+id); } + public DBInterface getDatabaseInterface(){ + return this.dbi; + } + @Override public <T> T unwrap(Class<T> iface) throws SQLException { logger.error(EELFLoggerDelegate.errorLogger, "proxyconn unwrap: " + iface.getName()); @@ -263,6 +267,11 @@ public class MdbcConnection implements Connection { jdbcConn.close(); logger.debug("Connection was closed for id:" + id); } + try { + mi.relinquish(partition.getLockId(),partition.getMRIIndex().toString()); + } catch (MDBCServiceException e) { + throw new SQLException("Failure during relinquish of partition",e); + } } @Override @@ -509,7 +518,7 @@ public class MdbcConnection implements Connection { this.partition.updateDatabasePartition(tempPartition); statemanager.getOwnAndCheck().reloadAlreadyApplied(this.partition); } - dbi.preStatementHook(sql); + dbi.preStatementHook(sql); } @@ -603,4 +612,16 @@ public class MdbcConnection implements Connection { mi.relinquishIfRequired(partition); } + public Connection getConnection(){ + return jdbcConn; + } + + public DatabasePartition getPartition() { + return partition; + } + + public StagingTable getTransactionDigest(){ + return transactionDigest; + } + } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServer.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServer.java index 1712c30..42b9710 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServer.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServer.java @@ -20,7 +20,7 @@ package org.onap.music.mdbc; import org.onap.music.mdbc.configurations.NodeConfiguration; -import org.onap.music.mdbc.tables.MusicTxDigest; +import org.onap.music.mdbc.tables.MusicTxDigestDaemon; import org.apache.calcite.avatica.remote.Driver.Serialization; import org.apache.calcite.avatica.remote.LocalService; import org.apache.calcite.avatica.server.HttpServer; diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServerLogic.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServerLogic.java index 9fb36ae..8f79840 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServerLogic.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServerLogic.java @@ -29,7 +29,6 @@ import java.util.concurrent.TimeUnit; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.mdbc.configurations.NodeConfiguration; -import org.onap.music.mdbc.tables.MusicTxDigest; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java b/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java index c498952..16e7170 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java @@ -30,9 +30,7 @@ import java.util.Objects; * In the future we may decide to partition ranges differently * @author Enrique Saurez */ -public class Range implements Serializable, Cloneable{ - - private static final long serialVersionUID = 1610744496930800088L; +public class Range implements Cloneable{ private String table; @@ -61,7 +59,7 @@ public class Range implements Serializable, Cloneable{ } @Override - protected Range clone() { + public Range clone() { Range newRange = null; try{ newRange = (Range) super.clone(); diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/StateManager.java b/mdbc-server/src/main/java/org/onap/music/mdbc/StateManager.java index f0d9832..1105bda 100644 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/StateManager.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/StateManager.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.tuple.Pair; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.logging.EELFLoggerDelegate; @@ -36,7 +35,7 @@ import org.onap.music.mdbc.mixins.MusicInterface; import org.onap.music.mdbc.mixins.MusicInterface.OwnershipReturn; import org.onap.music.mdbc.ownership.OwnershipAndCheckpoint; import org.onap.music.mdbc.tables.MriReference; -import org.onap.music.mdbc.tables.MusicTxDigest; +import org.onap.music.mdbc.tables.MusicTxDigestDaemon; import org.onap.music.mdbc.tables.TxCommitProgress; import java.io.IOException; @@ -48,7 +47,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Properties; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -95,6 +93,7 @@ public class StateManager { /** map of transactions that have already been applied/updated in this sites SQL db */ private Map<Range, Pair<MriReference, Integer>> alreadyApplied; private OwnershipAndCheckpoint ownAndCheck; + private Thread txDaemon ; /** * For testing purposes only @@ -122,19 +121,22 @@ public class StateManager { musicmixin = info.getProperty(Configuration.KEY_MUSIC_MIXIN_NAME, Configuration.MUSIC_MIXIN_DEFAULT); initMusic(); - initSqlDatabase(); - + initSqlDatabase(); + initTxDaemonThread(); String t = info.getProperty(Configuration.KEY_OWNERSHIP_TIMEOUT); - long timeoutMs = (t == null) ? Configuration.DEFAULT_OWNERSHIP_TIMEOUT : Integer.parseInt(t); + long timeout = (t == null) ? Configuration.DEFAULT_OWNERSHIP_TIMEOUT : Integer.parseInt(t); alreadyApplied = new ConcurrentHashMap<>(); - ownAndCheck = new OwnershipAndCheckpoint(alreadyApplied, timeoutMs); - - rangesToWarmup = initWarmupRanges(); - logger.info("Warmup ranges for this site is " + rangesToWarmup); + ownAndCheck = new OwnershipAndCheckpoint(alreadyApplied, timeout); + } - MusicTxDigest txDaemon = new MusicTxDigest(this); - txDaemon.startBackgroundDaemon(Integer.parseInt( - info.getProperty(Configuration.TX_DAEMON_SLEEPTIME_S, Configuration.TX_DAEMON_SLEEPTIME_S_DEFAULT))); + protected void initTxDaemonThread(){ + txDaemon = new Thread( + new MusicTxDigestDaemon(Integer.parseInt( + info.getProperty(Configuration.TX_DAEMON_SLEEPTIME_S, Configuration.TX_DAEMON_SLEEPTIME_S_DEFAULT)), + this)); + txDaemon.setName("TxDaemon"); + txDaemon.setDaemon(true); + txDaemon.start(); } /** @@ -163,6 +165,7 @@ public class StateManager { .append(";"); Statement stmt = sqlConnection.createStatement(); stmt.execute(sql.toString()); + sqlConnection.close(); } catch (SQLException e) { logger.error(EELFLoggerDelegate.errorLogger, e.getMessage(),AppMessages.UNKNOWNERROR, ErrorSeverity.CRITICAL, ErrorTypes.GENERALSERVICEERROR); @@ -243,7 +246,7 @@ public class StateManager { returnArray = new ArrayList<>(eventualRanges); } else{ - returnArray= null; + returnArray= new ArrayList<>(); } } finally{ @@ -291,7 +294,7 @@ public class StateManager { } if(connectionRanges.containsKey(connectionId)){ //We relinquish all locks obtained by a given - relinquish(connectionRanges.get(connectionId)); + //relinquish(connectionRanges.get(connectionId)); connectionRanges.remove(connectionId); } } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/ClusterConfiguration.java b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/ClusterConfiguration.java new file mode 100644 index 0000000..943fd46 --- /dev/null +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/ClusterConfiguration.java @@ -0,0 +1,88 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ +package org.onap.music.mdbc.configurations; + +import com.google.gson.Gson; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import org.onap.music.datastore.PreparedQueryObject; +import org.onap.music.exceptions.MDBCServiceException; +import org.onap.music.exceptions.MusicServiceException; +import org.onap.music.logging.EELFLoggerDelegate; +import org.onap.music.main.MusicCore; +import org.onap.music.mdbc.mixins.MusicMixin; + +public class ClusterConfiguration { + private String internalNamespace; + private int internalReplicationFactor; + private String musicNamespace; + private int musicReplicationFactor; + private String mriTableName; + private String mtxdTableName; + private String eventualMtxdTableName; + private String nodeInfoTableName; + private String rangeDependencyTableName; + + private transient static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(ClusterConfiguration.class); + + public void initNamespaces() throws MDBCServiceException{ + MusicMixin.createKeyspace(internalNamespace,internalReplicationFactor); + MusicMixin.createKeyspace(musicNamespace,musicReplicationFactor); + } + + public void initTables() throws MDBCServiceException{ + MusicMixin.createMusicRangeInformationTable(musicNamespace, mriTableName); + MusicMixin.createMusicTxDigest(mtxdTableName,musicNamespace, -1); + MusicMixin.createMusicEventualTxDigest(eventualMtxdTableName,musicNamespace, -1); + MusicMixin.createMusicNodeInfoTable(nodeInfoTableName,musicNamespace,-1); + MusicMixin.createMusicRangeDependencyTable(musicNamespace,rangeDependencyTableName); + } + + private void initInternalTable() throws MDBCServiceException { + StringBuilder createKeysTableCql = new StringBuilder("CREATE TABLE IF NOT EXISTS ") + .append(internalNamespace) + .append(".unsynced_keys (key text PRIMARY KEY);"); + PreparedQueryObject queryObject = new PreparedQueryObject(); + queryObject.appendQueryString(createKeysTableCql.toString()); + try { + MusicCore.createTable(internalNamespace,"unsynced_keys", queryObject,"critical"); + } catch (MusicServiceException e) { + logger.error("Error creating unsynced keys table" ); + throw new MDBCServiceException("Error creating unsynced keys table", e); + } + } + + public static ClusterConfiguration readJsonFromFile(String filepath) throws FileNotFoundException { + BufferedReader br; + try { + br = new BufferedReader( + new FileReader(filepath)); + } catch (FileNotFoundException e) { + logger.error(EELFLoggerDelegate.errorLogger,"File was not found when reading json"+e); + throw e; + } + Gson gson = new Gson(); + ClusterConfiguration config = gson.fromJson(br, ClusterConfiguration.class); + return config; + } + + +} diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/NodeConfiguration.java b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/NodeConfiguration.java index 5349219..38309d5 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/NodeConfiguration.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/NodeConfiguration.java @@ -50,6 +50,9 @@ public class NodeConfiguration { } protected List<Range> toRanges(String tables){ + if(tables.isEmpty()){ + return new ArrayList<>(); + } List<Range> newRange = new ArrayList<>(); String[] tablesArray=tables.split(","); for(String table: tablesArray) { diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/TablesConfiguration.java b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/TablesConfiguration.java index a9d179f..343a8b8 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/TablesConfiguration.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/TablesConfiguration.java @@ -20,14 +20,12 @@ package org.onap.music.mdbc.configurations; import com.datastax.driver.core.ResultSet; -import java.util.stream.Collectors; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.logging.EELFLoggerDelegate; import org.onap.music.mdbc.MDBCUtils; import org.onap.music.mdbc.Range; -import org.onap.music.mdbc.RedoRow; import org.onap.music.mdbc.mixins.MusicMixin; -import org.onap.music.mdbc.tables.MusicTxDigest; +import org.onap.music.mdbc.tables.MusicRangeInformationRow; import com.google.gson.Gson; import org.onap.music.datastore.PreparedQueryObject; @@ -48,11 +46,8 @@ public class TablesConfiguration { private transient static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(TablesConfiguration.class); private List<PartitionInformation> partitions; - private String internalNamespace; - private int internalReplicationFactor; + String tableToPartitionName; private String musicNamespace; - private int musicReplicationFactor; - private String tableToPartitionName; private String partitionInformationTableName; private String redoHistoryTableName; private String sqlDatabaseName; @@ -67,8 +62,6 @@ public class TablesConfiguration { */ public List<NodeConfiguration> initializeAndCreateNodeConfigurations() throws MDBCServiceException { logger.info("initializing the required spaces"); - createKeyspaces(); - initInternalNamespace(); List<NodeConfiguration> nodeConfigs = new ArrayList<>(); if(partitions == null){ @@ -77,10 +70,7 @@ public class TablesConfiguration { } for(PartitionInformation partitionInfo : partitions){ String mriTableName = partitionInfo.mriTableName; - checkIfMriIsEmpty(mriTableName); - //0) Create the corresponding Music Range Information table - MusicMixin.createMusicRangeInformationTable(musicNamespace,mriTableName); - + checkIfMriDoesNotExists(mriTableName,partitionInfo); String partitionId; if(partitionInfo.partitionId==null || partitionInfo.partitionId.isEmpty()){ //1) Create a row in the partition info table @@ -108,24 +98,7 @@ public class TablesConfiguration { return nodeConfigs; } - private void createKeyspaces() throws MDBCServiceException { - MusicMixin.createKeyspace(internalNamespace,internalReplicationFactor); - MusicMixin.createKeyspace(musicNamespace,musicReplicationFactor); - - } - - private void checkIfMriIsEmpty(String mriTableName) throws MDBCServiceException { - //First check if table exists - StringBuilder checkTableExistsString = new StringBuilder("SELECT table_name FROM system_schema.tables WHERE keyspace_name='") - .append(musicNamespace) - .append("';"); - PreparedQueryObject checkTableExists = new PreparedQueryObject(); - checkTableExists.appendQueryString(checkTableExistsString.toString()); - final ResultSet resultSet = MusicCore.quorumGet(checkTableExists); - if(resultSet.isExhausted()){ - //Table doesn't exist - return; - } + private void checkIfMriDoesNotExists(String mriTableName, PartitionInformation partition) throws MDBCServiceException { //If exists, check if empty StringBuilder checkRowsInTableString = new StringBuilder("SELECT * FROM ") .append(musicNamespace) @@ -134,27 +107,19 @@ public class TablesConfiguration { .append("';"); PreparedQueryObject checkRowsInTable = new PreparedQueryObject(); checkRowsInTable.appendQueryString(checkRowsInTableString.toString()); - final ResultSet resultSet2 = MusicCore.quorumGet(checkTableExists); - if(!resultSet2.isExhausted()) { - throw new MDBCServiceException("When initializing the configuration of the system, the MRI should not exits " - + "be empty"); + final ResultSet resultSet = MusicCore.quorumGet(checkRowsInTable); + while(resultSet!=null && !resultSet.isExhausted()){ + final MusicRangeInformationRow mriRowFromCassandraRow = MusicMixin.getMRIRowFromCassandraRow(resultSet.one()); + List<Range> ranges = mriRowFromCassandraRow.getDBPartition().getSnapshot(); + for(Range range: partition.getTables()) { + if (Range.overlaps(ranges,range.getTable())){ + throw new MDBCServiceException("MRI row already exists"); + } + } } } - private void initInternalNamespace() throws MDBCServiceException { - StringBuilder createKeysTableCql = new StringBuilder("CREATE TABLE IF NOT EXISTS ") - .append(internalNamespace) - .append(".unsynced_keys (key text PRIMARY KEY);"); - PreparedQueryObject queryObject = new PreparedQueryObject(); - queryObject.appendQueryString(createKeysTableCql.toString()); - try { - MusicCore.createTable(internalNamespace,"unsynced_keys", queryObject,"critical"); - } catch (MusicServiceException e) { - logger.error("Error creating unsynced keys table" ); - throw new MDBCServiceException("Error creating unsynced keys table", e); - } - } public static TablesConfiguration readJsonFromFile(String filepath) throws FileNotFoundException { BufferedReader br; @@ -174,7 +139,6 @@ public class TablesConfiguration { private List<Range> tables; private String owner; private String mriTableName; - private String mtxdTableName; private String partitionId; public List<Range> getTables() { @@ -209,12 +173,5 @@ public class TablesConfiguration { this.partitionId = partitionId; } - public String getMtxdTableName(){ - return mtxdTableName; - } - - public void setMtxdTableName(String mtxdTableName) { - this.mtxdTableName = mtxdTableName; - } } } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/examples/MdbcTestClient.java b/mdbc-server/src/main/java/org/onap/music/mdbc/examples/MdbcTestClient.java index fd8651a..60c97d1 100644 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/examples/MdbcTestClient.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/examples/MdbcTestClient.java @@ -51,8 +51,8 @@ public class MdbcTestClient { }
Connection connection;
try {
- String metricURL = "http://localhost:300000/test";
- if(args[0] != null) {
+ String metricURL = "http://localhost:30000/test";
+ if (args.length>0 && args[0] != null) {
metricURL = args[0];
}
connection = DriverManager.getConnection("jdbc:avatica:remote:url=" + metricURL+ ";serialization=protobuf");
@@ -60,7 +60,6 @@ public class MdbcTestClient { e.printStackTrace();
return;
}
-
try {
connection.setAutoCommit(false);
} catch (SQLException e) {
@@ -68,7 +67,6 @@ public class MdbcTestClient { return;
}
-
final String sql = "CREATE TABLE IF NOT EXISTS Persons (\n" +
" PersonID int,\n" +
" LastName varchar(255),\n" +
@@ -84,7 +82,6 @@ public class MdbcTestClient { e.printStackTrace();
return;
}
-
boolean execute = true;
// try {
// execute = stmt.execute(sql);
@@ -123,9 +120,9 @@ public class MdbcTestClient { }
try {
- //execute = insertStmt.execute(insertSQL);
+ execute = insertStmt.execute(insertSQL);
execute = insertStmt.execute(insertSQL1);
- //execute = insertStmt.execute(insertSQL2);
+ execute = insertStmt.execute(insertSQL2);
//execute = insertStmt.execute(insertSQL3);
//execute = insertStmt.execute(insertSQL4);
diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java index a514dc6..a594918 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java @@ -19,6 +19,7 @@ */ package org.onap.music.mdbc.mixins; +import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; @@ -130,4 +131,6 @@ public interface DBInterface { void enableForeignKeyChecks() throws SQLException; void applyTxDigest(StagingTable txDigest, List<Range> ranges) throws SQLException, MDBCServiceException; + + Connection getSQLConnection(); } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java index 00f6d00..8b91b28 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java @@ -21,11 +21,8 @@ package org.onap.music.mdbc.mixins; import com.datastax.driver.core.ResultSet; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; + import org.json.JSONObject; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.exceptions.MusicLockingException; @@ -230,12 +227,12 @@ public interface MusicInterface { /** * This function is used to append an index to the redo log in a MRI row - * @param partition information related to ownership of partitions, used to verify ownership + * @param MRIIndex index of the row to which the record is going to be added (obtained from the Partition) + * @param lockId reference to lock associated to the row in the MRI table MRIIndex. * @param newRecord index of the new record to be appended to the redo log * @throws MDBCServiceException */ - void appendToRedoLog( DatabasePartition partition, MusicTxDigestId newRecord) throws MDBCServiceException; - + void appendToRedoLog(UUID MRIIndex, String lockId, MusicTxDigestId newRecord)throws MDBCServiceException; /** * This functions adds the tx digest to * @param newId id used as index in the MTD table @@ -261,7 +258,7 @@ public interface MusicInterface { * @throws MDBCServiceException */ - public LinkedHashMap<UUID, StagingTable> getEveTxDigest(String nodeName) throws MDBCServiceException; + LinkedHashMap<UUID, StagingTable> getEveTxDigest(String nodeName) throws MDBCServiceException; /** * Function used to retrieve a given transaction digest and deserialize it * @param id of the transaction digest to be retrieved @@ -277,17 +274,6 @@ public interface MusicInterface { */ void relinquishIfRequired(DatabasePartition partition) throws MDBCServiceException; - /** - * This function is in charge of owning all the ranges requested and creating a new row that show the ownership of all - * those ranges. - * @param rangeId new id to be used in the new row - * @param ranges ranges to be owned by the end of the function called - * @param partition current ownership status - * @return - * @throws MDBCServiceException - */ - //OwnershipReturn appendRange(String rangeId, List<Range> ranges, DatabasePartition partition) throws MDBCServiceException; - /** * This functions relinquishes a range * @param ownerId id of the current ownerh @@ -318,10 +304,17 @@ public interface MusicInterface { List<MusicRangeInformationRow> getAllMriRows() throws MDBCServiceException; - public void updateNodeInfoTableWithTxTimeIDKey(UUID txTimeID, String nodeName) throws MDBCServiceException; - public LockResult requestLock(LockRequest request) throws MDBCServiceException; - public void releaseLocks(Map<UUID, LockResult> newLocks) throws MDBCServiceException; - public OwnershipReturn mergeLatestRows(Dag extendedDag, List<MusicRangeInformationRow> latestRows, List<Range> ranges, + + void deleteMriRow(MusicRangeInformationRow row) throws MDBCServiceException; + + void updateNodeInfoTableWithTxTimeIDKey(UUID txTimeID, String nodeName) throws MDBCServiceException; + + LockResult requestLock(LockRequest request) throws MDBCServiceException; + + void releaseLocks(Map<UUID, LockResult> newLocks) throws MDBCServiceException; + + OwnershipReturn mergeLatestRows(Dag extendedDag, List<MusicRangeInformationRow> latestRows, List<Range> ranges, Map<UUID, LockResult> locks, UUID ownershipId) throws MDBCServiceException; + } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java index c6cc512..e548f1a 100644 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java @@ -37,8 +37,9 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.BiFunction; +import java.util.concurrent.*; + +import com.datastax.driver.core.*; import org.apache.commons.lang3.tuple.Pair; import org.json.JSONObject; import org.onap.music.datastore.Condition; @@ -67,14 +68,6 @@ import org.onap.music.mdbc.tables.MusicTxDigestId; import org.onap.music.mdbc.tables.RangeDependency; import org.onap.music.mdbc.tables.StagingTable; import org.onap.music.mdbc.tables.TxCommitProgress; -import com.datastax.driver.core.BoundStatement; -import com.datastax.driver.core.ColumnDefinitions; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.PreparedStatement; -import com.datastax.driver.core.ResultSet; -import com.datastax.driver.core.Row; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.TupleValue; /** * This class provides the methods that MDBC needs to access Cassandra directly in order to provide persistence @@ -106,16 +99,19 @@ public class MusicMixin implements MusicInterface { public static final String KEY_MUSIC_RFACTOR = "music_rfactor"; /** The property name to use to provide the replication factor for Cassandra. */ public static final String KEY_MUSIC_NAMESPACE = "music_namespace"; + /** The property name to use to provide a flag indicating if compression is required */ + public static final String KEY_COMPRESSION = "mdbc_compression"; /** Namespace for the tables in MUSIC (Cassandra) */ public static final String DEFAULT_MUSIC_NAMESPACE = "namespace"; /** The default property value to use for the Cassandra IP address. */ public static final String DEFAULT_MUSIC_ADDRESS = "localhost"; /** The default property value to use for the Cassandra replication factor. */ - public static final int DEFAULT_MUSIC_RFACTOR = 1; + public static final int DEFAULT_MUSIC_RFACTOR = 3; /** The default primary string column, if none is provided. */ public static final String MDBC_PRIMARYKEY_NAME = "mdbc_cuid"; /** Type of the primary key, if none is defined by the user */ public static final String MDBC_PRIMARYKEY_TYPE = "uuid"; + public static final boolean DEFAULT_COMPRESSION = true; //\TODO Add logic to change the names when required and create the tables when necessary @@ -178,9 +174,12 @@ public class MusicMixin implements MusicInterface { //typemap.put(Types.DATE, "TIMESTAMP"); } + protected final String music_ns; protected final String myId; protected final String[] allReplicaIds; + protected ExecutorService commitExecutorThreads; + private final String musicAddress; private final int music_rfactor; private MusicConnector mCon = null; @@ -189,7 +188,7 @@ public class MusicMixin implements MusicInterface { private Map<String, PreparedStatement> ps_cache = new HashMap<>(); private Set<String> in_progress = Collections.synchronizedSet(new HashSet<String>()); private StateManager stateManager; - + private boolean useCompression; public MusicMixin() { @@ -206,14 +205,16 @@ public class MusicMixin implements MusicInterface { // Default to using the host_ids of the various peers as the replica IDs (this is probably preferred) this.musicAddress = info.getProperty(KEY_MUSIC_ADDRESS, DEFAULT_MUSIC_ADDRESS); logger.info(EELFLoggerDelegate.applicationLogger,"MusicSqlManager: musicAddress="+musicAddress); - - String s = info.getProperty(KEY_MUSIC_RFACTOR); - this.music_rfactor = (s == null) ? DEFAULT_MUSIC_RFACTOR : Integer.parseInt(s); + MusicDataStore dsHandle = null; + try { + dsHandle = MusicDataStoreHandle.getDSHandle(); + } catch (MusicServiceException e) { + e.printStackTrace(); + } this.myId = info.getProperty(KEY_MY_ID, getMyHostId()); logger.info(EELFLoggerDelegate.applicationLogger,"MusicSqlManager: myId="+myId); - this.allReplicaIds = info.getProperty(KEY_REPLICAS, getAllHostIds()).split(","); logger.info(EELFLoggerDelegate.applicationLogger,"MusicSqlManager: allReplicaIds="+info.getProperty(KEY_REPLICAS, this.myId)); @@ -222,7 +223,14 @@ public class MusicMixin implements MusicInterface { this.stateManager = stateManager; + String c = info.getProperty(KEY_COMPRESSION); + this.useCompression = (c == null) ? DEFAULT_COMPRESSION: Boolean.parseBoolean(c); + + String s = info.getProperty(KEY_MUSIC_RFACTOR); + this.music_rfactor = (s == null) ? DEFAULT_MUSIC_RFACTOR : Integer.parseInt(s); + initializeMetricTables(); + commitExecutorThreads = Executors.newFixedThreadPool(4); } public String getMusicTxDigestTableName(){ @@ -244,8 +252,16 @@ public class MusicMixin implements MusicInterface { public static void createKeyspace(String keyspace, int replicationFactor) throws MDBCServiceException { Map<String,Object> replicationInfo = new HashMap<>(); - replicationInfo.put("'class'", "'SimpleStrategy'"); - replicationInfo.put("'replication_factor'", replicationFactor); + replicationInfo.put("'class'", "'NetworkTopologyStrategy'"); + if(replicationFactor==3){ + replicationInfo.put("'dc1'", 1); + replicationInfo.put("'dc2'", 1); + replicationInfo.put("'dc3'", 1); + } + else { + replicationInfo.put("'class'", "'SimpleStrategy'"); + replicationInfo.put("'replication_factor'", replicationFactor); + } PreparedQueryObject queryObject = new PreparedQueryObject(); queryObject.appendQueryString( @@ -317,7 +333,7 @@ public class MusicMixin implements MusicInterface { createMusicEventualTxDigest(); createMusicNodeInfoTable(); createMusicRangeInformationTable(this.music_ns,this.musicRangeInformationTableName); - createMusicRangeDependencyTable(); + createMusicRangeDependencyTable(this.music_ns,this.musicRangeDependencyTableName); } catch(MDBCServiceException e){ logger.error(EELFLoggerDelegate.errorLogger,"Error creating tables in MUSIC"); @@ -1207,67 +1223,6 @@ public class MusicMixin implements MusicInterface { return previous.keySet().equals(current.keySet()); } - protected List<Range> waitForLock(LockRequest request, DatabasePartition partition, - Map<UUID, LockResult> rowLock) throws MDBCServiceException { - List<Range> newRanges = new ArrayList<>(); - if(partition.getMRIIndex()!=request.getId()){ - throw new MDBCServiceException("Invalid argument for wait for lock, range id in request and partition should match"); - } - String fullyQualifiedKey= music_ns+"."+ request.getTable()+"."+request.getId(); - String lockId = MusicCore.createLockReference(fullyQualifiedKey); - ReturnType lockReturn = acquireLock(fullyQualifiedKey,lockId); - if(lockReturn.getResult().compareTo(ResultType.SUCCESS) != 0 ) { - //\TODO Improve the exponential backoff - List<Range> pendingToLock = request.getToLockRanges(); - Map<UUID, String> currentLockRef = new HashMap<>(); - int n = 1; - int low = 1; - int high = 1000; - Random r = new Random(); - Map<Range, RangeMriRow> rangeRows = findRangeRows(pendingToLock); - NavigableMap<UUID, List<Range>> rowsToLock = getPendingRows(rangeRows); - NavigableMap<UUID, List<Range>> prevRows = new TreeMap<>(); - while (!pendingToLock.isEmpty() && isDifferent(prevRows,rowsToLock) ) { - pendingToLock.clear(); - try { - Thread.sleep(((int) Math.round(Math.pow(2, n)) * 1000) - + (r.nextInt(high - low) + low)); - } catch (InterruptedException e) { - continue; - } - n++; - if (n == 20) { - throw new MDBCServiceException("Lock was impossible to obtain, waited for 20 exponential backoffs!"); - } - //\TODO do this in parallel - //\TODO there is a race condition here, from the time we get the find range rows, to the time we lock the row, - //\TODO this race condition can only be solved if require to obtain lock to all related rows in MRI - //\TODO before fully owning the range - //\TODO The rows need to be lock in increasing order of timestamp - //there could be a new row created - // Note: This loop needs to be perfomed in sorted order of timebased UUID - for (Map.Entry<UUID, List<Range>> pending : rowsToLock.entrySet()) { - List<Range> rs = lockRow(request, pending, currentLockRef, fullyQualifiedKey, lockId, pendingToLock, rowLock); - newRanges.addAll(rs); - } - if (n++ == 20) { - throw new MDBCServiceException( - "Lock was impossible to obtain, waited for 20 exponential backoffs!"); - } - rangeRows = findRangeRows(pendingToLock); - prevRows = rowsToLock; - rowsToLock = getPendingRows(rangeRows); - } - } - else { - partition.setLockId(lockId); - rowLock.put(partition.getMRIIndex(),new LockResult(partition.getMRIIndex(), lockId, true, partition.getSnapshot())); - } - return newRanges; - } - - - protected String createAndAssignLock(String fullyQualifiedKey, DatabasePartition partition) throws MDBCServiceException { UUID mriIndex = partition.getMRIIndex(); String lockId; @@ -1315,13 +1270,14 @@ public class MusicMixin implements MusicInterface { } } - protected void appendIndexToMri(String lockId, UUID commitId, UUID MriIndex) throws MDBCServiceException{ - PreparedQueryObject appendQuery = createAppendMtxdIndexToMriQuery(musicRangeInformationTableName, MriIndex, musicTxDigestTableName, commitId); - ReturnType returnType = MusicCore.criticalPut(music_ns, musicRangeInformationTableName, MriIndex.toString(), appendQuery, lockId, null); - if(returnType.getResult().compareTo(ResultType.SUCCESS) != 0 ){ - logger.error(EELFLoggerDelegate.errorLogger, "Error when executing append operation with return type: "+returnType.getMessage()); - throw new MDBCServiceException("Error when executing append operation with return type: "+returnType.getMessage()); + public void createAndAddTxDigest(final StagingTable transactionDigest, UUID digestId) + throws MDBCServiceException { + ByteBuffer serializedTransactionDigest; + serializedTransactionDigest = transactionDigest.getSerializedStagingAndClean(); + if(useCompression){ + serializedTransactionDigest = StagingTable.Compress(serializedTransactionDigest); } + addTxDigest(digestId, serializedTransactionDigest); } /** @@ -1331,9 +1287,7 @@ public class MusicMixin implements MusicInterface { @Override public void commitLog(DatabasePartition partition,List<Range> eventualRanges, StagingTable transactionDigest, String txId ,TxCommitProgress progressKeeper) throws MDBCServiceException { - // first deal with commit for eventually consistent tables - filterAndAddEventualTxDigest(eventualRanges, transactionDigest, txId, progressKeeper); - + if(partition==null){ logger.warn("Trying tcommit log with null partition"); return; @@ -1343,7 +1297,9 @@ public class MusicMixin implements MusicInterface { logger.warn("Trying to commit log with empty ranges"); return; } - + + // first deal with commit for eventually consistent tables + filterAndAddEventualTxDigest(eventualRanges, transactionDigest, txId, progressKeeper); UUID mriIndex = partition.getMRIIndex(); String fullyQualifiedMriKey = music_ns+"."+ this.musicRangeInformationTableName+"."+mriIndex; @@ -1353,54 +1309,70 @@ public class MusicMixin implements MusicInterface { throw new MDBCServiceException("Not able to commit, as you are no longer the lock-holder for this partition"); } - UUID commitId; - //Generate a local commit id - if(progressKeeper.containsTx(txId)) { - commitId = progressKeeper.getCommitId(txId); - } - else{ - logger.error(EELFLoggerDelegate.errorLogger, "Tx with id "+txId+" was not created in the TxCommitProgress "); - throw new MDBCServiceException("Tx with id "+txId+" was not created in the TxCommitProgress "); - } //Add creation type of transaction digest - - //1. Push new row to RRT and obtain its index if(transactionDigest == null || transactionDigest.isEmpty()) { return; } - - ByteBuffer serializedTransactionDigest; - if(!transactionDigest.isEmpty()) { - serializedTransactionDigest = transactionDigest.getSerializedStagingAndClean(); - MusicTxDigestId digestId = new MusicTxDigestId(mriIndex, -1); - addTxDigest(digestId, serializedTransactionDigest); - //2. Save RRT index to RQ - if (progressKeeper != null) { - progressKeeper.setRecordId(txId, digestId); + + final MusicTxDigestId digestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1); + Callable<Boolean> insertDigestCallable =()-> { + try { + createAndAddTxDigest(transactionDigest,digestId.transactionId); + return true; + } catch (MDBCServiceException e) { + logger.error(EELFLoggerDelegate.errorLogger, "Error creating and pushing tx digest to music",e); + return false; } - //3. Append RRT index into the corresponding TIT row array - appendToRedoLog(partition, digestId); - List<Range> ranges = partition.getSnapshot(); - for(Range r : ranges) { - Map<Range, Pair<MriReference, Integer>> alreadyApplied = stateManager.getOwnAndCheck().getAlreadyApplied(); - if(!alreadyApplied.containsKey(r)){ - throw new MDBCServiceException("already applied data structure was not updated correctly and range " + }; + Callable<Boolean> appendCallable=()-> { + try { + appendToRedoLog(music_ns, mriIndex, digestId.transactionId, lockId, musicTxDigestTableName, + musicRangeInformationTableName); + return true; + } catch (MDBCServiceException e) { + logger.error(EELFLoggerDelegate.errorLogger, "Error creating and pushing tx digest to music",e); + return false; + } + }; + + Future<Boolean> appendResultFuture = commitExecutorThreads.submit(appendCallable); + Future<Boolean> digestFuture = commitExecutorThreads.submit(insertDigestCallable); + try { + //Boolean appendResult = appendResultFuture.get(); + Boolean digestResult = digestFuture.get(); + if(/*!appendResult ||*/ !digestResult){ + logger.error(EELFLoggerDelegate.errorLogger, "Error appending to log or adding tx digest"); + throw new MDBCServiceException("Error appending to log or adding tx digest"); + } + } catch (InterruptedException|ExecutionException e) { + logger.error(EELFLoggerDelegate.errorLogger, "Error executing futures for creating and pushing tx " + + "digest to music",e); + throw new MDBCServiceException("Failure when retrieving futures for execution of digestion creation and append", e); + } + + if (progressKeeper != null) { + progressKeeper.setRecordId(txId, digestId); + } + List<Range> ranges = partition.getSnapshot(); + for(Range r : ranges) { + Map<Range, Pair<MriReference, Integer>> alreadyApplied = stateManager.getOwnAndCheck().getAlreadyApplied(); + if(!alreadyApplied.containsKey(r)){ + throw new MDBCServiceException("already applied data structure was not updated correctly and range " +r+" is not contained"); - } - Pair<MriReference, Integer> rowAndIndex = alreadyApplied.get(r); - MriReference key = rowAndIndex.getKey(); - if(!mriIndex.equals(key.index)){ - throw new MDBCServiceException("already applied data structure was not updated correctly and range "+ - r+" is not pointing to row: "+mriIndex.toString()); - } - alreadyApplied.put(r, Pair.of(new MriReference(mriIndex), rowAndIndex.getValue()+1)); } + Pair<MriReference, Integer> rowAndIndex = alreadyApplied.get(r); + MriReference key = rowAndIndex.getKey(); + if(!mriIndex.equals(key.index)){ + throw new MDBCServiceException("already applied data structure was not updated correctly and range "+ + r+" is not pointing to row: "+mriIndex.toString()); + } + alreadyApplied.put(r, Pair.of(new MriReference(mriIndex), rowAndIndex.getValue()+1)); } } private void filterAndAddEventualTxDigest(List<Range> eventualRanges, - StagingTable transactionDigest, String txId, - TxCommitProgress progressKeeper) throws MDBCServiceException { + StagingTable transactionDigest, String txId, + TxCommitProgress progressKeeper) throws MDBCServiceException { if(eventualRanges == null || eventualRanges.isEmpty()) { return; @@ -1410,30 +1382,19 @@ public class MusicMixin implements MusicInterface { throw new MDBCServiceException(); } - UUID commitId = getCommitId(txId, progressKeeper); + if(!transactionDigest.isEmpty()) { + ByteBuffer serialized = transactionDigest.getSerializedEventuallyStagingAndClean(); - ByteBuffer serialized = transactionDigest.getSerializedEventuallyStagingAndClean(); + if (serialized!=null && useCompression) { + serialized = StagingTable.Compress(serialized); + } - if(serialized != null ) { - MusicTxDigestId digestId = new MusicTxDigestId(commitId,-1); - addEventualTxDigest(digestId, serialized); + if (serialized != null) { + MusicTxDigestId digestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1); + addEventualTxDigest(digestId, serialized); + } } - - } - - private UUID getCommitId(String txId, TxCommitProgress progressKeeper) - throws MDBCServiceException { - UUID commitId; - //Generate a local commit id - if(progressKeeper.containsTx(txId)) { - commitId = progressKeeper.getCommitId(txId); - } - else{ - logger.error(EELFLoggerDelegate.errorLogger, "Tx with id "+txId+" was not created in the TxCommitProgress "); - throw new MDBCServiceException("Tx with id "+txId+" was not created in the TxCommitProgress "); - } - return commitId; } /** @@ -1503,7 +1464,7 @@ public class MusicMixin implements MusicInterface { return partitions; } - public MusicRangeInformationRow getMRIRowFromCassandraRow(Row newRow){ + static public MusicRangeInformationRow getMRIRowFromCassandraRow(Row newRow){ UUID partitionIndex = newRow.getUUID("rangeid"); List<TupleValue> log = newRow.getList("txredolog",TupleValue.class); List<MusicTxDigestId> digestIds = new ArrayList<>(); @@ -1608,9 +1569,13 @@ public class MusicMixin implements MusicInterface { DatabasePartition newPartition = info.getDBPartition(); String fullyQualifiedMriKey = music_ns+"."+ musicRangeInformationTableName+"."+newPartition.getMRIIndex().toString(); - String lockId = createAndAssignLock(fullyQualifiedMriKey, newPartition); + String lockId; + int counter=0; + do { + lockId = createAndAssignLock(fullyQualifiedMriKey, newPartition); //TODO: fix this retry logic - if(lockId == null || lockId.isEmpty()){ + } while ((lockId ==null||lockId.isEmpty())&&(counter++<3)); + if (lockId == null || lockId.isEmpty()) { throw new MDBCServiceException("Error initializing music range information, error creating a lock for a new row" + "for key "+fullyQualifiedMriKey) ; } @@ -1632,7 +1597,7 @@ public class MusicMixin implements MusicInterface { .append(rangeAndDependencies.getRange().getTable()) .append(",{"); boolean first=true; - for(Range r: rangeAndDependencies.dependentRanges()){ + for (Range r: rangeAndDependencies.dependentRanges()) { if(first){ first=false; } else { insert.append(','); @@ -1658,7 +1623,7 @@ public class MusicMixin implements MusicInterface { .append(id) .append(",{"); boolean first=true; - for(Range r: rangesCopy){ + for (Range r: rangesCopy) { if(first){ first=false; } else { insert.append(','); @@ -1706,7 +1671,7 @@ public class MusicMixin implements MusicInterface { .append(id) .append(",{"); boolean first=true; - for(Range r: ranges){ + for (Range r: ranges) { if(first){ first=false; } else { insert.append(','); @@ -1731,24 +1696,32 @@ public class MusicMixin implements MusicInterface { } @Override - public void appendToRedoLog(DatabasePartition partition, MusicTxDigestId newRecord) throws MDBCServiceException { - logger.info("Appending to redo log for partition " + partition.getMRIIndex() + " txId=" + newRecord.txId); - PreparedQueryObject appendQuery = createAppendMtxdIndexToMriQuery(musicRangeInformationTableName, partition.getMRIIndex(), - musicTxDigestTableName, newRecord.txId); - ReturnType returnType = MusicCore.criticalPut(music_ns, musicRangeInformationTableName, partition.getMRIIndex().toString(), - appendQuery, partition.getLockId(), null); - if(returnType.getResult().compareTo(ResultType.SUCCESS) != 0 ){ + public void appendToRedoLog(UUID MRIIndex, String lockId, MusicTxDigestId newRecord) throws MDBCServiceException { + logger.debug("Appending to redo log for partition " + MRIIndex + " txId=" + newRecord.transactionId); + appendToRedoLog(music_ns,MRIIndex,newRecord.transactionId,lockId,musicTxDigestTableName, + musicRangeInformationTableName); + } + + public void appendToRedoLog(String musicNamespace, UUID MRIIndex, UUID transactionId, String lockId, + String musicTxDigestTableName, String musicRangeInformationTableName) + throws MDBCServiceException{ + PreparedQueryObject appendQuery = createAppendMtxdIndexToMriQuery(musicRangeInformationTableName, MRIIndex, + musicTxDigestTableName, transactionId); + ReturnType returnType = MusicCore.criticalPut(musicNamespace, musicRangeInformationTableName, MRIIndex.toString(), + appendQuery, lockId, null); + //returnType.getExecutionInfo() + if (returnType.getResult().compareTo(ResultType.SUCCESS) != 0) { logger.error(EELFLoggerDelegate.errorLogger, "Error when executing append operation with return type: "+returnType.getMessage()); throw new MDBCServiceException("Error when executing append operation with return type: "+returnType.getMessage()); } } public void createMusicTxDigest() throws MDBCServiceException { - createMusicTxDigest(-1); + createMusicTxDigest(this.musicTxDigestTableName,this.music_ns,-1); } public void createMusicEventualTxDigest() throws MDBCServiceException { - createMusicEventualTxDigest(-1); + createMusicEventualTxDigest(musicEventualTxDigestTableName,music_ns,-1); } @@ -1758,9 +1731,9 @@ public class MusicMixin implements MusicInterface { * * LeaseCounter: transaction number under this lease, bigint \TODO this may need to be a varint later * * TransactionDigest: text that contains all the changes in the transaction */ - private void createMusicEventualTxDigest(int musicTxDigestTableNumber) throws MDBCServiceException { - String tableName = this.musicEventualTxDigestTableName; - if(musicTxDigestTableNumber >= 0) { + public static void createMusicEventualTxDigest(String musicEventualTxDigestTableName, String musicNamespace, int musicTxDigestTableNumber) throws MDBCServiceException { + String tableName = musicEventualTxDigestTableName; + if (musicTxDigestTableNumber >= 0) { tableName = tableName + "-" + Integer.toString(musicTxDigestTableNumber); @@ -1769,12 +1742,13 @@ public class MusicMixin implements MusicInterface { StringBuilder fields = new StringBuilder(); fields.append("txid uuid, "); fields.append("transactiondigest blob, "); + fields.append("compressed boolean, "); fields.append("txTimeId TIMEUUID ");//notice lack of ',' - String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName, fields, priKey); + String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", musicNamespace, tableName, fields, priKey); try { - executeMusicWriteQuery(this.music_ns,tableName,cql); + executeMusicWriteQuery(musicNamespace,tableName,cql); } catch (MDBCServiceException e) { - logger.error("Initialization error: Failure to create redo records table"); + logger.error("Initialization error: Failure to create eventual tx digest table"); throw(e); } } @@ -1786,9 +1760,9 @@ public class MusicMixin implements MusicInterface { * * LeaseCounter: transaction number under this lease, bigint \TODO this may need to be a varint later * * TransactionDigest: text that contains all the changes in the transaction */ - private void createMusicTxDigest(int musicTxDigestTableNumber) throws MDBCServiceException { - String tableName = this.musicTxDigestTableName; - if(musicTxDigestTableNumber >= 0) { + public static void createMusicTxDigest(String musicTxDigestTableName, String musicNamespace, int musicTxDigestTableNumber) throws MDBCServiceException { + String tableName = musicTxDigestTableName; + if (musicTxDigestTableNumber >= 0) { tableName = tableName + "-" + Integer.toString(musicTxDigestTableNumber); @@ -1796,26 +1770,29 @@ public class MusicMixin implements MusicInterface { String priKey = "txid"; StringBuilder fields = new StringBuilder(); fields.append("txid uuid, "); + fields.append("compressed boolean, "); fields.append("transactiondigest blob ");//notice lack of ',' - String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName, fields, priKey); + String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", musicNamespace, + tableName, fields, priKey); try { - executeMusicWriteQuery(this.music_ns,tableName,cql); + executeMusicWriteQuery(musicNamespace,tableName,cql); } catch (MDBCServiceException e) { logger.error("Initialization error: Failure to create redo records table"); throw(e); } } - private void createMusicRangeDependencyTable() throws MDBCServiceException { - String tableName = this.musicRangeDependencyTableName; + public static void createMusicRangeDependencyTable(String musicNamespace,String musicRangeDependencyTableName) + throws MDBCServiceException { + String tableName = musicRangeDependencyTableName; String priKey = "range"; StringBuilder fields = new StringBuilder(); fields.append("range text, "); fields.append("dependencies set<text> ");//notice lack of ',' - String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName, + String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", musicNamespace, tableName, fields, priKey); try { - executeMusicWriteQuery(this.music_ns,tableName,cql); + executeMusicWriteQuery(musicNamespace,tableName,cql); } catch (MDBCServiceException e) { logger.error("Initialization error: Failure to create redo records table"); throw(e); @@ -1828,18 +1805,23 @@ public class MusicMixin implements MusicInterface { @Override public void addTxDigest(MusicTxDigestId newId, ByteBuffer transactionDigest) throws MDBCServiceException { //\TODO: Save Prepared query to history + addTxDigest(newId.transactionId,transactionDigest); + } + + private void addTxDigest(UUID digestId, ByteBuffer transactionDigest) throws MDBCServiceException{ PreparedQueryObject query = new PreparedQueryObject(); - String cql = String.format("INSERT INTO %s.%s (txid,transactiondigest) VALUES (?,?);",this.music_ns, + String cql = String.format("INSERT INTO %s.%s (txid,transactiondigest,compressed ) VALUES (?,?,?);",this.music_ns, this.musicTxDigestTableName); query.appendQueryString(cql); - query.addValue(newId.txId); + query.addValue(digestId); query.addValue(transactionDigest); + query.addValue(useCompression); //\TODO check if I am not shooting on my own foot try { MusicCore.nonKeyRelatedPut(query,"critical"); } catch (MusicServiceException e) { - logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for commit "+newId.txId.toString()+ "with error "+e.getErrorMessage()); - throw new MDBCServiceException("Transaction Digest serialization for commit "+newId.txId.toString(), e); + logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for digest id "+digestId.toString()+ "with error "+e.getErrorMessage()); + throw new MDBCServiceException("Transaction Digest serialization for digest id "+digestId.toString(), e); } } @@ -1854,11 +1836,11 @@ public class MusicMixin implements MusicInterface { this.music_ns + '.' + this.musicEventualTxDigestTableName + - " (txid,transactiondigest,txTimeId) " + + " (txid,transactiondigest,compressed,txTimeId) " + "VALUES (" + - newId.txId + ",'" + - transactionDigest + - "'," + + newId.transactionId+ ",'" + + transactionDigest + "'," + + useCompression + ","+ // "toTimestamp(now())" + "now()" + ");"; @@ -1867,8 +1849,8 @@ public class MusicMixin implements MusicInterface { try { MusicCore.nonKeyRelatedPut(query,"critical"); } catch (MusicServiceException e) { - logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for commit "+newId.txId.toString()+ "with error "+e.getErrorMessage()); - throw new MDBCServiceException("Transaction Digest serialization for commit "+newId.txId.toString(), e); + logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for commit "+newId.transactionId.toString()+ "with error "+e.getErrorMessage()); + throw new MDBCServiceException("Transaction Digest serialization for commit "+newId.transactionId.toString(), e); } } @@ -1877,17 +1859,21 @@ public class MusicMixin implements MusicInterface { String cql = String.format("SELECT * FROM %s.%s WHERE txid = ?;", music_ns, musicTxDigestTableName); PreparedQueryObject pQueryObject = new PreparedQueryObject(); pQueryObject.appendQueryString(cql); - pQueryObject.addValue(id.txId); + pQueryObject.addValue(id.transactionId); Row newRow; try { newRow = executeMusicUnlockedQuorumGet(pQueryObject); } catch (MDBCServiceException e) { - logger.error("Get operation error: Failure to get row from txdigesttable with id:"+id.txId); + logger.error("Get operation error: Failure to get row from txdigesttable with id:"+id.transactionId); throw new MDBCServiceException("Initialization error:Failure to add new row to transaction information", e); } ByteBuffer digest = newRow.getBytes("transactiondigest"); + Boolean compressed = newRow.getBool("compressed"); StagingTable changes; try { + if(compressed){ + digest = StagingTable.Decompress(digest); + } changes = new StagingTable(digest); } catch (MDBCServiceException e) { logger.error("Deserializng digest failed with an exception:"+e.getErrorMessage()); @@ -1921,10 +1907,14 @@ public class MusicMixin implements MusicInterface { while (!rs.isExhausted()) { Row row = rs.one(); ByteBuffer digest = row.getBytes("transactiondigest"); + Boolean compressed = row.getBool("compressed"); //String txTimeId = row.getString("txtimeid"); //??? UUID txTimeId = row.getUUID("txtimeid"); try { + if(compressed){ + digest=StagingTable.Decompress(digest); + } changes = new StagingTable(digest); } catch (MDBCServiceException e) { logger.error("Deserializng digest failed: "+e.getErrorMessage()); @@ -1935,9 +1925,6 @@ public class MusicMixin implements MusicInterface { return ecDigestInformation; } - - - ResultSet getAllMriCassandraRows() throws MDBCServiceException { StringBuilder cqlOperation = new StringBuilder(); cqlOperation.append("SELECT * FROM ") @@ -1959,34 +1946,6 @@ public class MusicMixin implements MusicInterface { return rows; } - private RangeMriRow findRangeRow(Range range) throws MDBCServiceException { - RangeMriRow row = null; - final ResultSet musicResults = getAllMriCassandraRows(); - while (!musicResults.isExhausted()) { - Row musicRow = musicResults.one(); - final MusicRangeInformationRow mriRow = getMRIRowFromCassandraRow(musicRow); - final List<Range> musicRanges = getRanges(musicRow); - //\TODO optimize this for loop to avoid redudant access - for(Range retrievedRange : musicRanges) { - if (retrievedRange.overlaps(range)) { - if(row==null){ - row = new RangeMriRow(range); - row.setCurrentRow(mriRow); - } - else if(row.getCurrentRow().getTimestamp() < mriRow.getTimestamp()){ - row.addOldRow(row.getCurrentRow()); - row.setCurrentRow(mriRow); - } - } - } - } - if(row==null){ - logger.error("Row in MRI doesn't exist for Range "+range.toString()); - throw new MDBCServiceException("Row in MRI doesn't exist for Range "+range.toString()); - } - return row; - } - /** * This function is used to find all the related uuids associated with the required ranges * @param ranges ranges to be find @@ -2037,18 +1996,6 @@ public class MusicMixin implements MusicInterface { return result; } - private List<Range> lockRow(LockRequest request, DatabasePartition partition,Map<UUID, LockResult> rowLock) - throws MDBCServiceException { - if(partition.getMRIIndex().equals(request.getId()) && partition.isLocked()){ - return new ArrayList<>(); - } - //\TODO: this function needs to be improved, to track possible changes in the owner of a set of ranges - String fullyQualifiedMriKey = music_ns+"."+ this.musicRangeInformationTableName+"."+request.getId().toString(); - //return List<Range> knownRanges, UUID mriIndex, String lockId - DatabasePartition newPartition = new DatabasePartition(request.getToLockRanges(),request.getId(),null); - return waitForLock(request,newPartition,rowLock); - } - private void unlockKeyInMusic(String table, String key, String lockref) { String fullyQualifiedKey= music_ns+"."+ table+"."+lockref; MusicCore.destroyLockRef(fullyQualifiedKey,lockref); @@ -2182,8 +2129,6 @@ public class MusicMixin implements MusicInterface { return new OwnershipReturn(ownershipId, ownRow.getOwnerId(), ownRow.getIndex(),ranges,extendedDag); } - - /** * This function is used to check if we need to create a new row in MRI, beacause one of the new ranges is not contained * @param ranges ranges that should be contained in the partition @@ -2199,78 +2144,7 @@ public class MusicMixin implements MusicInterface { return false; } - private Map<UUID,String> mergeMriRows(String newId, Map<UUID,LockResult> lock, DatabasePartition partition) - throws MDBCServiceException { - Map<UUID,String> oldIds = new HashMap<>(); - List<Range> newRanges = new ArrayList<>(); - for (Map.Entry<UUID,LockResult> entry : lock.entrySet()) { - oldIds.put(entry.getKey(),entry.getValue().getOwnerId()); - //\TODO check if we need to do a locked get? Is that even required? - final MusicRangeInformationRow mriRow = getMusicRangeInformation(entry.getKey()); - final DatabasePartition dbPartition = mriRow.getDBPartition(); - newRanges.addAll(dbPartition.getSnapshot()); - } - DatabasePartition newPartition = new DatabasePartition(newRanges,UUID.fromString(newId),null); - String fullyQualifiedMriKey = music_ns+"."+ this.musicRangeInformationTableName+"."+newId; - UUID newUUID = UUID.fromString(newId); - LockRequest newRequest = new LockRequest(musicRangeInformationTableName,newUUID,newRanges); - waitForLock(newRequest, newPartition,lock); - if(!lock.containsKey(newUUID)||!lock.get(newUUID).isNewLock()){ - logger.error("When merging rows, lock returned an invalid error"); - throw new MDBCServiceException("When merging MRI rows, lock returned an invalid error"); - } - final LockResult lockResult = lock.get(newUUID); - partition.updateDatabasePartition(newPartition); - logger.info("Creating MRI " +partition.getMRIIndex()+ " for ranges " + partition.getSnapshot()); - createEmptyMriRow(this.music_ns,this.musicRangeInformationTableName,partition.getMRIIndex(),myId, - lockResult.getOwnerId(),partition.getSnapshot(),true); - return oldIds; - } - - private void obtainAllLocks(NavigableMap<UUID, List<Range>> rowsToLock,DatabasePartition partition, - List<Range> newRanges,Map<UUID, LockResult> rowLock) throws MDBCServiceException { - //\TODO: perform this operations in parallel - for(Map.Entry<UUID,List<Range>> row : rowsToLock.entrySet()){ - List<Range> additionalRanges; - try { - LockRequest newRequest = new LockRequest(musicRangeInformationTableName,row.getKey(),row.getValue()); - additionalRanges =lockRow(newRequest, partition, rowLock); - } catch (MDBCServiceException e) { - //TODO: Make a decision if retry or just fail? - logger.error("Error locking row",e); - throw e; - } - newRanges.addAll(additionalRanges); - } - } - -/* @Override - public OwnershipReturn appendRange(String rangeId, List<Range> ranges, DatabasePartition partition) - throws MDBCServiceException { - if(!isAppendRequired(ranges,partition)){ - return new OwnershipReturn(partition.getLockId(),UUID.fromString(rangeId),null,null); - } - Map<Range, RangeMriRow> rows = findRangeRows(ranges); - final NavigableMap<UUID, List<Range>> rowsToLock = getPendingRows(rows); - HashMap<UUID, LockResult> rowLock = new HashMap<>(); - List<Range> newRanges = new ArrayList<>(); - - obtainAllLocks(rowsToLock,partition,newRanges,rowLock); - String lockId; - Map<UUID,String> oldIds = null; - if(rowLock.size()!=1){ - oldIds = mergeMriRows(rangeId, rowLock, partition); - lockId = partition.getLockId(); - } - else{ - List<LockResult> list = new ArrayList<>(rowLock.values()); - LockResult lockResult = list.get(0); - lockId = lockResult.getOwnerId(); - } - - return new OwnershipReturn(lockId,UUID.fromString(rangeId),oldIds,newRanges); - }*/ @Override public void relinquish(String ownerId, String rangeId) throws MDBCServiceException{ @@ -2290,7 +2164,9 @@ public class MusicMixin implements MusicInterface { * @return true if we should try to relinquish, else should avoid relinquishing in this iteration */ private boolean canTryRelinquishing(){ - return true; + //\TODO: Fix this!!!! REALLY IMPORTANT TO BE FIX + // This should actually have some mechanism to relinquish ownership + return false; } @Override @@ -2446,7 +2322,7 @@ public class MusicMixin implements MusicInterface { } public void createMusicNodeInfoTable() throws MDBCServiceException { - createMusicNodeInfoTable(-1); + createMusicNodeInfoTable(musicNodeInfoTableName,music_ns,-1); } /** @@ -2460,8 +2336,8 @@ public class MusicMixin implements MusicInterface { * * TxTimeID, TIMEUUID. * * LastTxDigestID, uuid. (not needed as of now!!) */ - private void createMusicNodeInfoTable(int nodeInfoTableNumber) throws MDBCServiceException { - String tableName = this.musicNodeInfoTableName; + public static void createMusicNodeInfoTable(String musicNodeInfoTableName, String musicNamespace, int nodeInfoTableNumber) throws MDBCServiceException { + String tableName = musicNodeInfoTableName; if(nodeInfoTableNumber >= 0) { tableName = tableName + "-" + @@ -2478,13 +2354,13 @@ public class MusicMixin implements MusicInterface { String cql = String.format( "CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", - this.music_ns, + musicNamespace, tableName, fields, priKey); try { - executeMusicWriteQuery(this.music_ns,tableName,cql); + executeMusicWriteQuery(musicNamespace,tableName,cql); } catch (MDBCServiceException e) { logger.error("Initialization error: Failure to create node information table"); throw(e); @@ -2514,4 +2390,19 @@ public class MusicMixin implements MusicInterface { } + @Override + public void deleteMriRow(MusicRangeInformationRow row) throws MDBCServiceException{ + String cql = String.format("DELETE FROM %s.%s WHERE rangeid = ?;", music_ns, musicRangeInformationTableName); + PreparedQueryObject pQueryObject = new PreparedQueryObject(); + pQueryObject.appendQueryString(cql); + pQueryObject.addValue(row.getPartitionIndex()); + ReturnType rt ; + try { + rt = MusicCore.atomicPut(music_ns, musicRangeDependencyTableName, row.getPartitionIndex().toString(), + pQueryObject, null); + } catch (MusicLockingException|MusicQueryException|MusicServiceException e) { + logger.error("Failure when deleting mri row"); + new MDBCServiceException("Error deleting mri row",e); + } + } } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java index 0afacea..af6b859 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java @@ -74,13 +74,15 @@ public class MySQLMixin implements DBInterface { public static final String TRANS_TBL = "MDBC_TRANSLOG"; private static final String CREATE_TBL_SQL = "CREATE TABLE IF NOT EXISTS "+TRANS_TBL+ - " (IX INT AUTO_INCREMENT, OP CHAR(1), TABLENAME VARCHAR(255),KEYDATA VARCHAR(1024), ROWDATA VARCHAR(1024), CONNECTION_ID INT,PRIMARY KEY (IX))"; + " (IX INT AUTO_INCREMENT, OP CHAR(1), TABLENAME VARCHAR(255),KEYDATA VARCHAR(1024), ROWDATA VARCHAR(1024), " + + "CONNECTION_ID INT, PRIMARY KEY (IX));"; private final MusicInterface mi; private final int connId; private final String dbName; private final Connection jdbcConn; private final Map<String, TableInfo> tables; + private PreparedStatement deleteStagingStatement; private boolean server_tbl_created = false; public MySQLMixin() { @@ -89,14 +91,20 @@ public class MySQLMixin implements DBInterface { this.dbName = null; this.jdbcConn = null; this.tables = null; + this.deleteStagingStatement = null; } - public MySQLMixin(MusicInterface mi, String url, Connection conn, Properties info) { + public MySQLMixin(MusicInterface mi, String url, Connection conn, Properties info) throws SQLException { this.mi = mi; this.connId = generateConnID(conn); this.dbName = getDBName(conn); this.jdbcConn = conn; this.tables = new HashMap<String, TableInfo>(); } + + private PreparedStatement getStagingDeletePreparedStatement() throws SQLException { + return jdbcConn.prepareStatement("DELETE FROM "+TRANS_TBL+" WHERE (IX BETWEEN ? AND ? ) AND " + + "CONNECTION_ID = ?;"); + } // This is used to generate a unique connId for this connection to the DB. private int generateConnID(Connection conn) { int rv = (int) System.currentTimeMillis(); // random-ish @@ -297,6 +305,7 @@ mysql> describe tables; Statement stmt = jdbcConn.createStatement(); stmt.execute(CREATE_TBL_SQL); stmt.close(); + this.deleteStagingStatement = getStagingDeletePreparedStatement(); logger.info(EELFLoggerDelegate.applicationLogger,"createSQLTriggers: Server side dirty table created."); server_tbl_created = true; } catch (SQLException e) { @@ -594,11 +603,15 @@ NEW.field refers to the new value // copy from DB.MDBC_TRANSLOG where connid == myconnid // then delete from MDBC_TRANSLOG String sql2 = "SELECT IX, TABLENAME, OP, ROWDATA,KEYDATA FROM "+TRANS_TBL +" WHERE CONNECTION_ID = " + this.connId; + Integer biggestIx = Integer.MIN_VALUE; + Integer smallestIx = Integer.MAX_VALUE; try { ResultSet rs = executeSQLRead(sql2); Set<Integer> rows = new TreeSet<Integer>(); while (rs.next()) { int ix = rs.getInt("IX"); + biggestIx = Integer.max(biggestIx,ix); + smallestIx = Integer.min(smallestIx,ix); String op = rs.getString("OP"); OperationType opType = toOpEnum(op); String tbl = rs.getString("TABLENAME"); @@ -609,17 +622,13 @@ NEW.field refers to the new value rows.add(ix); } rs.getStatement().close(); + // batch delete operations if (rows.size() > 0) { - //TODO: DO batch deletion - sql2 = "DELETE FROM "+TRANS_TBL+" WHERE IX = ?"; - PreparedStatement ps = jdbcConn.prepareStatement(sql2); - logger.debug("Executing: "+sql2); - logger.debug(" For ix = "+rows); - for (int ix : rows) { - ps.setInt(1, ix); - ps.execute(); - } - ps.close(); + this.deleteStagingStatement.setInt(1,smallestIx); + this.deleteStagingStatement.setInt(2,biggestIx); + this.deleteStagingStatement.setInt(3,this.connId); + logger.debug("Staging delete: Executing with vals ["+smallestIx+","+biggestIx+","+this.connId+"]"); + this.deleteStagingStatement.execute(); } } catch (SQLException e) { logger.warn("Exception in postStatementHook: "+e); @@ -1030,4 +1039,9 @@ NEW.field refers to the new value String sql = "DELETE FROM " + TRANS_TBL + " WHERE CONNECTION_ID = " + this.connId; jdbcStmt.executeQuery(sql); } + + @Override + public Connection getSQLConnection(){ + return jdbcConn; + } } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java b/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java index 057b550..e76e1b1 100644 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java @@ -214,12 +214,7 @@ public class OwnershipAndCheckpoint{ checkpointLock.unlock(); break; } else { - final StagingTable txDigest = mi.getTxDigest(pair.getKey()); - applyTxDigest(rangesToWarmup,di, txDigest); - for (Range r : pair.getValue()) { - MusicRangeInformationRow row = node.getRow(); - alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index)); - } + applyDigestAndUpdateDataStructures(mi, di, rangesToWarmup, node, pair); } pair = node.nextNotAppliedTransaction(rangeSet); enableForeignKeys(di); @@ -233,6 +228,24 @@ public class OwnershipAndCheckpoint{ } } + private void applyDigestAndUpdateDataStructures(MusicInterface mi, DBInterface di, List<Range> ranges, DagNode node, + Pair<MusicTxDigestId, List<Range>> pair) throws MDBCServiceException { + final StagingTable txDigest; + try { + txDigest = mi.getTxDigest(pair.getKey()); + } catch (MDBCServiceException e) { + logger.warn("Transaction digest was not found, this could be caused by a failure of the previous owner" + +"And would normally only happen as the last ID of the corresponding redo log. Please check that this is the" + +" case for txID "+pair.getKey().transactionId.toString()); + return; + } + applyTxDigest(ranges,di, txDigest); + for (Range r : pair.getValue()) { + MusicRangeInformationRow row = node.getRow(); + alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index)); + } + } + private void applyRequiredChanges(MusicInterface mi, DBInterface db, Dag extendedDag, List<Range> ranges, UUID ownOpId) throws MDBCServiceException { Set<Range> rangeSet = new HashSet<Range>(ranges); @@ -242,12 +255,7 @@ public class OwnershipAndCheckpoint{ if(node!=null) { Pair<MusicTxDigestId, List<Range>> pair = node.nextNotAppliedTransaction(rangeSet); while (pair != null) { - final StagingTable txDigest = mi.getTxDigest(pair.getKey()); - applyTxDigest(ranges, db, txDigest); - for (Range r : pair.getValue()) { - MusicRangeInformationRow row = node.getRow(); - alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index)); - } + applyDigestAndUpdateDataStructures(mi, db, ranges, node, pair); pair = node.nextNotAppliedTransaction(rangeSet); if (timeout(ownOpId)) { enableForeignKeys(db); @@ -393,6 +401,7 @@ public class OwnershipAndCheckpoint{ return rowsPerLatestRange; } + public Map<Range, Pair<MriReference, Integer>> getAlreadyApplied() { return this.alreadyApplied; } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigest.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigest.java deleted file mode 100644 index 5b3872a..0000000 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigest.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * ============LICENSE_START==================================================== - * org.onap.music.mdbc - * ============================================================================= - * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. - * ============================================================================= - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END====================================================== - */ -package org.onap.music.mdbc.tables; - -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.*; -import java.util.concurrent.TimeUnit; -import org.onap.music.exceptions.MDBCServiceException; -import org.onap.music.logging.EELFLoggerDelegate; -import org.onap.music.mdbc.DatabasePartition; -import org.onap.music.mdbc.MdbcConnection; -import org.onap.music.mdbc.Range; -import org.onap.music.mdbc.StateManager; -import org.onap.music.mdbc.mixins.DBInterface; -import org.onap.music.mdbc.mixins.MusicInterface; - -public class MusicTxDigest { - private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(MusicTxDigest.class); - - //private MdbcServerLogic mdbcServer; - //private NodeConfiguration config; - private StateManager stateManager; - - public MusicTxDigest(StateManager stateManager) { - this.stateManager = stateManager; - } - - /** - * Runs the body of the background daemon - * @param daemonSleepTimeS time, in seconds, between updates - * @throws InterruptedException - */ - public void backgroundDaemon(int daemonSleepTimeS) throws InterruptedException { - MusicInterface mi = stateManager.getMusicInterface(); - DBInterface dbi = ((MdbcConnection) stateManager.getConnection("daemon")).getDBInterface(); - - while (true) { - Thread.sleep(TimeUnit.SECONDS.toMillis(daemonSleepTimeS)); - //update - logger.info(String.format("[%s] Background MusicTxDigest daemon updating local db", - new Timestamp(System.currentTimeMillis()))); - - //1) get all other partitions from musicrangeinformation - List<UUID> partitions = null; - try { - partitions = mi.getPartitionIndexes(); - } catch (MDBCServiceException e) { - logger.error("Error obtainting partition indexes, trying again next iteration"); - continue; - } - //2) for each partition I don't own - final Set<Range> warmupRanges = stateManager.getRangesToWarmup(); - final List<DatabasePartition> currentPartitions = stateManager.getPartitions(); - if (currentPartitions.size() != 0) { - for (DatabasePartition part : currentPartitions) { - List<Range> partitionRanges = part.getSnapshot(); - warmupRanges.removeAll(partitionRanges); - } - try { - stateManager.getOwnAndCheck().warmup(mi, dbi, new ArrayList<>(warmupRanges)); - } catch (MDBCServiceException e) { - logger.error("Unable to update for partition : " + warmupRanges + ". " + e.getMessage()); - continue; - } - } - - //Step 3: ReplayDigest() for E.C conditions - try { - replayDigest(mi,dbi, stateManager.getEventualRanges()); - } catch (MDBCServiceException e) { - logger.error("Unable to perform Eventual Consistency operations" + e.getMessage()); - continue; - } - - } - } - - /** - * Replay the digest for eventual consistency. - * @param mi music interface - * @param dbi interface to the database that will replay the operations - * @param ranges only these ranges will be applied from the digests - * @throws MDBCServiceException - */ - public void replayDigest(MusicInterface mi, DBInterface dbi, List<Range> ranges) throws MDBCServiceException { - StagingTable transaction; - String nodeName = stateManager.getMdbcServerName(); - - LinkedHashMap<UUID,StagingTable> ecDigestInformation = mi.getEveTxDigest(nodeName); - Set<UUID> keys = ecDigestInformation.keySet(); - for(UUID txTimeID:keys){ - transaction = ecDigestInformation.get(txTimeID); - try { - dbi.replayTransaction(transaction, ranges); // I think this Might change if the data is coming from a new table.. ( what is the new table structure??) - } catch (SQLException e) { - logger.error("EC:Rolling back the entire digest replay."); - return; - } - logger.info("EC: Successfully replayed transaction "); - - try { - mi.updateNodeInfoTableWithTxTimeIDKey(txTimeID, nodeName); - } catch (MDBCServiceException e) { - logger.error("EC:Rolling back the entire digest replay."); - } - } -; - } - - - /** - * Replay the digest for a given partition - * @param mi music interface - * @param partitionId the partition to be replayed - * @param dbi interface to the database that will replay the operations - * @throws MDBCServiceException - */ - public static void replayDigestForPartition(MusicInterface mi, UUID partitionId, DBInterface dbi) throws MDBCServiceException { - final MusicRangeInformationRow row = mi.getMusicRangeInformation(partitionId); - List<MusicTxDigestId> partitionsRedoLogTxIds = row.getRedoLog(); - for (MusicTxDigestId txId: partitionsRedoLogTxIds) { - StagingTable transaction = mi.getTxDigest(txId); - try { - //\TODO do this two operations in parallel - dbi.replayTransaction(transaction, row.getDBPartition().getSnapshot()); - mi.replayTransaction(transaction); - } catch (SQLException e) { - logger.error("Rolling back the entire digest replay. " + partitionId); - return; - } - logger.info("Successfully replayed transaction " + txId); - } - //todo, keep track of where I am in pointer - } - - /** - * Start the background daemon defined by this object - * Spawns a new thread and runs "backgroundDaemon" - * @param daemonSleepTimeS time, in seconds, between updates run by daemon - */ - public void startBackgroundDaemon(int daemonSleepTimeS) { - class MusicTxBackgroundDaemon implements Runnable { - public void run() { - while (true) { - try { - logger.info("MusicTxDigest background daemon started"); - backgroundDaemon(daemonSleepTimeS); - } catch (InterruptedException e) { - logger.error("MusicTxDigest background daemon stopped " + e.getMessage()); - } - } - } - } - Thread t = new Thread(new MusicTxBackgroundDaemon()); - t.start(); - - } -} diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestDaemon.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestDaemon.java new file mode 100644 index 0000000..4f3a3bf --- /dev/null +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestDaemon.java @@ -0,0 +1,144 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ +package org.onap.music.mdbc.tables; + +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.*; +import java.util.concurrent.TimeUnit; +import org.onap.music.exceptions.MDBCServiceException; +import org.onap.music.logging.EELFLoggerDelegate; +import org.onap.music.mdbc.DatabasePartition; +import org.onap.music.mdbc.MdbcConnection; +import org.onap.music.mdbc.Range; +import org.onap.music.mdbc.StateManager; +import org.onap.music.mdbc.mixins.DBInterface; +import org.onap.music.mdbc.mixins.MusicInterface; + +public class MusicTxDigestDaemon implements Runnable { + + private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(MusicTxDigestDaemon.class); + + private StateManager stateManager; + private int daemonSleepTimeS; + + public MusicTxDigestDaemon(int daemonSleepTimeS, StateManager stateManager) { + this.stateManager = stateManager; + this.daemonSleepTimeS = daemonSleepTimeS; + } + + /** + * Replay the digest for eventual consistency. + * + * @param mi music interface + * @param dbi interface to the database that will replay the operations + * @param ranges only these ranges will be applied from the digests + */ + public void replayDigest(MusicInterface mi, DBInterface dbi, List<Range> ranges) throws MDBCServiceException { + StagingTable transaction; + String nodeName = stateManager.getMdbcServerName(); + + LinkedHashMap<UUID, StagingTable> ecDigestInformation = mi.getEveTxDigest(nodeName); + Set<UUID> keys = ecDigestInformation.keySet(); + for (UUID txTimeID : keys) { + transaction = ecDigestInformation.get(txTimeID); + try { + dbi.replayTransaction(transaction, + ranges); // I think this Might change if the data is coming from a new table.. ( what is the new table structure??) + } catch (SQLException e) { + logger.error("EC:Rolling back the entire digest replay."); + return; + } + logger.info("EC: Successfully replayed transaction "); + + try { + mi.updateNodeInfoTableWithTxTimeIDKey(txTimeID, nodeName); + } catch (MDBCServiceException e) { + logger.error("EC:Rolling back the entire digest replay."); + } + } + } + + @Override + public void run() { + logger.info("MusicTxDigest background daemon started"); + if (stateManager == null) { + logger.error("State manager is null in background daemon"); + return; + } + MusicInterface mi = stateManager.getMusicInterface(); + + if (mi == null) { + logger.error("Music interface or DB interface is null in background daemon"); + return; + } + while (true) { + try { + MdbcConnection conn = (MdbcConnection) stateManager.getConnection("daemon"); + if (conn == null) { + logger.error("Connection created is null in background daemon"); + return; + } + DBInterface dbi = (conn).getDBInterface(); + //update + logger.info(String.format("[%s] Background MusicTxDigest daemon updating local db", + new Timestamp(System.currentTimeMillis()))); + + //1) get all other partitions from musicrangeinformation + List<UUID> partitions = null; + try { + partitions = mi.getPartitionIndexes(); + } catch (MDBCServiceException e) { + logger.error("Error obtainting partition indexes, trying again next iteration"); + continue; + } + //2) for each partition I don't own + final Set<Range> warmupRanges = stateManager.getRangesToWarmup(); + final List<DatabasePartition> currentPartitions = stateManager.getPartitions(); + List<Range> missingRanges = new ArrayList<>(); + if (currentPartitions.size() != 0) { + for (DatabasePartition part : currentPartitions) { + List<Range> partitionRanges = part.getSnapshot(); + warmupRanges.removeAll(partitionRanges); + } + try { + stateManager.getOwnAndCheck().warmup(mi, dbi, new ArrayList<>(warmupRanges)); + } catch (MDBCServiceException e) { + logger.error("Unable to update for partition : " + warmupRanges + ". " + e.getMessage()); + continue; + } + } + + //Step 3: ReplayDigest() for E.C conditions + try { + replayDigest(mi, dbi, stateManager.getEventualRanges()); + } catch (MDBCServiceException e) { + logger.error("Unable to perform Eventual Consistency operations" + e.getMessage()); + continue; + } + conn.close(); + Thread.sleep(TimeUnit.SECONDS.toMillis(this.daemonSleepTimeS)); + } catch (InterruptedException | SQLException e) { + logger.error("MusicTxDigest background daemon stopped " + e.getMessage(), e); + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestId.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestId.java index 1c37db0..8fa49a9 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestId.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestId.java @@ -22,16 +22,16 @@ package org.onap.music.mdbc.tables; import java.util.UUID; public final class MusicTxDigestId { - public final UUID txId; + public final UUID transactionId; public final int index; - public MusicTxDigestId(UUID primaryKey, int index) { - this.txId= primaryKey; + public MusicTxDigestId(UUID digestId, int index) { + this.transactionId= digestId; this.index=index; } public boolean isEmpty() { - return (this.txId==null); + return (this.transactionId==null); } @Override @@ -40,11 +40,11 @@ public final class MusicTxDigestId { if(o == null) return false; if(!(o instanceof MusicTxDigestId)) return false; MusicTxDigestId other = (MusicTxDigestId) o; - return other.txId.equals(this.txId); + return other.transactionId.equals(this.transactionId); } @Override public int hashCode(){ - return txId.hashCode(); + return transactionId.hashCode(); } } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java index a9ab25f..eda6191 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java @@ -43,6 +43,18 @@ public final class Operation implements Serializable{ KEY = key; } + @Override + protected Object clone() throws CloneNotSupportedException { + Operation clone = null; + try { + clone = (Operation) super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + return clone; + } + public String getTable(){ return TABLE; } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java index 03c7259..dbed9e4 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java @@ -21,11 +21,14 @@ package org.onap.music.mdbc.tables; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; +import javax.validation.constraints.Null; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.logging.EELFLoggerDelegate; import org.onap.music.mdbc.Range; @@ -36,7 +39,7 @@ import org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType; public class StagingTable { - private transient static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(StagingTable.class); + private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(StagingTable.class); private ArrayList<Operation> operations; boolean builderInitialized; Builder digestBuilder; @@ -47,6 +50,40 @@ public class StagingTable { this(new HashSet<>()); logger.debug("Creating staging table with no parameters, most likely this is wrong, unless you are testing"); } + + public StagingTable(StagingTable other) throws CloneNotSupportedException { + if(other==null){ + throw new NullPointerException("Invalid constructor parameter passed, it is null"); + } + //TODO this is a highly inefficient deep copy, please don't use in prod + operations=null; + if(other.operations!=null) { + Iterator<Operation> iterator = other.operations.iterator(); + operations = new ArrayList<>(); + while (iterator.hasNext()) { + operations.add((Operation) iterator.next().clone()); + } + } + builderInitialized=other.builderInitialized; + digestBuilder=null; + if(other.digestBuilder!=null) { + CompleteDigest build = other.digestBuilder.build(); + digestBuilder = build.toBuilder(); + } + eventuallyBuilder=null; + if(other.eventuallyBuilder!=null) { + CompleteDigest build2 = other.digestBuilder.build(); + eventuallyBuilder = build2.toBuilder(); + } + eventuallyConsistentRanges=null; + if(other.eventuallyConsistentRanges!=null) { + eventuallyConsistentRanges = new HashSet<>(); + Iterator<Range> rangeIter = other.eventuallyConsistentRanges.iterator(); + while (rangeIter.hasNext()) { + eventuallyConsistentRanges.add(rangeIter.next().clone()); + } + } + } public StagingTable(Set<Range> eventuallyConsistentRanges) { //operations = new ArrayList<Operation>(); @@ -74,6 +111,67 @@ public class StagingTable { } } + public static ByteBuffer Compress(ByteBuffer serializedStaging) throws MDBCServiceException { + if(serializedStaging.hasArray()) { + //\TODO: Use JAVA 11 to simplify this process using ByteBuffer natively + Deflater compressor = new Deflater(); + final byte[] inputArray = serializedStaging.array(); + compressor.setInput(inputArray); + compressor.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(serializedStaging.array().length); + byte[] buf = new byte[1024]; + try { + while (!compressor.finished()) { + int i = compressor.deflate(buf); + bos.write(buf, 0, i); + } + } finally { + compressor.end(); + try { + bos.close(); + } catch (IOException e) { + throw new MDBCServiceException("Error closing ByetArrayOutputStream:",e); + } + } + byte[] output = bos.toByteArray(); + logger.debug("Staging table compressed from: "+inputArray.length+" to "+output.length); + return ByteBuffer.wrap(output); + } + else{ + throw new MDBCServiceException("Byte buffer was not created correctly, it should wrap an array"); + } + } + + public static ByteBuffer Decompress(ByteBuffer compressedStaging) throws MDBCServiceException { + if(compressedStaging.hasArray()) { + //\TODO: Use JAVA 11 to simplify this process using ByteBuffer natively + Inflater decompressor = new Inflater(); + byte[] inputArray = compressedStaging.array(); + decompressor.setInput(inputArray); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(inputArray.length); + byte[] buffer = new byte[1024]; + while (!decompressor.finished()) { + int decompressSize = 0; + try { + decompressSize = decompressor.inflate(buffer); + } catch (DataFormatException e) { + throw new MDBCServiceException("error decompressing input data",e); + } + outputStream.write(buffer, 0, decompressSize); + } + try { + outputStream.close(); + } catch (IOException e) { + throw new MDBCServiceException("Error closing output byte stream",e); + } + byte[] output = outputStream.toByteArray(); + return ByteBuffer.wrap(output); + } + else{ + throw new MDBCServiceException("Byte buffer was not created correctly, it should wrap an array"); + } + } + synchronized public boolean isBuilderInitialized(){ return isBuilderInitialized(); } @@ -108,7 +206,7 @@ public class StagingTable { } logger.warn("Get operation list with this type of initialization is not suggested for the" + "staging table"); - ArrayList newOperations = new ArrayList(); + ArrayList<Operation> newOperations = new ArrayList<>(); for(Row row : digestBuilder.getRowsList()){ final OpType type = row.getType(); OperationType newType = (type==OpType.INSERT)?OperationType.INSERT:(type==OpType.DELETE)? @@ -123,9 +221,10 @@ public class StagingTable { throw new MDBCServiceException("This type of staging table is unmutable, please use the constructor" + "with no parameters"); } - ByteString serialized = digestBuilder.build().toByteString(); + byte[] bytes = digestBuilder.build().toByteArray(); + ByteBuffer serialized = ByteBuffer.wrap(bytes); digestBuilder.clear(); - return serialized.asReadOnlyByteBuffer(); + return serialized; } synchronized public ByteBuffer getSerializedEventuallyStagingAndClean() throws MDBCServiceException { @@ -136,9 +235,10 @@ public class StagingTable { if(eventuallyBuilder == null || eventuallyBuilder.getRowsCount()==0){ return null; } - ByteString serialized = eventuallyBuilder.build().toByteString(); + byte[] bytes = eventuallyBuilder.build().toByteArray(); + ByteBuffer serialized = ByteBuffer.wrap(bytes); eventuallyBuilder.clear(); - return serialized.asReadOnlyByteBuffer(); + return serialized; } synchronized public boolean isEmpty() { diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tools/ClusterSetup.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tools/ClusterSetup.java new file mode 100644 index 0000000..d560915 --- /dev/null +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tools/ClusterSetup.java @@ -0,0 +1,80 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ +package org.onap.music.mdbc.tools; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import java.io.FileNotFoundException; +import org.onap.music.exceptions.MDBCServiceException; +import org.onap.music.logging.EELFLoggerDelegate; +import org.onap.music.main.MusicUtil; +import org.onap.music.mdbc.configurations.ClusterConfiguration; + +public class ClusterSetup { + public static final EELFLoggerDelegate LOG = EELFLoggerDelegate.getLogger(ClusterSetup.class); + + @Parameter(names = { "-c", "--configuration" }, required = true, + description = "This is the input file that is going to have the configuration to setup the cluster") + private String configurationFile; + @Parameter(names = { "-h", "-help", "--help" }, help = true, + description = "Print the help message") + private boolean help = false; + + private ClusterConfiguration inputConfig; + + public ClusterSetup(){} + + + public void readInput(){ + LOG.info("Reading inputs"); + try { + inputConfig = ClusterConfiguration.readJsonFromFile(configurationFile); + } catch (FileNotFoundException e) { + LOG.error("Input file is invalid or not found"); + System.exit(1); + } + } + + public void createAll() throws MDBCServiceException { + inputConfig.initNamespaces(); + inputConfig.initTables(); + } + + public static void main(String[] args) { + LOG.info("Starting cassandra cluster initializer"); + LOG.info("Using music file configuration:"+ MusicUtil.getMusicPropertiesFilePath()); + ClusterSetup configs = new ClusterSetup(); + @SuppressWarnings("deprecation") + JCommander jc = new JCommander(configs, args); + if (configs.help) { + jc.usage(); + System.exit(1); + return; + } + configs.readInput(); + try { + configs.createAll(); + } catch (MDBCServiceException e) { + e.printStackTrace(); + System.exit(1); + } + System.exit(0); + } +} diff --git a/mdbc-server/src/main/resources/music.properties b/mdbc-server/src/main/resources/music.properties index 0b34ff9..1aaf7fd 100755 --- a/mdbc-server/src/main/resources/music.properties +++ b/mdbc-server/src/main/resources/music.properties @@ -5,4 +5,4 @@ cassandra.user =\ cassandra.password =\ cassandra music_namespace =\ - mdbc_namespace
\ No newline at end of file + mdbc_namespace diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/MusicTxDigestTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/MusicTxDigestTest.java index 11ec272..7f1c0e1 100644 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/MusicTxDigestTest.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/MusicTxDigestTest.java @@ -26,7 +26,7 @@ import java.io.IOException; import java.util.HashMap; import org.junit.Test; -import org.onap.music.mdbc.tables.MusicTxDigest; +import org.onap.music.mdbc.tables.MusicTxDigestDaemon; import org.onap.music.mdbc.tables.StagingTable; diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/TestUtils.java b/mdbc-server/src/test/java/org/onap/music/mdbc/TestUtils.java index 291179a..e5a3252 100755 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/TestUtils.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/TestUtils.java @@ -20,11 +20,8 @@ package org.onap.music.mdbc; import com.datastax.driver.core.*; -import com.datastax.driver.core.exceptions.QueryExecutionException; -import com.datastax.driver.core.exceptions.SyntaxError; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.exceptions.MusicLockingException; -import org.onap.music.lockingservice.cassandra.CassaLockStore; import org.onap.music.lockingservice.cassandra.MusicLockState; import org.onap.music.logging.EELFLoggerDelegate; import org.onap.music.main.MusicCore; @@ -38,16 +35,12 @@ import java.util.*; import org.onap.music.mdbc.mixins.MusicInterface; import org.onap.music.mdbc.tables.MusicRangeInformationRow; -import static junit.framework.TestCase.assertNotNull; -import static junit.framework.TestCase.fail; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class TestUtils { private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(TestUtils.class); - public static DatabasePartition createBasicRow(Range range, MusicInterface mixin, String mdbcServerName){ + public static DatabasePartition createBasicRow(Range range, MusicInterface mixin, String mdbcServerName) + throws MDBCServiceException { final UUID uuid = MDBCUtils.generateTimebasedUniqueKey(); List<Range> ranges = new ArrayList<>(); ranges.add(range); @@ -55,21 +48,14 @@ public class TestUtils { MusicRangeInformationRow newRow = new MusicRangeInformationRow(uuid,dbPartition, new ArrayList<>(), "", mdbcServerName, true); DatabasePartition partition=null; - try { - partition = mixin.createMusicRangeInformation(newRow); - } catch (MDBCServiceException e) { - fail("failure when creating new row"); - } + partition = mixin.createMusicRangeInformation(newRow); return partition; } - public static void unlockRow(String keyspace, String mriTableName, DatabasePartition partition){ + public static void unlockRow(String keyspace, String mriTableName, DatabasePartition partition) + throws MusicLockingException { String fullyQualifiedMriKey = keyspace+"."+ mriTableName+"."+partition.getMRIIndex().toString(); - try { - MusicLockState musicLockState = MusicCore.voluntaryReleaseLock(fullyQualifiedMriKey, partition.getLockId()); - } catch (MusicLockingException e) { - fail("failure when releasing lock"); - } + MusicLockState musicLockState = MusicCore.voluntaryReleaseLock(fullyQualifiedMriKey, partition.getLockId()); } public static void createKeyspace(String keyspace, Session session) { @@ -78,15 +64,7 @@ public class TestUtils { " WITH REPLICATION " + "= {'class':'SimpleStrategy', 'replication_factor':1}; "; ResultSet res=null; - try { - res = session.execute(queryOp); - } - catch(QueryExecutionException e){ - fail("Failure executing creation of keyspace with error: " + e.getMessage()); - } catch(SyntaxError e){ - fail("Failure executing creation of keyspace with syntax error: " + e.getMessage()); - } - assertTrue("Keyspace "+keyspace+" is already being used, please change it to avoid loosing data",res.wasApplied()); + res = session.execute(queryOp); } public static void deleteKeyspace(String keyspace, Session session){ @@ -94,7 +72,6 @@ public class TestUtils { keyspace + ";"; ResultSet res = session.execute(queryBuilder); - assertTrue("Keyspace "+keyspace+" doesn't exist and it should",res.wasApplied()); } public static HashSet<String> getMriColNames(){ @@ -109,14 +86,18 @@ public class TestUtils { ); } - public static HashMap<String, DataType> getMriColTypes(Cluster cluster){ + public static HashMap<String, DataType> getMriColTypes(Cluster cluster) throws Exception { HashMap<String, DataType> expectedTypes = new HashMap<>(); expectedTypes.put("rangeid",DataType.uuid()); expectedTypes.put("keys",DataType.set(DataType.text())); ProtocolVersion currentVer = cluster.getConfiguration().getProtocolOptions().getProtocolVersion(); - assertNotNull("Protocol version for cluster is invalid", currentVer); + if(currentVer != null) { + throw new Exception("Protocol version for cluster is invalid"); + } CodecRegistry registry = cluster.getConfiguration().getCodecRegistry(); - assertNotNull("Codec registry for cluster is invalid", registry); + if(registry!= null) { + throw new Exception("Codec registry for cluster is invalid"); + } expectedTypes.put("txredolog",DataType.list(TupleType.of(currentVer,registry,DataType.text(),DataType.uuid()))); expectedTypes.put("ownerid",DataType.text()); expectedTypes.put("metricprocessid",DataType.text()); @@ -131,15 +112,19 @@ public class TestUtils { } public static void checkDataTypeForTable(List<ColumnMetadata> columnsMeta, HashSet<String> expectedColumns, - HashMap<String,DataType> expectedTypes){ + HashMap<String,DataType> expectedTypes) throws Exception { for(ColumnMetadata cMeta : columnsMeta){ String columnName = cMeta.getName(); DataType type = cMeta.getType(); - assertTrue("Invalid column name: "+columnName,expectedColumns.contains(columnName)); - assertTrue("Fix the contents of expectedtypes for column: "+columnName, - expectedTypes.containsKey(columnName)); - assertEquals("Invalid type for column: "+columnName, - expectedTypes.get(columnName),type); + if(!expectedColumns.contains(columnName)){ + throw new Exception("Invalid column name: "); + } + if(!expectedTypes.containsKey(columnName)){ + throw new Exception("Fix the contents of expectedtypes for column: "+columnName); + } + if(expectedTypes.get(columnName)!=type) { + throw new Exception("Invalid type for column: "+columnName); + } } } diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java index e8b7511..aba8cb4 100644 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java @@ -122,8 +122,18 @@ public class MusicMixinTest { // Range range = new Range("TABLE1"); // List<Range> ranges = new ArrayList<>(); // ranges.add(range); -// final DatabasePartition partition = TestUtils.createBasicRow(range, mixin, mdbcServerName); -// TestUtils.unlockRow(keyspace,mriTableName,partition); +// DatabasePartition partition=null; +// try { +// partition = TestUtils.createBasicRow(range, mixin, mdbcServerName); +// } +// catch(Exception e){ +// fail(e.getMessage()); +// } +// try { +// TestUtils.unlockRow(keyspace,mriTableName,partition); +// } catch (MusicLockingException e) { +// fail(e.getMessage()); +// } // // DatabasePartition currentPartition = new DatabasePartition(MDBCUtils.generateTimebasedUniqueKey()); // try { @@ -229,7 +239,6 @@ public class MusicMixinTest { // assertFalse(node3Row.getIsLatest()); // } - @Test public void relinquish() { } diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java index 7db973c..eb01bcd 100644 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java @@ -149,6 +149,7 @@ public class OwnershipAndCheckpointTest { Properties properties = new Properties(); properties.setProperty(MusicMixin.KEY_MY_ID,mdbcServerName); properties.setProperty(MusicMixin.KEY_MUSIC_NAMESPACE,keyspace); + properties.setProperty(MusicMixin.KEY_MUSIC_RFACTOR,"1"); //StateManager stateManager = new StateManager("dbUrl", properties, "serverName", "dbName"); ownAndCheck = new OwnershipAndCheckpoint(); musicMixin =new MusicMixin(stateManager, mdbcServerName,properties); @@ -175,7 +176,12 @@ public class OwnershipAndCheckpointTest { TxCommitProgress progressKeeper = new TxCommitProgress(); progressKeeper.createNewTransactionTracker(id ,this.conn); musicMixin.commitLog(partition, null, stagingTable, id, progressKeeper); - TestUtils.unlockRow(keyspace,mriTableName,partition); + try { + TestUtils.unlockRow(keyspace, mriTableName, partition); + } + catch(Exception e){ + fail(e.getMessage()); + } } private OwnershipReturn cleanAndOwnPartition(List<Range> ranges, UUID ownOpId) throws SQLException { @@ -265,4 +271,4 @@ public class OwnershipAndCheckpointTest { private void cleanAlreadyApplied(OwnershipAndCheckpoint ownAndCheck) { ownAndCheck.getAlreadyApplied().clear(); } -}
\ No newline at end of file +} diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/query/QueryProcessorTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/query/QueryProcessorTest.java index 63147e3..e39cc95 100644 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/query/QueryProcessorTest.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/query/QueryProcessorTest.java @@ -22,7 +22,7 @@ import java.util.HashMap; import java.util.List; import org.apache.calcite.sql.parser.SqlParseException; import org.junit.Test; -import org.onap.music.mdbc.tables.MusicTxDigest; +import org.onap.music.mdbc.tables.MusicTxDigestDaemon; import org.onap.music.mdbc.tables.StagingTable; @@ -99,6 +99,7 @@ <module>mdbc-server</module> <module>mdbc-packages</module> <module>mdbc-benchmark</module> + <module>mdbc-internal-benchmark</module> </modules> <dependencies> @@ -128,11 +129,6 @@ <artifactId>json</artifactId> <version>20160810</version> </dependency> - <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - <version>2.3.0</version> - </dependency> <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 --> <dependency> <groupId>org.apache.commons</groupId> |