diff options
20 files changed, 2992 insertions, 639 deletions
diff --git a/mdbc-benchmark/pom.xml b/mdbc-benchmark/pom.xml index f18bab3..84b7e56 100644 --- a/mdbc-benchmark/pom.xml +++ b/mdbc-benchmark/pom.xml @@ -1,123 +1,205 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - ============LICENSE_START========================================== - org.onap.music - =================================================================== - Copyright (c) 2018 AT&T Intellectual Property - =================================================================== - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Copyright (c) 2014, Oracle America, Inc. +All rights reserved. - http://www.apache.org/licenses/LICENSE-2.0 +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ============LICENSE_END============================================= + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Oracle nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <parent> - <artifactId>mdbc</artifactId> - <groupId>org.onap.music.mdbc</groupId> - <version>0.0.1-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>mdbc</artifactId> + <groupId>org.onap.music.mdbc</groupId> + <version>0.0.1-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>benchmark</artifactId> + <packaging>jar</packaging> + + <name>JMH benchmark sample: Java</name> + + <!-- + This is the demo/sample template build script for building Java benchmarks with JMH. + Edit as needed. + --> + + <dependencies> + <dependency> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + <version>20160810</version> + </dependency> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-core</artifactId> + <version>${jmh.version}</version> + </dependency> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-generator-annprocess</artifactId> + <version>${jmh.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.att.eelf</groupId> + <artifactId>eelf-core</artifactId> + <version>1.0.0</version> + </dependency> + <dependency> + <groupId>org.apache.calcite.avatica</groupId> + <artifactId>avatica-server</artifactId> + <version>1.12.0</version> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.8.5</version> + </dependency> + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <version>1.1.7</version> + </dependency> + <dependency> + <groupId>postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>9.1-901-1.jdbc4</version> + </dependency> + </dependencies> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + + <!-- + JMH version to use with this project. + --> + <jmh.version>1.21</jmh.version> - <artifactId>mdbc-benchmark</artifactId> - <version>0.0.1-SNAPSHOT</version> - <name>mdbc-benchmark</name> - <description>MDBC Benchmark</description> - <packaging>jar</packaging> + <!-- + Java source/target to use for compilation. + --> + <javac.target>1.8</javac.target> - <dependencies> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <version>20160810</version> - </dependency> - <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 --> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <version>3.7</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.12</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.vmlens</groupId> - <artifactId>concurrent-junit</artifactId> - <version>1.0.0</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.att.eelf</groupId> - <artifactId>eelf-core</artifactId> - <version>1.0.0</version> - </dependency> - <dependency> - <groupId>org.apache.calcite.avatica</groupId> - <artifactId>avatica-server</artifactId> - <version>1.12.0</version> - </dependency> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <version>2.8.5</version> - </dependency> - <dependency> - <groupId>org.mariadb.jdbc</groupId> - <artifactId>mariadb-java-client</artifactId> - <version>1.1.7</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-math3</artifactId> - <version>3.0</version> - </dependency> - </dependencies> + <!-- + Name of the benchmark Uber-JAR to generate. + --> + <uberjar.name>benchmarks</uberjar.name> + </properties> - <build> - <plugins> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <version>3.1.0</version> - <configuration> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> - <executions> - <execution> - <id>node-configuration</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - <configuration> - <archive> - <manifest> - <mainClass>org.onap.music.mdbc.Benchmark</mainClass> - </manifest> - </archive> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - <finalName>mdbc-benchmark</finalName> - <appendAssemblyId>false</appendAssemblyId> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.1</version> + <configuration> + <compilerVersion>${javac.target}</compilerVersion> + <source>${javac.target}</source> + <target>${javac.target}</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.2</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <finalName>${uberjar.name}</finalName> + <transformers> + <transformer + implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.openjdk.jmh.Main</mainClass> + </transformer> + </transformers> + <filters> + <filter> + <!-- + Shading signed JARs will fail without this. + http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar + --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + <pluginManagement> + <plugins> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <version>2.5</version> + </plugin> + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8.1</version> + </plugin> + <plugin> + <artifactId>maven-install-plugin</artifactId> + <version>2.5.1</version> + </plugin> + <plugin> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + </plugin> + <plugin> + <artifactId>maven-javadoc-plugin</artifactId> + <version>2.9.1</version> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + </plugin> + <plugin> + <artifactId>maven-site-plugin</artifactId> + <version>3.3</version> + </plugin> + <plugin> + <artifactId>maven-source-plugin</artifactId> + <version>2.2.1</version> + </plugin> + <plugin> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.17</version> + </plugin> + </plugins> + </pluginManagement> + </build> </project> diff --git a/mdbc-benchmark/src/main/java/org/onap/music/mdbc/Benchmark.java b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricBenchmark.java index a87f8da..1fb584c 100644 --- a/mdbc-benchmark/src/main/java/org/onap/music/mdbc/Benchmark.java +++ b/mdbc-benchmark/src/main/java/org/onap/music/mdbc/MetricBenchmark.java @@ -17,45 +17,130 @@ * limitations under the License. * ============LICENSE_END====================================================== */ - package org.onap.music.mdbc; -import java.sql.*; -import java.util.*; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import org.apache.commons.math3.stat.descriptive.rank.Percentile; - -public class Benchmark -{ +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode({Mode.AverageTime, Mode.SampleTime}) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class MetricBenchmark { + public StringBuilder updateBuilder = new StringBuilder() + .append("UPDATE PERSONS ") + .append("SET Counter = Counter + 1,") + .append("City = 'Sandy Springs'") + .append(";"); + public String update = updateBuilder.toString(); + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(MetricBenchmark.class.getSimpleName()) + .param("type", ExecutionType.METRIC.name()) + .build(); + new Runner(opt).run(); + } - @Parameter(names = { "-m", "--ismariadb" }, - description = "Is MARIADB evaluation") - private boolean isMariaDb = false; + @Benchmark + public boolean testMethod(MyState state) { + Statement stmt = null; + try { + stmt = state.testConnection.createStatement(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + Boolean execute = null; + try { + execute = stmt.execute(update); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + //TODO: check if state need to be consumed by blackhole to guarantee execution + state.testConnection.commit(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + try { + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + System.exit(1); + } + return execute; + } - @Parameter(names = { "-h", "-help", "--help" }, help = true, - description = "Print the help message") - private boolean help = false; + public static enum ExecutionType { + MARIA_DB, COCKROACH_DB, METRIC + } + @State(Scope.Benchmark) public static class MyState { - private Connection createConnection(){ + public final String driver = "org.apache.calcite.avatica.remote.Driver"; + public final String mariaDriver = "org.mariadb.jdbc.Driver"; + public final String cockroachDriver = "org.postgresql.Driver"; + final String user = "root"; + final String password = "metriccluster"; + @Param({"104.209.240.219"}) + public String ip; + @Param({"1", "10", "50", "80", "100", "200", "300", "400"}) + public int rows; + @Param({"MARIA_DB", "COCKROACH_DB", "METRIC"}) + public ExecutionType type; + + public Connection testConnection; + + + private Connection createConnection() { + final String connectionUrl = "jdbc:avatica:remote:url=http://" + ip + ":30000;serialization=protobuf"; + final String mariaConnectionUrl = "jdbc:mariadb://" + ip + ":3306/test"; + final String cockroachUrl = "jdbc:postgresql://" + ip + ":26257/test"; + try { - Class.forName(this.driver); + switch (type) { + case MARIA_DB: + Class.forName(this.mariaDriver); + break; + case METRIC: + Class.forName(this.driver); + break; + case COCKROACH_DB: + Class.forName(this.cockroachDriver); + break; + } } catch (ClassNotFoundException e) { e.printStackTrace(); System.exit(1); } - Connection connection=null; + Connection connection = null; try { - if(!isMariaDb) { + if (type == ExecutionType.METRIC) { connection = DriverManager.getConnection(connectionUrl); - } - else{ + } else { Properties connectionProps = new Properties(); connectionProps.put("user", user); - connectionProps.put("password", password); - connection = DriverManager.getConnection(connectionUrl,connectionProps); + if(type == ExecutionType.COCKROACH_DB){ + connectionProps.setProperty("sslmode", "disable"); + } + else{ + connectionProps.put("password", password); + } + final String url = (type == ExecutionType.MARIA_DB) ? mariaConnectionUrl : cockroachUrl; + connection = DriverManager.getConnection(url, connectionProps); } } catch (SQLException e) { e.printStackTrace(); @@ -70,7 +155,7 @@ public class Benchmark return connection; } - private void createTable(Connection connection){ + private void createTable(Connection connection) { final String sql = "CREATE TABLE IF NOT EXISTS PERSONS (\n" + " PersonID int,\n" + " Counter int,\n" + @@ -88,7 +173,7 @@ public class Benchmark System.exit(1); } - Boolean execute=null; + Boolean execute = null; try { execute = stmt.execute(sql); } catch (SQLException e) { @@ -111,7 +196,7 @@ public class Benchmark } } - private boolean cleanTable(){ + private boolean cleanTable() { String cleanCmd = "DELETE FROM PERSONS;"; Statement stmt = null; try { @@ -121,7 +206,7 @@ public class Benchmark System.exit(1); } - Boolean execute=null; + Boolean execute = null; try { execute = stmt.execute(cleanCmd); } catch (SQLException e) { @@ -143,8 +228,8 @@ public class Benchmark return execute; } - private void addRowsToTable(int totalNumberOfRows){ - for(int i=0; i<totalNumberOfRows; i++) { + private void addRowsToTable(int totalNumberOfRows) { + for (int i = 0; i < totalNumberOfRows; i++) { final StringBuilder insertSQLBuilder = new StringBuilder() .append("INSERT INTO PERSONS VALUES (") .append(i) @@ -187,8 +272,8 @@ public class Benchmark } } + @Setup(Level.Iteration) public void doSetup() { - System.out.println("Do Global Setup"); Connection connection = createConnection(); createTable(connection); try { @@ -197,18 +282,12 @@ public class Benchmark e.printStackTrace(); System.exit(1); } - } - - public void doWarmup(int rows) { - System.out.println("Do Setup"); //Setup connection testConnection = createConnection(); - //Empty database boolean cleanResult = cleanTable(); - //Add new lines - addRowsToTable(rows); + addRowsToTable(Integer.valueOf(rows)); //Commit try { @@ -217,9 +296,9 @@ public class Benchmark e.printStackTrace(); System.exit(1); } - } + @TearDown(Level.Iteration) public void doTearDown() { System.out.println("Do TearDown"); try { @@ -230,139 +309,6 @@ public class Benchmark } } - boolean isMariaDb = false; - String user = "root"; - String password = "metriccluster"; - public String driver = "org.apache.calcite.avatica.remote.Driver"; - //public final String driver = "org.mariadb.jdbc.Driver"; - - public String connectionUrl = "jdbc:avatica:remote:url=http://localhost:30000;serialization=protobuf"; - //public final String connectionUrl = "jdbc:mariadb://localhost:3306/test"; - - public Connection testConnection; - } - - public static void testMethod(MyState state) { - //UPDATE table_name - //SET column1 = value1, column2 = value2, ... - //WHERE condition; - final StringBuilder updateBuilder = new StringBuilder() - .append("UPDATE PERSONS ") - .append("SET Counter = Counter + 1,") - .append("City = 'Sandy Springs'") - .append(";"); - Statement stmt = null; - try { - stmt = state.testConnection.createStatement(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - - Boolean execute = null; - try { - execute = stmt.execute(updateBuilder.toString()); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - try { - state.testConnection.commit(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - System.exit(1); - } - } - - private static void printResults(Map<Integer,List<Long>> results){ - System.out.println("ROWS,P1,P5,P10,P25,P50,P75,P90,P95,P99,AVG,DEVIATION"); - for(Map.Entry<Integer,List<Long>> result : results.entrySet()) { - Percentile resultsPercentiles = new Percentile(); - double[] tmpList= new double[result.getValue().size()]; - int counter = 0; - for(Long val : result.getValue()) { - tmpList[counter++]= Double.valueOf(val); - } - resultsPercentiles.setData(tmpList); - final double average = result.getValue() - .stream() - .mapToDouble((x) -> x.doubleValue()) - .summaryStatistics() - .getAverage(); - - final double rawSum = result.getValue() - .stream() - .mapToDouble((x) -> Math.pow(x.doubleValue() - average, - 2.0)) - .sum(); - - final double deviation = Math.sqrt(rawSum / (result.getValue().size() - 1)); - System.out.printf("%d,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f\n", - result.getKey(), - resultsPercentiles.evaluate(1.0), - resultsPercentiles.evaluate(5.0), - resultsPercentiles.evaluate(10.0), - resultsPercentiles.evaluate(25.0), - resultsPercentiles.evaluate(50.0), - resultsPercentiles.evaluate(75.0), - resultsPercentiles.evaluate(90.0), - resultsPercentiles.evaluate(95.0), - resultsPercentiles.evaluate(99.0), - average, - deviation - ); - } - } - - public static void main( String[] args ) - { - final Benchmark testApp = new Benchmark(); - @SuppressWarnings("deprecation") - JCommander jc = new JCommander(testApp, args); - if (testApp.help) { - jc.usage(); - System.exit(1); - return; - } - MyState state = new MyState(); - if(testApp.isMariaDb){ - state.isMariaDb = true; - state.driver = "org.mariadb.jdbc.Driver"; - state.connectionUrl = "jdbc:mariadb://192.168.1.30:3306/test"; - } - else{ - state.isMariaDb = false; - state.driver = "org.apache.calcite.avatica.remote.Driver"; - state.connectionUrl = "jdbc:avatica:remote:url=http://localhost:30000;serialization=protobuf"; - - } - //iterations - Map<Integer, List<Long>> results = new HashMap<>(); - final int totalIterations = 20; - final int[] rows = { 1,10,50,80//}; - ,100,200,300,400};//, 500}; - //, 1000}; - for(int row: rows) { - System.out.println("Running for rows: "+Integer.toString(row)); - results.put(row,new ArrayList<Long>()); - for (int i = 0; i < totalIterations; i++) { - System.out.println("Running iteration: "+Integer.toString(i)); - state.doSetup(); - state.doWarmup(row); - long startTime = System.nanoTime(); - testMethod(state); - long endTime = System.nanoTime(); - results.get(row).add(endTime - startTime); - state.doTearDown(); - } - } - printResults(results); } } diff --git a/mdbc-benchmark/src/main/resources/logback.xml b/mdbc-benchmark/src/main/resources/logback.xml new file mode 100755 index 0000000..e4d0030 --- /dev/null +++ b/mdbc-benchmark/src/main/resources/logback.xml @@ -0,0 +1,369 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ============LICENSE_START========================================== + mdbc + =================================================================== + Copyright © 2017 AT&T Intellectual Property. All rights reserved. + =================================================================== + + Unless otherwise specified, all software contained herein is licensed + under the Apache License, Version 2.0 (the “License”); + you may not use this software except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Unless otherwise specified, all documentation contained herein is licensed + under the Creative Commons License, Attribution 4.0 Intl. (the “License”); + you may not use this documentation except in compliance with the License. + You may obtain a copy of the License at + + https://creativecommons.org/licenses/by/4.0/ + + Unless required by applicable law or agreed to in writing, documentation + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ============LICENSE_END============================================ + + + --> + +<configuration scan="true" scanPeriod="3 seconds" debug="true"> + <!-- + Logback files for the mdbc Driver "mdbc" + are created in directory ${catalina.base}/logs/mdbc; + e.g., apache-tomcat-8.0.35/logs/mdbc/application.log + --> + <!--<jmxConfigurator /> --> + + <!-- specify the component name --> + <property name="catalina.home" value="/var/log/metric/"/> + <property name="componentName" value="mdbc"></property> + + <!-- specify the base path of the log directory --> + <property name="logDirPrefix" value="${catalina.base}/logs"></property> + + <!-- The directories where logs are written --> + <property name="logDirectory" value="${logDirPrefix}/${componentName}"/> + <!-- Can easily relocate debug logs by modifying this path. --> + <property name="debugLogDirectory" value="${logDirPrefix}/${componentName}"/> + + <!-- log file names --> + <property name="generalLogName" value="application"/> + <property name="errorLogName" value="error"/> + <property name="metricsLogName" value="metrics"/> + <property name="auditLogName" value="audit"/> + <property name="debugLogName" value="debug"/> + <!-- + These loggers are not used in code (yet). + <property name="securityLogName" value="security" /> + <property name="policyLogName" value="policy" /> + <property name="performanceLogName" value="performance" /> + <property name="serverLogName" value="server" /> + --> + + <!-- 1610 Logging Fields Format Revisions --> + <property name="auditLoggerPattern" + value="%X{AuditLogBeginTimestamp}|%X{AuditLogEndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{ClientIPAddress}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}| %msg%n"/> + + <property name="metricsLoggerPattern" + value="%X{MetricsLogBeginTimestamp}|%X{MetricsLogEndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{ClientIPAddress}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{TargetVisualEntity}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}| %msg%n"/> + + <property name="errorLoggerPattern" + value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{ClassName}|%X{AlertSeverity}|%X{ErrorCode}|%X{ErrorDescription}| %msg%n"/> + + <property name="defaultLoggerPattern" + value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%X{ClassName}| %msg%n"/> + + <!-- use %class so library logging calls yield their class name --> + <property name="applicationLoggerPattern" + value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%class{36}| %msg%n"/> + + <!-- Example evaluator filter applied against console appender --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>${defaultLoggerPattern}</pattern> + </encoder> + </appender> + + <!-- ============================================================================ --> + <!-- EELF Appenders --> + <!-- ============================================================================ --> + + <!-- The EELFAppender is used to record events to the general application + log --> + + + <appender name="EELF" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${generalLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${applicationLoggerPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <!-- Class name is part of caller data --> + <includeCallerData>true</includeCallerData> + <appender-ref ref="EELF"/> + </appender> + + <!-- EELF Security Appender. This appender is used to record security events + to the security log file. Security events are separate from other loggers + in EELF so that security log records can be captured and managed in a secure + way separate from the other logs. This appender is set to never discard any + events. --> + <!-- + <appender name="EELFSecurity" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${securityLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${securityLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFSecurity" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <discardingThreshold>0</discardingThreshold> + <appender-ref ref="EELFSecurity" /> + </appender> + --> + + <!-- EELF Performance Appender. This appender is used to record performance + records. --> + <!-- + <appender name="EELFPerformance" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${performanceLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${performanceLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <outputPatternAsHeader>true</outputPatternAsHeader> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFPerformance" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFPerformance" /> + </appender> + --> + + <!-- EELF Server Appender. This appender is used to record Server related + logging events. The Server logger and appender are specializations of the + EELF application root logger and appender. This can be used to segregate Server + events from other components, or it can be eliminated to record these events + as part of the application root log. --> + <!-- + <appender name="EELFServer" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${serverLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${serverLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFServer" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFServer" /> + </appender> + --> + + <!-- EELF Policy Appender. This appender is used to record Policy engine + related logging events. The Policy logger and appender are specializations + of the EELF application root logger and appender. This can be used to segregate + Policy engine events from other components, or it can be eliminated to record + these events as part of the application root log. --> + <!-- + <appender name="EELFPolicy" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${policyLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/${policyLogName}.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>${defaultPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFPolicy" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFPolicy" /> + </appender> + --> + + <!-- EELF Audit Appender. This appender is used to record audit engine + related logging events. The audit logger and appender are specializations + of the EELF application root logger and appender. This can be used to segregate + Policy engine events from other components, or it can be eliminated to record + these events as part of the application root log. --> + + <appender name="EELFAudit" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${auditLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${auditLoggerPattern}</pattern> + </encoder> + </appender> + <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFAudit"/> + </appender> + + <appender name="EELFMetrics" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${metricsLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${metricsLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${metricsLoggerPattern}</pattern> + </encoder> + </appender> + + + <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFMetrics"/> + </appender> + + <appender name="EELFError" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${errorLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${errorLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${errorLoggerPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFError" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFError"/> + </appender> + + <appender name="EELFDebug" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${debugLogDirectory}/${debugLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- daily rollover --> + <fileNamePattern>${logDirectory}/${debugLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern> + + <!-- keep 30 days' worth of history capped at 3GB total size --> + <maxHistory>30</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + + </rollingPolicy> + <encoder> + <pattern>${defaultLoggerPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFDebug" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFDebug"/> + </appender> + + + <logger name="com.att.eelf" level="error" additivity="false"> + <appender-ref ref="asyncEELF"/> + </logger> + + <logger name="com.att.eelf" level="error" additivity="false"> + <appender-ref ref="asyncEELFAudit"/> + </logger> + + <logger name="com.att.eelf" level="error" additivity="false"> + <appender-ref ref="asyncEELFDebug"/> + </logger> + + <logger name="com.att.eelf.error" level="error" additivity="false"> + <appender-ref ref="asyncEELFError"/> + </logger> + + <logger name="com.att.eelf.metrics" level="error" additivity="false"> + <appender-ref ref="asyncEELFMetrics"/> + </logger> + + <root level="ERROR"> + <appender-ref ref="asyncEELF"/> + </root> + +</configuration> diff --git a/mdbc-server/pom.xml b/mdbc-server/pom.xml index 0a2ca72..874fce7 100755 --- a/mdbc-server/pom.xml +++ b/mdbc-server/pom.xml @@ -188,6 +188,12 @@ <artifactId>netty-handler</artifactId> <version>4.1.30.Final</version> </dependency> + <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java --> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + <version>3.6.1</version> + </dependency> </dependencies> <build> diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MDBCUtils.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MDBCUtils.java index 8aca034..ae115bf 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/MDBCUtils.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/MDBCUtils.java @@ -46,40 +46,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.json.JSONObject; public class MDBCUtils { - /** Write the object to a Base64 string. */ - public static String toString( Serializable o ) throws IOException { - //TODO We may want to also compress beside serialize - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try { - ObjectOutputStream oos = new ObjectOutputStream(baos); - oos.writeObject(o); - oos.close(); - return Base64.getEncoder().encodeToString(baos.toByteArray()); - } - finally{ - baos.close(); - } - } - - public static String toString( JSONObject o) throws IOException { - //TODO We may want to also compress beside serialize - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream( baos ); - oos.writeObject( o ); - oos.close(); - return Base64.getEncoder().encodeToString(baos.toByteArray()); - } - - /** Read the object from Base64 string. */ - public static Object fromString( String s ) throws IOException , - ClassNotFoundException { - byte [] data = Base64.getDecoder().decode( s ); - ObjectInputStream ois = new ObjectInputStream( - new ByteArrayInputStream( data ) ); - Object o = ois.readObject(); - ois.close(); - return o; - } public static void saveToFile(String serializedContent, String filename, EELFLoggerDelegate logger) throws IOException { try (PrintWriter fout = new PrintWriter(filename)) { diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java index d336eef..12c7c29 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java @@ -75,7 +75,7 @@ public class MdbcConnection implements Connection { private final MusicInterface mi; private final TxCommitProgress progressKeeper; private final DBInterface dbi; - private final HashMap<Range,StagingTable> transactionDigest; + private final StagingTable transactionDigest; private final Set<String> table_set; private final StateManager statemanager; private DatabasePartition partition; @@ -84,8 +84,7 @@ public class MdbcConnection implements Connection { TxCommitProgress progressKeeper, DatabasePartition partition, StateManager statemanager) throws MDBCServiceException { this.id = id; this.table_set = Collections.synchronizedSet(new HashSet<String>()); - this.transactionDigest = new HashMap<Range,StagingTable>(); - + this.transactionDigest = new StagingTable(new HashSet<>(statemanager.getEventualRanges())); if (c == null) { throw new MDBCServiceException("Connection is null"); } @@ -241,7 +240,11 @@ public class MdbcConnection implements Connection { @Override public void rollback() throws SQLException { logger.debug(EELFLoggerDelegate.applicationLogger, "Rollback");; - transactionDigest.clear(); + try { + transactionDigest.clear(); + } catch (MDBCServiceException e) { + throw new SQLException("Failure to clear the transaction digest",e); + } jdbcConn.rollback(); progressKeeper.reinitializeTxProgress(id); } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java b/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java index bc1dad7..c498952 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java @@ -20,6 +20,7 @@ package org.onap.music.mdbc; import java.io.Serializable; +import java.util.List; import java.util.Objects; @@ -72,6 +73,12 @@ public class Range implements Serializable, Cloneable{ return newRange; } + + public static boolean overlaps(List<Range> ranges, String table){ + //\TODO check if parallel stream makes sense here + return ranges.stream().map((Range r) -> r.table.equals(table)).anyMatch((Boolean b) -> b); + } + public boolean overlaps(Range other) { return table.equals(other.table); } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java index 01d346c..85645f3 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/DBInterface.java @@ -28,7 +28,6 @@ import java.util.Set; import org.onap.music.mdbc.Range; import org.onap.music.mdbc.TableInfo; -import org.onap.music.mdbc.tables.Operation; import org.onap.music.mdbc.tables.StagingTable; /** @@ -97,7 +96,7 @@ public interface DBInterface { * @param sql the SQL statement that was executed * @param transactionDigest */ - void postStatementHook(final String sql,Map<Range,StagingTable> transactionDigest); + void postStatementHook(final String sql,StagingTable transactionDigest); /** * This method executes a read query in the SQL database. Methods that call this method should be sure * to call resultset.getStatement().close() when done in order to free up resources. @@ -117,11 +116,11 @@ public interface DBInterface { * @param digest * @throws SQLException if replay cannot occur correctly */ - void replayTransaction(HashMap<Range,StagingTable> digest) throws SQLException; + void replayTransaction(StagingTable digest, List<Range> ranges) throws SQLException; void disableForeignKeyChecks() throws SQLException; void enableForeignKeyChecks() throws SQLException; - void applyTxDigest(HashMap<Range, StagingTable> txDigest) throws SQLException; + void applyTxDigest(StagingTable txDigest, List<Range> ranges) throws SQLException; } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java index 22c532b..c38efb7 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicInterface.java @@ -20,7 +20,12 @@ package org.onap.music.mdbc.mixins; import com.datastax.driver.core.ResultSet; -import java.util.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import org.json.JSONObject; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.exceptions.MusicLockingException; @@ -182,13 +187,12 @@ public interface MusicInterface { * * @param partition information related to ownership of partitions, used to verify ownership when commiting the Tx * @param eventualRanges - * @param transactionDigest digest of the transaction that is being committed into the Redo log in music. It has to - * be a HashMap, because it is required to be serializable + * @param transactionDigest digest of the transaction that is being committed into the Redo log in music. * @param txId id associated with the log being send * @param progressKeeper data structure that is used to handle to detect failures, and know what to do * @throws MDBCServiceException */ - void commitLog(DatabasePartition partition, List<Range> eventualRanges, HashMap<Range,StagingTable> transactionDigest, String txId,TxCommitProgress progressKeeper) throws MDBCServiceException; + void commitLog(DatabasePartition partition, List<Range> eventualRanges, StagingTable transactionDigest, String txId,TxCommitProgress progressKeeper) throws MDBCServiceException; /** @@ -236,7 +240,7 @@ public interface MusicInterface { * @param transactionDigest digest that contains all the changes performed in the transaction * @throws MDBCServiceException */ - void addTxDigest(MusicTxDigestId newId, String transactionDigest) throws MDBCServiceException; + void addTxDigest(MusicTxDigestId newId, ByteBuffer transactionDigest) throws MDBCServiceException; /** * This functions adds the eventual tx digest to @@ -245,17 +249,9 @@ public interface MusicInterface { * @throws MDBCServiceException */ - void addEventualTxDigest(MusicTxDigestId newId, String transactionDigest) + void addEventualTxDigest(MusicTxDigestId newId, ByteBuffer transactionDigest) throws MDBCServiceException; - /** - * Function used to retrieve a given transaction digest and deserialize it - * @param id of the transaction digest to be retrieved - * @return the deserialize transaction digest that can be applied to the local SQL database - * @throws MDBCServiceException - */ - HashMap<Range,StagingTable> getTxDigest(MusicTxDigestId id) throws MDBCServiceException; - /** * Function used to retrieve a given eventual transaction digest for the current node and deserialize it * @param nodeName that identifies a node @@ -263,7 +259,14 @@ public interface MusicInterface { * @throws MDBCServiceException */ - public LinkedHashMap<UUID, HashMap<Range,StagingTable>> getEveTxDigest(String nodeName) throws MDBCServiceException; + public LinkedHashMap<UUID, StagingTable> getEveTxDigest(String nodeName) throws MDBCServiceException; + /** + * Function used to retrieve a given transaction digest and deserialize it + * @param id of the transaction digest to be retrieved + * @return the deserialize transaction digest that can be applied to the local SQL database + * @throws MDBCServiceException + */ + StagingTable getTxDigest(MusicTxDigestId id) throws MDBCServiceException; /** * Use this functions to verify ownership, and own new ranges @@ -313,7 +316,7 @@ public interface MusicInterface { * @param digest this contain all the changes that were perfomed in this digest * @throws MDBCServiceException */ - void replayTransaction(HashMap<Range,StagingTable> digest) throws MDBCServiceException; + void replayTransaction(StagingTable digest) throws MDBCServiceException; /** * This function is in charge of deleting old mri rows that are not longer contain diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java index 0210cd1..cdf0140 100644 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java @@ -42,6 +42,8 @@ import java.util.function.BiFunction; import org.apache.commons.lang3.tuple.Pair; import org.json.JSONObject; import org.onap.music.datastore.Condition; +import org.onap.music.datastore.MusicDataStore; +import org.onap.music.datastore.MusicDataStoreHandle; import org.onap.music.datastore.PreparedQueryObject; import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.exceptions.MusicLockingException; @@ -1302,9 +1304,24 @@ public class MusicMixin implements MusicInterface { if(lockId==null) { throw new MDBCServiceException("lock reference is null"); } - ReturnType lockReturn = acquireLock(fullyQualifiedKey,lockId); + ReturnType lockReturn; + int counter=0; + do { + if(counter > 0){ + //TODO: Improve backoff + try { + Thread.sleep(50); + } catch (InterruptedException e) { + logger.warn("Error sleeping for acquiring the lock"); + } + logger.warn("Error acquiring lock id: ["+lockId+"] for key: ["+fullyQualifiedKey+"]"); + } + lockReturn = acquireLock(fullyQualifiedKey,lockId); + }while((lockReturn == null||lockReturn.getResult().compareTo(ResultType.SUCCESS) != 0 )&&(counter++<3)); + //\TODO this is wrong, we should have a better way to obtain a lock forcefully, clean the queue and obtain the lock if(lockReturn.getResult().compareTo(ResultType.SUCCESS) != 0 ) { + logger.error("Lock acquire returned invalid error: "+lockReturn.getResult().name()); return null; } partition.setLockId(lockId); @@ -1336,7 +1353,7 @@ public class MusicMixin implements MusicInterface { * This officially commits the transaction globally */ @Override - public void commitLog(DatabasePartition partition,List<Range> eventualRanges, HashMap<Range,StagingTable> transactionDigest, + public void commitLog(DatabasePartition partition,List<Range> eventualRanges, StagingTable transactionDigest, String txId ,TxCommitProgress progressKeeper) throws MDBCServiceException{ // first deal with commit for eventually consistent tables filterAndAddEventualTxDigest(eventualRanges, transactionDigest, txId, progressKeeper); @@ -1377,13 +1394,9 @@ public class MusicMixin implements MusicInterface { return; } - String serializedTransactionDigest; + ByteBuffer serializedTransactionDigest; if(!transactionDigest.isEmpty()) { - try { - serializedTransactionDigest = MDBCUtils.toString(transactionDigest); - } catch (IOException e) { - throw new MDBCServiceException("Failed to serialized transaction digest with error " + e.toString(), e); - } + serializedTransactionDigest = transactionDigest.getSerializedStagingAndClean(); MusicTxDigestId digestId = new MusicTxDigestId(mriIndex, -1); addTxDigest(digestId, serializedTransactionDigest); //2. Save RRT index to RQ @@ -1409,47 +1422,30 @@ public class MusicMixin implements MusicInterface { } } + public void cleanAlreadyApplied(){ + logger.warn("Use this only in test environments"); + alreadyApplied.clear(); + } + private void filterAndAddEventualTxDigest(List<Range> eventualRanges, - HashMap<Range, StagingTable> transactionDigest, String txId, + StagingTable transactionDigest, String txId, TxCommitProgress progressKeeper) throws MDBCServiceException { if(eventualRanges == null || eventualRanges.isEmpty()) { return; } - - HashMap<Range,StagingTable> eventualTransactionDigest = new HashMap<Range,StagingTable>(); - - for(Range eventualRange: eventualRanges) { - transactionDigest.computeIfPresent(eventualRange, new BiFunction<Range,StagingTable,StagingTable>() { - - @Override - public StagingTable apply(Range key, StagingTable value) { - eventualTransactionDigest.put(key, value); - //transactionDigest.remove(key); - return null; - } - - }); + + if(!transactionDigest.areEventualContained(eventualRanges)){ + throw new MDBCServiceException(); } UUID commitId = getCommitId(txId, progressKeeper); - - //1. Push new row to RRT - - String serializedTransactionDigest; - if(eventualTransactionDigest != null && !eventualTransactionDigest.isEmpty()) { - - try { - serializedTransactionDigest = MDBCUtils.toString(eventualTransactionDigest); - } catch (IOException e) { - throw new MDBCServiceException("Failed to serialized transaction digest with error "+e.toString(), e); - } + + ByteBuffer serialized = transactionDigest.getSerializedEventuallyStagingAndClean(); + + if(serialized != null ) { MusicTxDigestId digestId = new MusicTxDigestId(commitId,-1); - addEventualTxDigest(digestId, serializedTransactionDigest); - - if(progressKeeper!= null) { - progressKeeper.setRecordId(txId,digestId); - } + addEventualTxDigest(digestId, serialized); } @@ -1641,9 +1637,11 @@ public class MusicMixin implements MusicInterface { DatabasePartition newPartition = info.getDBPartition(); String fullyQualifiedMriKey = music_ns+"."+ musicRangeInformationTableName+"."+newPartition.getMRIIndex().toString(); - String lockId = createAndAssignLock(fullyQualifiedMriKey,newPartition); + String lockId = createAndAssignLock(fullyQualifiedMriKey, newPartition); + //TODO: fix this retry logic if(lockId == null || lockId.isEmpty()){ - throw new MDBCServiceException("Error initializing music range information, error creating a lock for a new row") ; + throw new MDBCServiceException("Error initializing music range information, error creating a lock for a new row" + + "for key "+fullyQualifiedMriKey) ; } logger.info("Creating MRI " + newPartition.getMRIIndex() + " for ranges " + newPartition.getSnapshot()); createEmptyMriRow(this.music_ns,this.musicRangeInformationTableName,newPartition.getMRIIndex(),info.getMetricProcessId(), @@ -1799,7 +1797,7 @@ public class MusicMixin implements MusicInterface { String priKey = "txTimeId"; StringBuilder fields = new StringBuilder(); fields.append("txid uuid, "); - fields.append("transactiondigest text, "); + fields.append("transactiondigest blob, "); fields.append("txTimeId TIMEUUID ");//notice lack of ',' String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName, fields, priKey); try { @@ -1827,7 +1825,7 @@ public class MusicMixin implements MusicInterface { String priKey = "txid"; StringBuilder fields = new StringBuilder(); fields.append("txid uuid, "); - fields.append("transactiondigest text ");//notice lack of ',' + fields.append("transactiondigest blob ");//notice lack of ',' String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName, fields, priKey); try { executeMusicWriteQuery(this.music_ns,tableName,cql); @@ -1857,19 +1855,14 @@ public class MusicMixin implements MusicInterface { * Writes the transaction history to the txDigest */ @Override - public void addTxDigest(MusicTxDigestId newId, String transactionDigest) throws MDBCServiceException { - //createTxDigestRow(music_ns,musicTxDigestTable,newId,transactionDigest); + public void addTxDigest(MusicTxDigestId newId, ByteBuffer transactionDigest) throws MDBCServiceException { + //\TODO: Save Prepared query to history PreparedQueryObject query = new PreparedQueryObject(); - String cqlQuery = "INSERT INTO " + - this.music_ns + - '.' + - this.musicTxDigestTableName + - " (txid,transactiondigest) " + - "VALUES (" + - newId.txId + ",'" + - transactionDigest + - "');"; - query.appendQueryString(cqlQuery); + String cql = String.format("INSERT INTO %s.%s (txid,transactiondigest) VALUES (?,?);",this.music_ns, + this.musicTxDigestTableName); + query.appendQueryString(cql); + query.addValue(newId.txId); + query.addValue(transactionDigest); //\TODO check if I am not shooting on my own foot try { MusicCore.nonKeyRelatedPut(query,"critical"); @@ -1883,7 +1876,7 @@ public class MusicMixin implements MusicInterface { * Writes the Eventual transaction history to the evetxDigest */ @Override - public void addEventualTxDigest(MusicTxDigestId newId, String transactionDigest) throws MDBCServiceException { + public void addEventualTxDigest(MusicTxDigestId newId, ByteBuffer transactionDigest) throws MDBCServiceException { //createTxDigestRow(music_ns,musicTxDigestTable,newId,transactionDigest); PreparedQueryObject query = new PreparedQueryObject(); String cqlQuery = "INSERT INTO " + @@ -1909,7 +1902,7 @@ public class MusicMixin implements MusicInterface { } @Override - public HashMap<Range,StagingTable> getTxDigest(MusicTxDigestId id) throws MDBCServiceException { + public StagingTable getTxDigest(MusicTxDigestId id) throws MDBCServiceException { String cql = String.format("SELECT * FROM %s.%s WHERE txid = ?;", music_ns, musicTxDigestTableName); PreparedQueryObject pQueryObject = new PreparedQueryObject(); pQueryObject.appendQueryString(cql); @@ -1921,26 +1914,23 @@ public class MusicMixin implements MusicInterface { logger.error("Get operation error: Failure to get row from txdigesttable with id:"+id.txId); throw new MDBCServiceException("Initialization error:Failure to add new row to transaction information", e); } - String digest = newRow.getString("transactiondigest"); - HashMap<Range,StagingTable> changes; + ByteBuffer digest = newRow.getBytes("transactiondigest"); + StagingTable changes; try { - changes = (HashMap<Range, StagingTable>) MDBCUtils.fromString(digest); - } catch (IOException e) { - logger.error("IOException when deserializing digest failed with an invalid class for id:"+id.txId); - throw new MDBCServiceException("Deserializng digest failed with ioexception", e); - } catch (ClassNotFoundException e) { - logger.error("Deserializng digest failed with an invalid class for id:"+id.txId); - throw new MDBCServiceException("Deserializng digest failed with an invalid class", e); + changes = new StagingTable(digest); + } catch (MDBCServiceException e) { + logger.error("Deserializng digest failed with an exception:"+e.getErrorMessage()); + throw e; } return changes; } @Override - public LinkedHashMap<UUID, HashMap<Range,StagingTable>> getEveTxDigest(String nodeName) throws MDBCServiceException { - HashMap<Range,StagingTable> changes; + public LinkedHashMap<UUID, StagingTable> getEveTxDigest(String nodeName) throws MDBCServiceException { + StagingTable changes; String cql; - LinkedHashMap<UUID, HashMap<Range,StagingTable>> ecDigestInformation = new LinkedHashMap<UUID, HashMap<Range,StagingTable>>(); + LinkedHashMap<UUID, StagingTable> ecDigestInformation = new LinkedHashMap<>(); UUID musicevetxdigestNodeinfoTimeID = getTxTimeIdFromNodeInfo(nodeName); PreparedQueryObject pQueryObject = new PreparedQueryObject(); @@ -1949,7 +1939,6 @@ public class MusicMixin implements MusicInterface { cql = String.format("SELECT * FROM %s.%s WHERE txtimeid > ? LIMIT 10 ALLOW FILTERING;", music_ns, this.musicEventualTxDigestTableName); pQueryObject.appendQueryString(cql); pQueryObject.addValue(musicevetxdigestNodeinfoTimeID); - } else { // This is going to Fetch all the Transactiondigest records from the musicevetxdigest table. cql = String.format("SELECT * FROM %s.%s LIMIT 10;", music_ns, this.musicEventualTxDigestTableName); @@ -1960,23 +1949,17 @@ public class MusicMixin implements MusicInterface { ResultSet rs = executeMusicRead(pQueryObject); while (!rs.isExhausted()) { Row row = rs.one(); - String digest = row.getString("transactiondigest"); + ByteBuffer digest = row.getBytes("transactiondigest"); //String txTimeId = row.getString("txtimeid"); //??? UUID txTimeId = row.getUUID("txtimeid"); try { - changes = (HashMap<Range, StagingTable>) MDBCUtils.fromString(digest); - - } catch (IOException e) { - logger.error("IOException when deserializing digest"); - throw new MDBCServiceException("Deserializng digest failed with ioexception", e); - } catch (ClassNotFoundException e) { - logger.error("Deserializng digest failed with an invalid class"); - throw new MDBCServiceException("Deserializng digest failed with an invalid class", e); + changes = new StagingTable(digest); + } catch (MDBCServiceException e) { + logger.error("Deserializng digest failed: "+e.getErrorMessage()); + throw e; } - ecDigestInformation.put(txTimeId, changes); - } return ecDigestInformation; } @@ -2555,7 +2538,7 @@ public class MusicMixin implements MusicInterface { } @Override - public void replayTransaction(HashMap<Range,StagingTable> digest) throws MDBCServiceException{ + public void replayTransaction(StagingTable digest) throws MDBCServiceException{ //\TODO: implement logic to move data from digests to Music Data Tables //throw new NotImplementedException("Error, replay transaction in music mixin needs to be implemented"); return; diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java index 64f4e0c..420f9d4 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MySQLMixin.java @@ -37,6 +37,7 @@ import java.util.TreeSet; import org.json.JSONObject; import org.json.JSONTokener; +import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.logging.EELFLoggerDelegate; import org.onap.music.mdbc.MDBCUtils; import org.onap.music.mdbc.Range; @@ -72,7 +73,7 @@ public class MySQLMixin implements DBInterface { public static final String TRANS_TBL = "MDBC_TRANSLOG"; private static final String CREATE_TBL_SQL = "CREATE TABLE IF NOT EXISTS "+TRANS_TBL+ - " (IX INT AUTO_INCREMENT, OP CHAR(1), TABLENAME VARCHAR(255), NEWROWDATA VARCHAR(1024), KEYDATA VARCHAR(1024), CONNECTION_ID INT,PRIMARY KEY (IX))"; + " (IX INT AUTO_INCREMENT, OP CHAR(1), TABLENAME VARCHAR(255),KEYDATA VARCHAR(1024), ROWDATA VARCHAR(1024), CONNECTION_ID INT,PRIMARY KEY (IX))"; private final MusicInterface mi; private final int connId; @@ -290,6 +291,7 @@ mysql> describe tables; // No SELECT trigger executeSQLWrite(generateTrigger(tableName, "INSERT")); executeSQLWrite(generateTrigger(tableName, "UPDATE")); + //\TODO: save key row instead of the whole row for delete executeSQLWrite(generateTrigger(tableName, "DELETE")); } catch (SQLException e) { if (e.getMessage().equals("Trigger already exists")) { @@ -311,46 +313,47 @@ NEW.field refers to the new value */ private String generateTrigger(String tableName, String op) { boolean isdelete = op.equals("DELETE"); - boolean isinsert = op.equals("INSERT"); + boolean isupdate = op.equals("UPDATE"); TableInfo ti = getTableInfo(tableName); StringBuilder newJson = new StringBuilder("JSON_OBJECT("); // JSON_OBJECT(key, val, key, val) page 1766 - StringBuilder keyJson = new StringBuilder("JSON_OBJECT("); + StringBuilder keyJson = new StringBuilder("JSON_OBJECT("); // JSON_OBJECT(key, val, key, val) page 1766 String pfx = ""; - String keypfx = ""; + String kfx = ""; for (String col : ti.columns) { newJson.append(pfx) .append("'").append(col).append("', ") .append(isdelete ? "OLD." : "NEW.") .append(col); - if (ti.iskey(col) || !ti.hasKey()) { - keyJson.append(keypfx) - .append("'").append(col).append("', ") - .append(isinsert ? "NEW." : "OLD.") - .append(col); - keypfx = ", "; + if (isupdate && (ti.iskey(col) || !ti.hasKey())) { + keyJson.append(kfx) + .append("'").append(col).append("', ") + .append("OLD.") + .append(col); + kfx = ", "; } pfx = ", "; } newJson.append(")"); keyJson.append(")"); //\TODO check if using mysql driver, so instead check the exception + //\TODO add conditional for update, if primary key is still the same, use null in the KEYDATA col StringBuilder sb = new StringBuilder() .append("CREATE TRIGGER ") // IF NOT EXISTS not supported by MySQL! .append(String.format("%s_%s", op.substring(0, 1), tableName)) .append(" AFTER ") .append(op) .append(" ON ") - .append(tableName) + .append(tableName.toUpperCase()) .append(" FOR EACH ROW INSERT INTO ") .append(TRANS_TBL) - .append(" (TABLENAME, OP, NEWROWDATA, KEYDATA, CONNECTION_ID) VALUES('") - .append(tableName) + .append(" (TABLENAME, OP, KEYDATA, ROWDATA, CONNECTION_ID) VALUES('") + .append(tableName.toUpperCase()) .append("', ") .append(isdelete ? "'D'" : (op.equals("INSERT") ? "'I'" : "'U'")) .append(", ") - .append(newJson.toString()) + .append(isupdate ? keyJson.toString() : "NULL") .append(", ") - .append(keyJson.toString()) + .append(newJson.toString()) .append(", ") .append("CONNECTION_ID()") .append(")"); @@ -528,14 +531,14 @@ NEW.field refers to the new value * @param sql the SQL statement that was executed */ @Override - public void postStatementHook(final String sql,Map<Range,StagingTable> transactionDigest) { + public void postStatementHook(final String sql,StagingTable transactionDigest) { if (sql != null) { String[] parts = sql.trim().split(" "); String cmd = parts[0].toLowerCase(); if ("delete".equals(cmd) || "insert".equals(cmd) || "update".equals(cmd)) { try { this.updateStagingTable(transactionDigest); - } catch (NoSuchFieldException e) { + } catch (NoSuchFieldException|MDBCServiceException e) { // TODO Auto-generated catch block e.printStackTrace(); } @@ -564,10 +567,11 @@ NEW.field refers to the new value * @param transactionDigests * @throws NoSuchFieldException */ - private void updateStagingTable(Map<Range,StagingTable> transactionDigests) throws NoSuchFieldException { + private void updateStagingTable(StagingTable transactionDigests) + throws NoSuchFieldException, MDBCServiceException { // copy from DB.MDBC_TRANSLOG where connid == myconnid // then delete from MDBC_TRANSLOG - String sql2 = "SELECT IX, TABLENAME, OP, KEYDATA, NEWROWDATA FROM "+TRANS_TBL +" WHERE CONNECTION_ID = " + this.connId; + String sql2 = "SELECT IX, TABLENAME, OP, ROWDATA,KEYDATA FROM "+TRANS_TBL +" WHERE CONNECTION_ID = " + this.connId; try { ResultSet rs = executeSQLRead(sql2); Set<Integer> rows = new TreeSet<Integer>(); @@ -576,43 +580,15 @@ NEW.field refers to the new value String op = rs.getString("OP"); OperationType opType = toOpEnum(op); String tbl = rs.getString("TABLENAME"); - JSONObject keydataStr = new JSONObject(new JSONTokener(rs.getString("KEYDATA"))); - String newRowStr = rs.getString("NEWROWDATA"); - JSONObject newRow = new JSONObject(new JSONTokener(newRowStr)); - TableInfo ti = getTableInfo(tbl); - if (!ti.hasKey()) { - //create music key - //\TODO fix, this is completely broken - //if (op.startsWith("I")) { - //\TODO Improve the generation of primary key, it should be generated using - // the actual columns, otherwise performance when doing range queries are going - // to be even worse (see the else bracket down) - // - String musicKey = MDBCUtils.generateUniqueKey().toString(); - /*} else { - //get key from data - musicKey = msm.getMusicKeyFromRowWithoutPrimaryIndexes(tbl,newRow); - }*/ - newRow.put(mi.getMusicDefaultPrimaryKeyName(), musicKey); - keydataStr.put(mi.getMusicDefaultPrimaryKeyName(), musicKey); - } - /*else { - //Use the keys - musicKey = msm.getMusicKeyFromRow(tbl, newRow); - if(musicKey.isEmpty()) { - logger.error(EELFLoggerDelegate.errorLogger,"Primary key is invalid: ["+tbl+","+op+"]"); - throw new NoSuchFieldException("Invalid operation enum"); - } - }*/ + String newRowStr = rs.getString("ROWDATA"); + String rowStr = rs.getString("KEYDATA"); Range range = new Range(tbl); - if(!transactionDigests.containsKey(range)) { - transactionDigests.put(range, new StagingTable()); - } - transactionDigests.get(range).addOperation(opType, newRow.toString(), keydataStr.toString()); + transactionDigests.addOperation(range,opType,newRowStr,rowStr); rows.add(ix); } rs.getStatement().close(); if (rows.size() > 0) { + //TODO: DO batch deletion sql2 = "DELETE FROM "+TRANS_TBL+" WHERE IX = ?"; PreparedStatement ps = jdbcConn.prepareStatement(sql2); logger.debug("Executing: "+sql2); @@ -817,28 +793,26 @@ NEW.field refers to the new value * Parse the transaction digest into individual events * @param transaction - base 64 encoded, serialized digest */ - public void replayTransaction(HashMap<Range,StagingTable> transaction) throws SQLException { + public void replayTransaction(StagingTable transaction, List<Range> ranges) throws SQLException { boolean autocommit = jdbcConn.getAutoCommit(); jdbcConn.setAutoCommit(false); Statement jdbcStmt = jdbcConn.createStatement(); - for (Map.Entry<Range,StagingTable> entry: transaction.entrySet()) { - Range r = entry.getKey(); - StagingTable st = entry.getValue(); - ArrayList<Operation> opList = st.getOperationList(); - - for (Operation op: opList) { - try { - replayOperationIntoDB(jdbcStmt, r, op); - } catch (SQLException e) { - //rollback transaction - logger.error("Unable to replay: " + op.getOperationType() + "->" + op.getNewVal() + "." - + "Rolling back the entire digest replay."); - jdbcConn.rollback(); - throw e; - } - } - } - + ArrayList<Operation> opList = transaction.getOperationList(); + + for (Operation op: opList) { + if(Range.overlaps(ranges,op.getTable())) { + try { + replayOperationIntoDB(jdbcStmt, op); + } catch (SQLException e) { + //rollback transaction + logger.error("Unable to replay: " + op.getOperationType() + "->" + op.getVal() + "." + + "Rolling back the entire digest replay."); + jdbcConn.rollback(); + throw e; + } + } + } + clearReplayedOperations(jdbcStmt); jdbcConn.commit(); jdbcStmt.close(); @@ -861,8 +835,8 @@ NEW.field refers to the new value } @Override - public void applyTxDigest(HashMap<Range, StagingTable> txDigest) throws SQLException { - replayTransaction(txDigest); + public void applyTxDigest(StagingTable txDigest,List<Range> ranges) throws SQLException { + replayTransaction(txDigest,ranges); } /** @@ -872,11 +846,10 @@ NEW.field refers to the new value * @param op * @throws SQLException */ - private void replayOperationIntoDB(Statement jdbcStmt, Range r, Operation op) throws SQLException { - logger.info("Replaying Operation: " + op.getOperationType() + "->" + op.getNewVal()); - JSONObject jsonOp = op.getNewVal(); - JSONObject key = op.getKey(); - + private void replayOperationIntoDB(Statement jdbcStmt, Operation op) throws SQLException { + logger.info("Replaying Operation: " + op.getOperationType() + "->" + op.getVal()); + JSONObject jsonOp = op.getVal(); + ArrayList<String> cols = new ArrayList<String>(); ArrayList<Object> vals = new ArrayList<Object>(); Iterator<String> colIterator = jsonOp.keys(); @@ -897,7 +870,7 @@ NEW.field refers to the new value switch (op.getOperationType()) { case INSERT: sql.append(op.getOperationType() + " INTO "); - sql.append(r.getTable() + " (") ; + sql.append(op.getTable() + " (") ; sep = ""; for (String col: cols) { sql.append(sep + col); @@ -913,20 +886,24 @@ NEW.field refers to the new value break; case UPDATE: sql.append(op.getOperationType() + " "); - sql.append(r.getTable() + " SET "); + sql.append(op.getTable() + " SET "); sep=""; for (int i=0; i<cols.size(); i++) { sql.append(sep + cols.get(i) + "=\"" + vals.get(i) +"\""); sep = ", "; } sql.append(" WHERE "); - sql.append(getPrimaryKeyConditional(op.getKey())); + try { + sql.append(getPrimaryKeyConditional(op.getKey())); + } catch (MDBCServiceException e) { + throw new SQLException("Update operatoin doesn't contain the required primary key",e); + } sql.append(";"); break; case DELETE: sql.append(op.getOperationType() + " FROM "); - sql.append(r.getTable() + " WHERE "); - sql.append(getPrimaryKeyConditional(op.getKey())); + sql.append(op.getTable() + " WHERE "); + sql.append(getPrimaryKeyConditional(op.getVal())); sql.append(";"); break; case SELECT: diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java b/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java index 8ec1793..6b1e566 100644 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpoint.java @@ -150,10 +150,10 @@ public class OwnershipAndCheckpoint{ } } - private void applyTxDigest(DBInterface di, HashMap<Range, StagingTable> txDigest) + private void applyTxDigest(List<Range> ranges, DBInterface di, StagingTable txDigest) throws MDBCServiceException { try { - di.applyTxDigest(txDigest); + di.applyTxDigest(txDigest,ranges); } catch (SQLException e) { throw new MDBCServiceException("Error applying tx digest in local SQL",e); } @@ -185,8 +185,8 @@ public class OwnershipAndCheckpoint{ checkpointLock.unlock(); break; } else { - final HashMap<Range, StagingTable> txDigest = mi.getTxDigest(pair.getKey()); - applyTxDigest(di, txDigest); + final StagingTable txDigest = mi.getTxDigest(pair.getKey()); + applyTxDigest(ranges,di, txDigest); for (Range r : pair.getValue()) { MusicRangeInformationRow row = node.getRow(); alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index)); @@ -213,8 +213,8 @@ public class OwnershipAndCheckpoint{ if(node!=null) { Pair<MusicTxDigestId, List<Range>> pair = node.nextNotAppliedTransaction(rangeSet); while (pair != null) { - final HashMap<Range, StagingTable> txDigest = mi.getTxDigest(pair.getKey()); - applyTxDigest(db, txDigest); + final StagingTable txDigest = mi.getTxDigest(pair.getKey()); + applyTxDigest(ranges, db, txDigest); for (Range r : pair.getValue()) { MusicRangeInformationRow row = node.getRow(); alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index)); diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/proto/ProtoDigest/Digest.java b/mdbc-server/src/main/java/org/onap/music/mdbc/proto/ProtoDigest/Digest.java new file mode 100644 index 0000000..a5cd88f --- /dev/null +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/proto/ProtoDigest/Digest.java @@ -0,0 +1,1899 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: digest.proto + +package org.onap.music.mdbc.proto.ProtoDigest; + +public final class Digest { + private Digest() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions( + (com.google.protobuf.ExtensionRegistryLite) registry); + } + public interface RowOrBuilder extends + // @@protoc_insertion_point(interface_extends:org.onap.music.mdbc.proto.ProtoDigest.Row) + com.google.protobuf.MessageOrBuilder { + + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + int getTypeValue(); + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType getType(); + + /** + * <code>string key = 2;</code> + */ + java.lang.String getKey(); + /** + * <code>string key = 2;</code> + */ + com.google.protobuf.ByteString + getKeyBytes(); + + /** + * <code>string val = 3;</code> + */ + java.lang.String getVal(); + /** + * <code>string val = 3;</code> + */ + com.google.protobuf.ByteString + getValBytes(); + + /** + * <code>string table = 4;</code> + */ + java.lang.String getTable(); + /** + * <code>string table = 4;</code> + */ + com.google.protobuf.ByteString + getTableBytes(); + } + /** + * Protobuf type {@code org.onap.music.mdbc.proto.ProtoDigest.Row} + */ + public static final class Row extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:org.onap.music.mdbc.proto.ProtoDigest.Row) + RowOrBuilder { + private static final long serialVersionUID = 0L; + // Use Row.newBuilder() to construct. + private Row(com.google.protobuf.GeneratedMessageV3.Builder<?> builder) { + super(builder); + } + private Row() { + type_ = 0; + key_ = ""; + val_ = ""; + table_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private Row( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 8: { + int rawValue = input.readEnum(); + + type_ = rawValue; + break; + } + case 18: { + java.lang.String s = input.readStringRequireUtf8(); + + key_ = s; + break; + } + case 26: { + java.lang.String s = input.readStringRequireUtf8(); + + val_ = s; + break; + } + case 34: { + java.lang.String s = input.readStringRequireUtf8(); + + table_ = s; + break; + } + default: { + if (!parseUnknownFieldProto3( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.class, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder.class); + } + + /** + * Protobuf enum {@code org.onap.music.mdbc.proto.ProtoDigest.Row.OpType} + */ + public enum OpType + implements com.google.protobuf.ProtocolMessageEnum { + /** + * <code>INSERT = 0;</code> + */ + INSERT(0), + /** + * <code>UPDATE = 1;</code> + */ + UPDATE(1), + /** + * <code>DELETE = 2;</code> + */ + DELETE(2), + UNRECOGNIZED(-1), + ; + + /** + * <code>INSERT = 0;</code> + */ + public static final int INSERT_VALUE = 0; + /** + * <code>UPDATE = 1;</code> + */ + public static final int UPDATE_VALUE = 1; + /** + * <code>DELETE = 2;</code> + */ + public static final int DELETE_VALUE = 2; + + + public final int getNumber() { + if (this == UNRECOGNIZED) { + throw new java.lang.IllegalArgumentException( + "Can't get the number of an unknown enum value."); + } + return value; + } + + /** + * @deprecated Use {@link #forNumber(int)} instead. + */ + @java.lang.Deprecated + public static OpType valueOf(int value) { + return forNumber(value); + } + + public static OpType forNumber(int value) { + switch (value) { + case 0: return INSERT; + case 1: return UPDATE; + case 2: return DELETE; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap<OpType> + internalGetValueMap() { + return internalValueMap; + } + private static final com.google.protobuf.Internal.EnumLiteMap< + OpType> internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap<OpType>() { + public OpType findValueByNumber(int number) { + return OpType.forNumber(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(ordinal()); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.getDescriptor().getEnumTypes().get(0); + } + + private static final OpType[] VALUES = values(); + + public static OpType valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + if (desc.getIndex() == -1) { + return UNRECOGNIZED; + } + return VALUES[desc.getIndex()]; + } + + private final int value; + + private OpType(int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:org.onap.music.mdbc.proto.ProtoDigest.Row.OpType) + } + + public static final int TYPE_FIELD_NUMBER = 1; + private int type_; + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + public int getTypeValue() { + return type_; + } + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType getType() { + @SuppressWarnings("deprecation") + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType result = org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType.valueOf(type_); + return result == null ? org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType.UNRECOGNIZED : result; + } + + public static final int KEY_FIELD_NUMBER = 2; + private volatile java.lang.Object key_; + /** + * <code>string key = 2;</code> + */ + public java.lang.String getKey() { + java.lang.Object ref = key_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + key_ = s; + return s; + } + } + /** + * <code>string key = 2;</code> + */ + public com.google.protobuf.ByteString + getKeyBytes() { + java.lang.Object ref = key_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + key_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int VAL_FIELD_NUMBER = 3; + private volatile java.lang.Object val_; + /** + * <code>string val = 3;</code> + */ + public java.lang.String getVal() { + java.lang.Object ref = val_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + val_ = s; + return s; + } + } + /** + * <code>string val = 3;</code> + */ + public com.google.protobuf.ByteString + getValBytes() { + java.lang.Object ref = val_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + val_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int TABLE_FIELD_NUMBER = 4; + private volatile java.lang.Object table_; + /** + * <code>string table = 4;</code> + */ + public java.lang.String getTable() { + java.lang.Object ref = table_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + table_ = s; + return s; + } + } + /** + * <code>string table = 4;</code> + */ + public com.google.protobuf.ByteString + getTableBytes() { + java.lang.Object ref = table_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + table_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + @java.lang.Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + @java.lang.Override + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (type_ != org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType.INSERT.getNumber()) { + output.writeEnum(1, type_); + } + if (!getKeyBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, key_); + } + if (!getValBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, val_); + } + if (!getTableBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 4, table_); + } + unknownFields.writeTo(output); + } + + @java.lang.Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + if (type_ != org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType.INSERT.getNumber()) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(1, type_); + } + if (!getKeyBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, key_); + } + if (!getValBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, val_); + } + if (!getTableBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(4, table_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.onap.music.mdbc.proto.ProtoDigest.Digest.Row)) { + return super.equals(obj); + } + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row other = (org.onap.music.mdbc.proto.ProtoDigest.Digest.Row) obj; + + boolean result = true; + result = result && type_ == other.type_; + result = result && getKey() + .equals(other.getKey()); + result = result && getVal() + .equals(other.getVal()); + result = result && getTable() + .equals(other.getTable()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + TYPE_FIELD_NUMBER; + hash = (53 * hash) + type_; + hash = (37 * hash) + KEY_FIELD_NUMBER; + hash = (53 * hash) + getKey().hashCode(); + hash = (37 * hash) + VAL_FIELD_NUMBER; + hash = (53 * hash) + getVal().hashCode(); + hash = (37 * hash) + TABLE_FIELD_NUMBER; + hash = (53 * hash) + getTable().hashCode(); + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + @java.lang.Override + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(org.onap.music.mdbc.proto.ProtoDigest.Digest.Row prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + @java.lang.Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.onap.music.mdbc.proto.ProtoDigest.Row} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder<Builder> implements + // @@protoc_insertion_point(builder_implements:org.onap.music.mdbc.proto.ProtoDigest.Row) + org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.class, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder.class); + } + + // Construct using org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + } + } + @java.lang.Override + public Builder clear() { + super.clear(); + type_ = 0; + + key_ = ""; + + val_ = ""; + + table_ = ""; + + return this; + } + + @java.lang.Override + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_descriptor; + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row getDefaultInstanceForType() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.getDefaultInstance(); + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row build() { + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row buildPartial() { + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row result = new org.onap.music.mdbc.proto.ProtoDigest.Digest.Row(this); + result.type_ = type_; + result.key_ = key_; + result.val_ = val_; + result.table_ = table_; + onBuilt(); + return result; + } + + @java.lang.Override + public Builder clone() { + return (Builder) super.clone(); + } + @java.lang.Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.setField(field, value); + } + @java.lang.Override + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + @java.lang.Override + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + @java.lang.Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + @java.lang.Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + @java.lang.Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.onap.music.mdbc.proto.ProtoDigest.Digest.Row) { + return mergeFrom((org.onap.music.mdbc.proto.ProtoDigest.Digest.Row)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.onap.music.mdbc.proto.ProtoDigest.Digest.Row other) { + if (other == org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.getDefaultInstance()) return this; + if (other.type_ != 0) { + setTypeValue(other.getTypeValue()); + } + if (!other.getKey().isEmpty()) { + key_ = other.key_; + onChanged(); + } + if (!other.getVal().isEmpty()) { + val_ = other.val_; + onChanged(); + } + if (!other.getTable().isEmpty()) { + table_ = other.table_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + @java.lang.Override + public final boolean isInitialized() { + return true; + } + + @java.lang.Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.onap.music.mdbc.proto.ProtoDigest.Digest.Row) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int type_ = 0; + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + public int getTypeValue() { + return type_; + } + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + public Builder setTypeValue(int value) { + type_ = value; + onChanged(); + return this; + } + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType getType() { + @SuppressWarnings("deprecation") + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType result = org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType.valueOf(type_); + return result == null ? org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType.UNRECOGNIZED : result; + } + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + public Builder setType(org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType value) { + if (value == null) { + throw new NullPointerException(); + } + + type_ = value.getNumber(); + onChanged(); + return this; + } + /** + * <code>.org.onap.music.mdbc.proto.ProtoDigest.Row.OpType type = 1;</code> + */ + public Builder clearType() { + + type_ = 0; + onChanged(); + return this; + } + + private java.lang.Object key_ = ""; + /** + * <code>string key = 2;</code> + */ + public java.lang.String getKey() { + java.lang.Object ref = key_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + key_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * <code>string key = 2;</code> + */ + public com.google.protobuf.ByteString + getKeyBytes() { + java.lang.Object ref = key_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + key_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * <code>string key = 2;</code> + */ + public Builder setKey( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + key_ = value; + onChanged(); + return this; + } + /** + * <code>string key = 2;</code> + */ + public Builder clearKey() { + + key_ = getDefaultInstance().getKey(); + onChanged(); + return this; + } + /** + * <code>string key = 2;</code> + */ + public Builder setKeyBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + key_ = value; + onChanged(); + return this; + } + + private java.lang.Object val_ = ""; + /** + * <code>string val = 3;</code> + */ + public java.lang.String getVal() { + java.lang.Object ref = val_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + val_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * <code>string val = 3;</code> + */ + public com.google.protobuf.ByteString + getValBytes() { + java.lang.Object ref = val_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + val_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * <code>string val = 3;</code> + */ + public Builder setVal( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + val_ = value; + onChanged(); + return this; + } + /** + * <code>string val = 3;</code> + */ + public Builder clearVal() { + + val_ = getDefaultInstance().getVal(); + onChanged(); + return this; + } + /** + * <code>string val = 3;</code> + */ + public Builder setValBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + val_ = value; + onChanged(); + return this; + } + + private java.lang.Object table_ = ""; + /** + * <code>string table = 4;</code> + */ + public java.lang.String getTable() { + java.lang.Object ref = table_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + table_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * <code>string table = 4;</code> + */ + public com.google.protobuf.ByteString + getTableBytes() { + java.lang.Object ref = table_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + table_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * <code>string table = 4;</code> + */ + public Builder setTable( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + table_ = value; + onChanged(); + return this; + } + /** + * <code>string table = 4;</code> + */ + public Builder clearTable() { + + table_ = getDefaultInstance().getTable(); + onChanged(); + return this; + } + /** + * <code>string table = 4;</code> + */ + public Builder setTableBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + table_ = value; + onChanged(); + return this; + } + @java.lang.Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFieldsProto3(unknownFields); + } + + @java.lang.Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + + // @@protoc_insertion_point(builder_scope:org.onap.music.mdbc.proto.ProtoDigest.Row) + } + + // @@protoc_insertion_point(class_scope:org.onap.music.mdbc.proto.ProtoDigest.Row) + private static final org.onap.music.mdbc.proto.ProtoDigest.Digest.Row DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new org.onap.music.mdbc.proto.ProtoDigest.Digest.Row(); + } + + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.Row getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser<Row> + PARSER = new com.google.protobuf.AbstractParser<Row>() { + @java.lang.Override + public Row parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Row(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser<Row> parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser<Row> getParserForType() { + return PARSER; + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface CompleteDigestOrBuilder extends + // @@protoc_insertion_point(interface_extends:org.onap.music.mdbc.proto.ProtoDigest.CompleteDigest) + com.google.protobuf.MessageOrBuilder { + + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + java.util.List<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row> + getRowsList(); + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row getRows(int index); + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + int getRowsCount(); + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + java.util.List<? extends org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder> + getRowsOrBuilderList(); + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder getRowsOrBuilder( + int index); + } + /** + * Protobuf type {@code org.onap.music.mdbc.proto.ProtoDigest.CompleteDigest} + */ + public static final class CompleteDigest extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:org.onap.music.mdbc.proto.ProtoDigest.CompleteDigest) + CompleteDigestOrBuilder { + private static final long serialVersionUID = 0L; + // Use CompleteDigest.newBuilder() to construct. + private CompleteDigest(com.google.protobuf.GeneratedMessageV3.Builder<?> builder) { + super(builder); + } + private CompleteDigest() { + rows_ = java.util.Collections.emptyList(); + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private CompleteDigest( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + rows_ = new java.util.ArrayList<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row>(); + mutable_bitField0_ |= 0x00000001; + } + rows_.add( + input.readMessage(org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.parser(), extensionRegistry)); + break; + } + default: { + if (!parseUnknownFieldProto3( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + rows_ = java.util.Collections.unmodifiableList(rows_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.class, org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.Builder.class); + } + + public static final int ROWS_FIELD_NUMBER = 1; + private java.util.List<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row> rows_; + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public java.util.List<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row> getRowsList() { + return rows_; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public java.util.List<? extends org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder> + getRowsOrBuilderList() { + return rows_; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public int getRowsCount() { + return rows_.size(); + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row getRows(int index) { + return rows_.get(index); + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder getRowsOrBuilder( + int index) { + return rows_.get(index); + } + + private byte memoizedIsInitialized = -1; + @java.lang.Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + @java.lang.Override + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + for (int i = 0; i < rows_.size(); i++) { + output.writeMessage(1, rows_.get(i)); + } + unknownFields.writeTo(output); + } + + @java.lang.Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < rows_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, rows_.get(i)); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest)) { + return super.equals(obj); + } + org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest other = (org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest) obj; + + boolean result = true; + result = result && getRowsList() + .equals(other.getRowsList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getRowsCount() > 0) { + hash = (37 * hash) + ROWS_FIELD_NUMBER; + hash = (53 * hash) + getRowsList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + @java.lang.Override + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + @java.lang.Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.onap.music.mdbc.proto.ProtoDigest.CompleteDigest} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder<Builder> implements + // @@protoc_insertion_point(builder_implements:org.onap.music.mdbc.proto.ProtoDigest.CompleteDigest) + org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.class, org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.Builder.class); + } + + // Construct using org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + getRowsFieldBuilder(); + } + } + @java.lang.Override + public Builder clear() { + super.clear(); + if (rowsBuilder_ == null) { + rows_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + rowsBuilder_.clear(); + } + return this; + } + + @java.lang.Override + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_descriptor; + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest getDefaultInstanceForType() { + return org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.getDefaultInstance(); + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest build() { + org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest buildPartial() { + org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest result = new org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest(this); + int from_bitField0_ = bitField0_; + if (rowsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + rows_ = java.util.Collections.unmodifiableList(rows_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.rows_ = rows_; + } else { + result.rows_ = rowsBuilder_.build(); + } + onBuilt(); + return result; + } + + @java.lang.Override + public Builder clone() { + return (Builder) super.clone(); + } + @java.lang.Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.setField(field, value); + } + @java.lang.Override + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + @java.lang.Override + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + @java.lang.Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + @java.lang.Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + @java.lang.Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest) { + return mergeFrom((org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest other) { + if (other == org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.getDefaultInstance()) return this; + if (rowsBuilder_ == null) { + if (!other.rows_.isEmpty()) { + if (rows_.isEmpty()) { + rows_ = other.rows_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureRowsIsMutable(); + rows_.addAll(other.rows_); + } + onChanged(); + } + } else { + if (!other.rows_.isEmpty()) { + if (rowsBuilder_.isEmpty()) { + rowsBuilder_.dispose(); + rowsBuilder_ = null; + rows_ = other.rows_; + bitField0_ = (bitField0_ & ~0x00000001); + rowsBuilder_ = + com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? + getRowsFieldBuilder() : null; + } else { + rowsBuilder_.addAllMessages(other.rows_); + } + } + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + @java.lang.Override + public final boolean isInitialized() { + return true; + } + + @java.lang.Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + private java.util.List<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row> rows_ = + java.util.Collections.emptyList(); + private void ensureRowsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + rows_ = new java.util.ArrayList<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row>(rows_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3< + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder, org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder> rowsBuilder_; + + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public java.util.List<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row> getRowsList() { + if (rowsBuilder_ == null) { + return java.util.Collections.unmodifiableList(rows_); + } else { + return rowsBuilder_.getMessageList(); + } + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public int getRowsCount() { + if (rowsBuilder_ == null) { + return rows_.size(); + } else { + return rowsBuilder_.getCount(); + } + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row getRows(int index) { + if (rowsBuilder_ == null) { + return rows_.get(index); + } else { + return rowsBuilder_.getMessage(index); + } + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder setRows( + int index, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row value) { + if (rowsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRowsIsMutable(); + rows_.set(index, value); + onChanged(); + } else { + rowsBuilder_.setMessage(index, value); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder setRows( + int index, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder builderForValue) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.set(index, builderForValue.build()); + onChanged(); + } else { + rowsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder addRows(org.onap.music.mdbc.proto.ProtoDigest.Digest.Row value) { + if (rowsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRowsIsMutable(); + rows_.add(value); + onChanged(); + } else { + rowsBuilder_.addMessage(value); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder addRows( + int index, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row value) { + if (rowsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRowsIsMutable(); + rows_.add(index, value); + onChanged(); + } else { + rowsBuilder_.addMessage(index, value); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder addRows( + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder builderForValue) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.add(builderForValue.build()); + onChanged(); + } else { + rowsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder addRows( + int index, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder builderForValue) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.add(index, builderForValue.build()); + onChanged(); + } else { + rowsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder addAllRows( + java.lang.Iterable<? extends org.onap.music.mdbc.proto.ProtoDigest.Digest.Row> values) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll( + values, rows_); + onChanged(); + } else { + rowsBuilder_.addAllMessages(values); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder clearRows() { + if (rowsBuilder_ == null) { + rows_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + rowsBuilder_.clear(); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public Builder removeRows(int index) { + if (rowsBuilder_ == null) { + ensureRowsIsMutable(); + rows_.remove(index); + onChanged(); + } else { + rowsBuilder_.remove(index); + } + return this; + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder getRowsBuilder( + int index) { + return getRowsFieldBuilder().getBuilder(index); + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder getRowsOrBuilder( + int index) { + if (rowsBuilder_ == null) { + return rows_.get(index); } else { + return rowsBuilder_.getMessageOrBuilder(index); + } + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public java.util.List<? extends org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder> + getRowsOrBuilderList() { + if (rowsBuilder_ != null) { + return rowsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(rows_); + } + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder addRowsBuilder() { + return getRowsFieldBuilder().addBuilder( + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.getDefaultInstance()); + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder addRowsBuilder( + int index) { + return getRowsFieldBuilder().addBuilder( + index, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.getDefaultInstance()); + } + /** + * <code>repeated .org.onap.music.mdbc.proto.ProtoDigest.Row rows = 1;</code> + */ + public java.util.List<org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder> + getRowsBuilderList() { + return getRowsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilderV3< + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder, org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder> + getRowsFieldBuilder() { + if (rowsBuilder_ == null) { + rowsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< + org.onap.music.mdbc.proto.ProtoDigest.Digest.Row, org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.Builder, org.onap.music.mdbc.proto.ProtoDigest.Digest.RowOrBuilder>( + rows_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + rows_ = null; + } + return rowsBuilder_; + } + @java.lang.Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFieldsProto3(unknownFields); + } + + @java.lang.Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + + // @@protoc_insertion_point(builder_scope:org.onap.music.mdbc.proto.ProtoDigest.CompleteDigest) + } + + // @@protoc_insertion_point(class_scope:org.onap.music.mdbc.proto.ProtoDigest.CompleteDigest) + private static final org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest(); + } + + public static org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser<CompleteDigest> + PARSER = new com.google.protobuf.AbstractParser<CompleteDigest>() { + @java.lang.Override + public CompleteDigest parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new CompleteDigest(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser<CompleteDigest> parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser<CompleteDigest> getParserForType() { + return PARSER; + } + + @java.lang.Override + public org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\014digest.proto\022%org.onap.music.mdbc.prot" + + "o.ProtoDigest\"\235\001\n\003Row\022?\n\004type\030\001 \001(\01621.or" + + "g.onap.music.mdbc.proto.ProtoDigest.Row." + + "OpType\022\013\n\003key\030\002 \001(\t\022\013\n\003val\030\003 \001(\t\022\r\n\005tabl" + + "e\030\004 \001(\t\",\n\006OpType\022\n\n\006INSERT\020\000\022\n\n\006UPDATE\020" + + "\001\022\n\n\006DELETE\020\002\"J\n\016CompleteDigest\0228\n\004rows\030" + + "\001 \003(\0132*.org.onap.music.mdbc.proto.ProtoD" + + "igest.Rowb\006proto3" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_org_onap_music_mdbc_proto_ProtoDigest_Row_descriptor, + new java.lang.String[] { "Type", "Key", "Val", "Table", }); + internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_org_onap_music_mdbc_proto_ProtoDigest_CompleteDigest_descriptor, + new java.lang.String[] { "Rows", }); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/proto/digest.proto b/mdbc-server/src/main/java/org/onap/music/mdbc/proto/digest.proto new file mode 100644 index 0000000..61b28ec --- /dev/null +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/proto/digest.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package org.onap.music.mdbc.proto.ProtoDigest; + +// *********************** +// To generate: +// 1. cd to this directory +// 2. Using protoc ver 3.6.1 (change following command to protoc) +// protoc-3.6.1 --java_out=../../../../../ digest.proto +// This is going to generate the folder ProtoDigest in this same directory +// *********************** + +message Row{ + enum OpType { + INSERT = 0; + UPDATE = 1; + DELETE = 2; + } + OpType type = 1; + string key = 2; + string val = 3; + string table = 4; +} + +message CompleteDigest{ + repeated Row rows = 1; +} diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigest.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigest.java index 204292c..03db7a7 100644 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigest.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigest.java @@ -88,7 +88,7 @@ public class MusicTxDigest { //Step 3: ReplayDigest() for E.C conditions try { - replayDigest(mi,dbi); + replayDigest(mi,dbi, stateManager.getEventualRanges()); } catch (MDBCServiceException e) { logger.error("Unable to perform Eventual Consistency operations" + e.getMessage()); continue; @@ -100,35 +100,34 @@ public class MusicTxDigest { /** * Replay the digest for eventual consistency. * @param mi music interface - * @param partitionId the partition to be replayed * @param dbi interface to the database that will replay the operations + * @param ranges only these ranges will be applied from the digests * @throws MDBCServiceException */ - public void replayDigest(MusicInterface mi, DBInterface dbi) throws MDBCServiceException { - HashMap<Range, StagingTable> transaction; - String nodeName = stateManager.getMdbcServerName(); - logger.info("Node Name: "+nodeName); - LinkedHashMap<UUID, HashMap<Range,StagingTable>> ecDigestInformation = mi.getEveTxDigest(nodeName); - Set<UUID> keys = ecDigestInformation.keySet(); - - for(UUID txTimeID:keys){ - transaction = (HashMap<Range,StagingTable>) ecDigestInformation.get(txTimeID); - - try { - dbi.replayTransaction(transaction); // I think this Might change if the data is coming from a new table.. ( what is the new table structure??) - } catch (SQLException e) { - logger.error("EC:Rolling back the entire digest replay."); - return; - } - logger.info("EC: Successfully replayed transaction for txTimeID key: "+txTimeID); - - try { - mi.updateNodeInfoTableWithTxTimeIDKey(txTimeID, nodeName); - } catch (MDBCServiceException e) { - logger.error("EC:Rolling back the entire digest replay."); - } - } - } + public void replayDigest(MusicInterface mi, DBInterface dbi, List<Range> ranges) throws MDBCServiceException { + StagingTable transaction; + String nodeName = stateManager.getMdbcServerName(); + + LinkedHashMap<UUID,StagingTable> ecDigestInformation = mi.getEveTxDigest(nodeName); + Set<UUID> keys = ecDigestInformation.keySet(); + for(UUID txTimeID:keys){ + transaction = ecDigestInformation.get(txTimeID); + try { + dbi.replayTransaction(transaction, ranges); // I think this Might change if the data is coming from a new table.. ( what is the new table structure??) + } catch (SQLException e) { + logger.error("EC:Rolling back the entire digest replay."); + return; + } + logger.info("EC: Successfully replayed transaction "); + + try { + mi.updateNodeInfoTableWithTxTimeIDKey(txTimeID, nodeName); + } catch (MDBCServiceException e) { + logger.error("EC:Rolling back the entire digest replay."); + } + } +; + } /** @@ -139,12 +138,13 @@ public class MusicTxDigest { * @throws MDBCServiceException */ public static void replayDigestForPartition(MusicInterface mi, UUID partitionId, DBInterface dbi) throws MDBCServiceException { - List<MusicTxDigestId> partitionsRedoLogTxIds = mi.getMusicRangeInformation(partitionId).getRedoLog(); + final MusicRangeInformationRow row = mi.getMusicRangeInformation(partitionId); + List<MusicTxDigestId> partitionsRedoLogTxIds = row.getRedoLog(); for (MusicTxDigestId txId: partitionsRedoLogTxIds) { - HashMap<Range, StagingTable> transaction = mi.getTxDigest(txId); + StagingTable transaction = mi.getTxDigest(txId); try { //\TODO do this two operations in parallel - dbi.replayTransaction(transaction); + dbi.replayTransaction(transaction, row.getDBPartition().getSnapshot()); mi.replayTransaction(transaction); } catch (SQLException e) { logger.error("Rolling back the entire digest replay. " + partitionId); diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java index dc1bcce..a9ab25f 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java @@ -23,6 +23,7 @@ import java.io.Serializable; import org.json.JSONObject; import org.json.JSONTokener; +import org.onap.music.exceptions.MDBCServiceException; import static java.util.Objects.hash; @@ -31,32 +32,41 @@ public final class Operation implements Serializable{ private static final long serialVersionUID = -1215301985078183104L; final OperationType TYPE; - final String NEW_VAL; + final String VAL; final String KEY; + final String TABLE; - public Operation(OperationType type, String newVal, String key) { + public Operation(String table,OperationType type, String newVal, String key) { + TABLE = table; TYPE = type; - NEW_VAL = newVal; + VAL = newVal; KEY = key; } - public JSONObject getNewVal(){ - JSONObject newRow = new JSONObject(new JSONTokener(NEW_VAL)); + public String getTable(){ + return TABLE; + } + + public JSONObject getVal(){ + JSONObject newRow = new JSONObject(new JSONTokener(VAL)); return newRow; } - public JSONObject getKey() { - JSONObject key = new JSONObject(new JSONTokener(KEY)); - return key; - } - + public JSONObject getKey() throws MDBCServiceException { + if(KEY==null){ + throw new MDBCServiceException("This operation ["+TYPE.toString()+"] doesn't contain a key"); + } + JSONObject keys = new JSONObject(new JSONTokener(KEY)); + return keys; + } + public OperationType getOperationType() { return this.TYPE; } @Override public int hashCode(){ - return hash(TYPE,NEW_VAL); + return hash(TYPE,VAL); } @Override @@ -64,6 +74,6 @@ public final class Operation implements Serializable{ if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Operation r = (Operation) o; - return TYPE.equals(r.TYPE) && NEW_VAL.equals(r.NEW_VAL); + return TABLE.equals(r.TABLE) && TYPE.equals(r.TYPE) && VAL.equals(r.VAL); } } diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java index fcff5ff..03c7259 100755 --- a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java @@ -19,39 +19,141 @@ */ package org.onap.music.mdbc.tables; -import java.io.Serializable; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Deque; -import java.util.HashMap; -import java.util.LinkedList; +import java.util.HashSet; +import java.util.List; import java.util.Set; -import org.apache.commons.lang3.tuple.Pair; -import org.json.JSONObject; - +import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.logging.EELFLoggerDelegate; +import org.onap.music.mdbc.Range; +import org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest; +import org.onap.music.mdbc.proto.ProtoDigest.Digest.CompleteDigest.Builder; +import org.onap.music.mdbc.proto.ProtoDigest.Digest.Row; +import org.onap.music.mdbc.proto.ProtoDigest.Digest.Row.OpType; + +public class StagingTable { -public class StagingTable implements Serializable{ - /** - * - */ - private static final long serialVersionUID = 7583182634761771943L; private transient static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(StagingTable.class); - //primary key -> Operation - private ArrayList<Operation> operations; + private ArrayList<Operation> operations; + boolean builderInitialized; + Builder digestBuilder; + Builder eventuallyBuilder; + Set<Range> eventuallyConsistentRanges; + + public StagingTable(){ + this(new HashSet<>()); + logger.debug("Creating staging table with no parameters, most likely this is wrong, unless you are testing"); + } - public StagingTable() { - operations = new ArrayList<Operation>(); + public StagingTable(Set<Range> eventuallyConsistentRanges) { + //operations = new ArrayList<Operation>(); + operations=null; + builderInitialized=true; + digestBuilder = CompleteDigest.newBuilder(); + this.eventuallyConsistentRanges=eventuallyConsistentRanges; + eventuallyBuilder = (!this.eventuallyConsistentRanges.isEmpty())?null:CompleteDigest.newBuilder(); } + + public StagingTable(ByteBuffer serialized) throws MDBCServiceException { + builderInitialized=false; + operations = new ArrayList<>(); + CompleteDigest completeDigest; + try { + completeDigest = CompleteDigest.parseFrom(serialized); + } catch (InvalidProtocolBufferException e) { + throw new MDBCServiceException("Invalid serialized input to protobuf deserializer",e); + } + for(Row row : completeDigest.getRowsList()){ + final OpType type = row.getType(); + OperationType newType = (type==OpType.INSERT)?OperationType.INSERT:(type==OpType.DELETE)? + OperationType.DELETE:OperationType.UPDATE; + operations.add(new Operation(row.getTable(),newType,row.getVal(),row.getKey())); + } + } + + synchronized public boolean isBuilderInitialized(){ + return isBuilderInitialized(); + } - synchronized public void addOperation(OperationType type, String newVal, String key) { - operations.add(new Operation(type,newVal, key)); + synchronized public void addOperation(Range range, OperationType type, String newVal, String keys) + throws MDBCServiceException { + if(!builderInitialized){ + throw new MDBCServiceException("This type of staging table is unmutable, please use the constructor" + + "with no parameters"); + } + OpType newType = (type==OperationType.INSERT)?OpType.INSERT:(type==OperationType.DELETE)? + OpType.DELETE:OpType.UPDATE; + Row.Builder rowBuilder = Row.newBuilder().setTable(range.getTable()).setType(newType).setVal(newVal); + if(keys!=null){ + rowBuilder.setKey(keys); + } + if(eventuallyConsistentRanges!=null && eventuallyConsistentRanges.contains(range)){ + if(eventuallyBuilder==null){ + throw new MDBCServiceException("INCONSISTENCY: trying to add eventual op with no eventual ranges"); + } + eventuallyBuilder.addRows(rowBuilder); + } + else { + digestBuilder.addRows(rowBuilder); + } + //operations.add(new Operation(table,type,newVal,keys)); } synchronized public ArrayList<Operation> getOperationList() { - return operations; - } + if(!builderInitialized) { + return operations; + } + logger.warn("Get operation list with this type of initialization is not suggested for the" + + "staging table"); + ArrayList newOperations = new ArrayList(); + for(Row row : digestBuilder.getRowsList()){ + final OpType type = row.getType(); + OperationType newType = (type==OpType.INSERT)?OperationType.INSERT:(type==OpType.DELETE)? + OperationType.DELETE:OperationType.UPDATE; + newOperations.add(new Operation(row.getTable(),newType,row.getVal(),row.getKey())); + } + return newOperations; + } + + synchronized public ByteBuffer getSerializedStagingAndClean() throws MDBCServiceException { + if(!builderInitialized){ + throw new MDBCServiceException("This type of staging table is unmutable, please use the constructor" + + "with no parameters"); + } + ByteString serialized = digestBuilder.build().toByteString(); + digestBuilder.clear(); + return serialized.asReadOnlyByteBuffer(); + } + + synchronized public ByteBuffer getSerializedEventuallyStagingAndClean() throws MDBCServiceException { + if(!builderInitialized){ + throw new MDBCServiceException("This type of staging table is unmutable, please use the constructor" + + "with no parameters"); + } + if(eventuallyBuilder == null || eventuallyBuilder.getRowsCount()==0){ + return null; + } + ByteString serialized = eventuallyBuilder.build().toByteString(); + eventuallyBuilder.clear(); + return serialized.asReadOnlyByteBuffer(); + } + + synchronized public boolean isEmpty() { + return (digestBuilder.getRowsCount()==0); + } - synchronized public void clean() { - operations.clear(); + synchronized public void clear() throws MDBCServiceException { + if(!builderInitialized){ + throw new MDBCServiceException("This type of staging table is unmutable, please use the constructor" + + "with no parameters"); + } + digestBuilder.clear(); } + + synchronized public boolean areEventualContained(List<Range> ranges){ + return eventuallyConsistentRanges.containsAll(ranges); + } } diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/MDBCUtilsTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/MDBCUtilsTest.java index 676d760..a9cf88a 100755 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/MDBCUtilsTest.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/MDBCUtilsTest.java @@ -23,73 +23,35 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.HashMap; +import java.util.HashSet; import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; +import org.onap.music.exceptions.MDBCServiceException; import org.onap.music.mdbc.tables.OperationType; import org.onap.music.mdbc.tables.StagingTable; -@Ignore public class MDBCUtilsTest { - @Test - public void toStringTest1() { - StagingTable table = new StagingTable(); - table.addOperation(OperationType.INSERT,(new JSONObject(new String[]{"test3", "Test4"})).toString(), - (new JSONObject(new String[]{"test_key", "test_value"})).toString()); - String output=null; - try { - output = MDBCUtils.toString(table); - } catch (IOException e) { - e.printStackTrace(); - fail(); - } - assertTrue(output!=null); - assertTrue(!output.isEmpty()); - } - @Test - public void toStringTest2() { - HashMap<String,StagingTable> mapToSerialize = new HashMap<>(); + public void toStringTest1() { StagingTable table = new StagingTable(); - table.addOperation(OperationType.INSERT,(new JSONObject(new String[]{"test3", "Test4"}).toString()), - (new JSONObject(new String[]{"test_key", "test_value"})).toString()); - mapToSerialize.put("table",table); - String output=null; try { - output = MDBCUtils.toString(mapToSerialize); - } catch (IOException e) { - e.printStackTrace(); - fail(); - } - assertTrue(output!=null); - assertTrue(!output.isEmpty()); - } - - @Test - public void toStringTest3() { - String testStr = "test"; - OperationType typeTest = OperationType.INSERT; - String output=null; - try { - output = MDBCUtils.toString(testStr); - } catch (IOException e) { - e.printStackTrace(); + table.addOperation(new Range("TABLE1"),OperationType.INSERT,(new JSONObject(new String[]{"test3", "Test4"})).toString(),null); + } catch (MDBCServiceException e) { fail(); } - assertTrue(output!=null); - assertTrue(!output.isEmpty()); - output=null; + ByteBuffer output=null; try { - output = MDBCUtils.toString(typeTest); - } catch (IOException e) { + output = table.getSerializedStagingAndClean(); + } catch (MDBCServiceException e) { e.printStackTrace(); fail(); } assertTrue(output!=null); - assertTrue(!output.isEmpty()); + assertTrue(output.toString().length() > 0); } - } diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java index 8a185ea..df673c9 100644 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/mixins/MusicMixinTest.java @@ -77,11 +77,11 @@ public class MusicMixinTest { } catch (Exception e) { System.out.println(e); } - - cluster = new Cluster.Builder().addContactPoint(cassaHost).withPort(9142).build(); + cluster=EmbeddedCassandraServerHelper.getCluster(); + //cluster = new Cluster.Builder().addContactPoint(cassaHost).withPort(9142).build(); cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis(20000); assertNotNull("Invalid configuration for cassandra", cluster); - session = cluster.connect(); + session = EmbeddedCassandraServerHelper.getSession(); assertNotNull("Invalid configuration for cassandra", session); MusicDataStoreHandle.mDstoreHandle = new MusicDataStore(cluster, session); @@ -92,8 +92,12 @@ public class MusicMixinTest { @AfterClass public static void close() throws MusicServiceException, MusicQueryException { //TODO: shutdown cassandra - session.close(); - cluster.close(); + mixin=null; + try { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra(); + } + catch(NullPointerException e){ + } } @Before @@ -146,7 +150,7 @@ public class MusicMixinTest { return partition; } - @Test(timeout=1000) + @Test(timeout=10000) public void own2() throws InterruptedException, MDBCServiceException { List<Range> range12 = new ArrayList<>( Arrays.asList( new Range("RANGE1"), diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java index 0c2a804..4950484 100644 --- a/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java +++ b/mdbc-server/src/test/java/org/onap/music/mdbc/ownership/OwnershipAndCheckpointTest.java @@ -61,7 +61,7 @@ public class OwnershipAndCheckpointTest { final private static String mtdTableName = "musictxdigest"; final private static String mdbcServerName = "name"; public static final String DATABASE = "mdbcTest"; - public static final String TABLE= "Persons"; + public static final String TABLE= "PERSONS"; public static final String CREATE_TABLE = "CREATE TABLE IF NOT EXISTS " + TABLE + " (\n" + " PersonID int,\n" + " LastName varchar(255),\n" + @@ -87,10 +87,11 @@ public class OwnershipAndCheckpointTest { } catch (Exception e) { fail(e.getMessage()); } - cluster = new Cluster.Builder().addContactPoint(cassaHost).withPort(9142).build(); + cluster=EmbeddedCassandraServerHelper.getCluster(); + //cluster = new Cluster.Builder().addContactPoint(cassaHost).withPort(9142).build(); cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis(20000); assertNotNull("Invalid configuration for cassandra", cluster); - session = cluster.connect(); + session = EmbeddedCassandraServerHelper.getSession(); assertNotNull("Invalid configuration for cassandra", session); Class.forName("org.mariadb.jdbc.Driver"); MusicDataStoreHandle.mDstoreHandle = new MusicDataStore(cluster, session); @@ -105,9 +106,13 @@ public class OwnershipAndCheckpointTest { @AfterClass public static void close() throws MusicServiceException, MusicQueryException, ManagedProcessException { //TODO: shutdown cassandra - session.close(); - cluster.close(); + musicMixin=null; db.stop(); + try { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra(); + } + catch(NullPointerException e){ + } } private void dropTable() throws SQLException { @@ -149,7 +154,8 @@ public class OwnershipAndCheckpointTest { final DatabasePartition partition = TestUtils.createBasicRow(range, musicMixin, mdbcServerName); String sqlOperation = "INSERT INTO "+TABLE+" (PersonID,LastName,FirstName,Address,City) VALUES "+ "(1,'SAUREZ','ENRIQUE','GATECH','ATLANTA');"; - HashMap<Range, StagingTable> stagingTable = new HashMap<>(); + StagingTable stagingTable = new StagingTable(); + musicMixin.reloadAlreadyApplied(partition); final Statement executeStatement = this.conn.createStatement(); executeStatement.execute(sqlOperation); this.conn.commit(); @@ -164,6 +170,7 @@ public class OwnershipAndCheckpointTest { private OwnershipReturn cleanAndOwnPartition(List<Range> ranges, UUID ownOpId) throws SQLException { dropAndCreateTable(); + musicMixin.cleanAlreadyApplied(); DatabasePartition currentPartition = new DatabasePartition(MDBCUtils.generateTimebasedUniqueKey()); OwnershipReturn own=null; @@ -196,9 +203,10 @@ public class OwnershipAndCheckpointTest { } @Test - @Ignore + //@Ignore public void checkpoint() throws MDBCServiceException, SQLException { - Range range = new Range(TABLE); + Range range = + new Range(TABLE); OwnershipAndCheckpoint ownAndCheck = musicMixin.getOwnAndCheck(); initDatabase(range); @@ -219,7 +227,7 @@ public class OwnershipAndCheckpointTest { } @Test - @Ignore + //@Ignore public void warmup() throws MDBCServiceException, SQLException { Range range = new Range(TABLE); OwnershipAndCheckpoint ownAndCheck = musicMixin.getOwnAndCheck(); |