diff options
| author |   Tal Gitelman <tg851x@intl.att.com> | 2017-06-29 19:30:00 +0300 |
|---|---|---|
| committer |   Michael Lando <ml636r@att.com> | 2017-06-29 21:32:19 +0300 |
| commit | ed7e1c3dfe332abc67ed943717db2ee94406f95e (patch) | |
| tree | e0f2f39596656456272900f59aadc3dff6c6e828 /openecomp-be | |
| parent | 68ccc45de18f41cddb79de33a245bceb3b063ffb (diff) | |
[SDC] rebase code
Change-Id: I456ec65a233d277e6bae35e140f2e3da5765bae6
Signed-off-by: Tal Gitelman <tg851x@intl.att.com>
Signed-off-by: Michael Lando <ml636r@att.com>
Diffstat (limited to 'openecomp-be')
31 files changed, 1334 insertions, 820 deletions
diff --git a/openecomp-be/lib/openecomp-migration-lib/pom.xml b/openecomp-be/lib/openecomp-migration-lib/pom.xml deleted file mode 100644 index 7c33315fbc..0000000000 --- a/openecomp-be/lib/openecomp-migration-lib/pom.xml +++ /dev/null @@ -1,192 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <artifactId>openecomp-migration-lib</artifactId> - <name>openecomp-migration-lib</name> - - <parent> - <groupId>org.openecomp.sdc</groupId> - <artifactId>openecomp-sdc-lib</artifactId> - <version>1.1.0-SNAPSHOT</version> - <relativePath>..</relativePath> - </parent> - - <properties> - <groovy.version>2.4.7</groovy.version> - <janino.version>2.7.7</janino.version> - </properties> - - <dependencies> - <dependency> - <groupId>org.openecomp.sdc</groupId> - <artifactId>openecomp-sdc-vendor-software-product-manager</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy</artifactId> - <version>${groovy.version}</version> - </dependency> - - <dependency> - <groupId>org.openecomp.sdc.common</groupId> - <artifactId>openecomp-configuration-management-core</artifactId> - <version>${openecomp.sdc.common.version}</version> - <scope>runtime</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.codehaus.janino</groupId> - <artifactId>janino</artifactId> - <version>${janino.version}</version> - </dependency> - - <dependency> - <groupId>org.openecomp.sdc.core</groupId> - <artifactId>openecomp-zusammen-core</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.codehaus.janino</groupId> - <artifactId>commons-compiler</artifactId> - <version>3.0.6</version> - </dependency> - - </dependencies> - - <build> - <plugins> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>copy-dependencies</id> - <phase>install</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - <configuration> - <outputDirectory>${project.build.directory}/lib</outputDirectory> - <overWriteReleases>false</overWriteReleases> - <overWriteSnapshots>false</overWriteSnapshots> - <overWriteIfNewer>true</overWriteIfNewer> - <!--includeScope>runtime</includeScope--> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>${mvn.jar.version}</version> - <executions> - <execution> - <phase>package</phase> - </execution> - </executions> - <configuration> - <archive> - <manifest> - <addClasspath>true</addClasspath> - <classpathLayoutType>custom</classpathLayoutType> - <customClasspathLayout>lib/${artifact.artifactId}-${baseVersion}.${artifact.extension}</customClasspathLayout> - <mainClass>org.openecomp.sdc.migration.ToscaNamespaceMigration</mainClass> - </manifest> - <manifestEntries> - <Class-Path>lib/</Class-Path> - </manifestEntries> - </archive> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.19.1</version> - <configuration> - <useSystemClassLoader>false</useSystemClassLoader> - <redirectTestOutputToFile>true</redirectTestOutputToFile> - </configuration> - </plugin> - - <plugin> - <groupId>com.google.code.maven-replacer-plugin</groupId> - <artifactId>maven-replacer-plugin</artifactId> - <version>1.3.5</version> - <executions> - <execution> - <id>replace-for-migration</id> - <phase>initialize</phase> - <goals> - <goal>replace</goal> - </goals> - <configuration> - <file>../../tools/migration/1702_to_1707.sh</file> - <preserveDir>false</preserveDir> - <outputDir>target</outputDir> - <replacements> - <replacement> - <token>1707.0.0-SNAPSHOT</token> - <value>${project.version}</value> - </replacement> - </replacements> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>ant-test</id> - <phase>package</phase> - <configuration> - <tasks> - <fixcrlf srcdir="target" eol="unix" includes="1702_to_1707.sh"/> - </tasks> - </configuration> - <goals> - <goal>run</goal> - </goals> - </execution> - </executions> - </plugin> - <!--<plugin> - <artifactId>maven-assembly-plugin</artifactId> - <version>${mvn.assembly.version}</version> - <executions> - <execution> - <id>Generate assembly</id> - <phase>install</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - <configuration> - <descriptors> - <descriptor>${basedir}/src/main/assembly/migration-lib-assembly.xml</descriptor> - </descriptors> - <finalName>openecomp-migration-lib</finalName> - </configuration> - </plugin>--> - - </plugins> - </build> -</project> diff --git a/openecomp-be/lib/openecomp-migration-lib/src/main/assembly/migration-lib-assembly.xml b/openecomp-be/lib/openecomp-migration-lib/src/main/assembly/migration-lib-assembly.xml deleted file mode 100644 index 3cb9fc07b4..0000000000 --- a/openecomp-be/lib/openecomp-migration-lib/src/main/assembly/migration-lib-assembly.xml +++ /dev/null @@ -1,26 +0,0 @@ -<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd"> - <formats> - <format>zip</format> - </formats> - - <includeBaseDirectory>false</includeBaseDirectory> - - <fileSets> - <fileSet> - <directory>${basedir}/target</directory> - <includes> - <include>openecomp-migration-lib*.jar</include> - <include>1702_to_1707.sh</include> - </includes> - <outputDirectory>/</outputDirectory> - <fileMode>0755</fileMode> - </fileSet> - <fileSet> - <directory>${basedir}/target/lib</directory> - <outputDirectory>/lib</outputDirectory> - </fileSet> - </fileSets> -</assembly> - diff --git a/openecomp-be/lib/openecomp-migration-lib/src/main/java/org/openecomp/sdc/migration/TestScript.java b/openecomp-be/lib/openecomp-migration-lib/src/main/java/org/openecomp/sdc/migration/TestScript.java deleted file mode 100644 index c6a23734e4..0000000000 --- a/openecomp-be/lib/openecomp-migration-lib/src/main/java/org/openecomp/sdc/migration/TestScript.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.openecomp.sdc.migration; - -/** - * Created by TALIO on 3/19/2017. - */ -public class TestScript { - - public static void main(String[] args){ - System.out.print("Test succeed!"); - } -} diff --git a/openecomp-be/lib/openecomp-migration-lib/src/main/java/org/openecomp/sdc/migration/ToscaNamespaceMigration.java b/openecomp-be/lib/openecomp-migration-lib/src/main/java/org/openecomp/sdc/migration/ToscaNamespaceMigration.java deleted file mode 100644 index 4fa0ab368a..0000000000 --- a/openecomp-be/lib/openecomp-migration-lib/src/main/java/org/openecomp/sdc/migration/ToscaNamespaceMigration.java +++ /dev/null @@ -1,290 +0,0 @@ -package org.openecomp.sdc.migration; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.IOUtils; -import org.openecomp.core.model.dao.EnrichedServiceModelDao; -import org.openecomp.core.model.dao.EnrichedServiceModelDaoFactory; -import org.openecomp.core.model.dao.ServiceModelDao; -import org.openecomp.core.model.dao.ServiceModelDaoFactory; -import org.openecomp.core.model.types.ServiceElement; -import org.openecomp.core.utilities.json.JsonUtil; -import org.openecomp.sdc.logging.api.Logger; -import org.openecomp.sdc.logging.api.LoggerFactory; -import org.openecomp.sdc.tosca.datatypes.ToscaServiceModel; -import org.openecomp.sdc.tosca.datatypes.model.Import; -import org.openecomp.sdc.tosca.datatypes.model.Old1610ServiceTemplate; -import org.openecomp.sdc.tosca.datatypes.model.ServiceTemplate; -import org.openecomp.sdc.tosca.services.yamlutil.ToscaExtensionYamlUtil; -import org.openecomp.sdc.vendorsoftwareproduct.OrchestrationTemplateCandidateManager; -import org.openecomp.sdc.vendorsoftwareproduct.OrchestrationTemplateCandidateManagerFactory; -import org.openecomp.sdc.vendorsoftwareproduct.VendorSoftwareProductConstants; -import org.openecomp.sdc.vendorsoftwareproduct.VendorSoftwareProductManager; -import org.openecomp.sdc.vendorsoftwareproduct.VspManagerFactory; -import org.openecomp.sdc.vendorsoftwareproduct.dao.ComponentDao; -import org.openecomp.sdc.vendorsoftwareproduct.dao.ComponentDaoFactory; -import org.openecomp.sdc.vendorsoftwareproduct.dao.PackageInfoDao; -import org.openecomp.sdc.vendorsoftwareproduct.dao.PackageInfoDaoFactory; -import org.openecomp.sdc.vendorsoftwareproduct.dao.VendorSoftwareProductDao; -import org.openecomp.sdc.vendorsoftwareproduct.dao.VendorSoftwareProductDaoFactory; -import org.openecomp.sdc.vendorsoftwareproduct.dao.VendorSoftwareProductInfoDao; -import org.openecomp.sdc.vendorsoftwareproduct.dao.VendorSoftwareProductInfoDaoFactory; -import org.openecomp.sdc.vendorsoftwareproduct.dao.type.ComponentEntity; -import org.openecomp.sdc.vendorsoftwareproduct.dao.type.PackageInfo; -import org.openecomp.sdc.vendorsoftwareproduct.dao.type.VspDetails; -import org.openecomp.sdc.vendorsoftwareproduct.types.composition.ComponentData; -import org.openecomp.sdc.vendorsoftwareproduct.types.composition.CompositionData; -import org.openecomp.sdc.versioning.dao.types.Version; -import org.openecomp.core.zusammen.impl.CassandraConnectionInitializer; - - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - - -/** - * Created by TALIO on 3/19/2017 - */ -public class ToscaNamespaceMigration { - - private static VendorSoftwareProductManager vendorSoftwareProductManager = - VspManagerFactory.getInstance().createInterface(); - private static OrchestrationTemplateCandidateManager orchestrationTemplateCandidateManager = - OrchestrationTemplateCandidateManagerFactory.getInstance().createInterface(); - private static VendorSoftwareProductDao vendorSoftwareProductDao = - VendorSoftwareProductDaoFactory.getInstance().createInterface(); - private static ServiceModelDao<ToscaServiceModel, ServiceElement> serviceModelDao = - ServiceModelDaoFactory.getInstance().createInterface(); - private static EnrichedServiceModelDao<ToscaServiceModel, ServiceElement> - enrichedServiceModelDao = - EnrichedServiceModelDaoFactory.getInstance().createInterface(); - private static VendorSoftwareProductInfoDao vspInfoDao = - VendorSoftwareProductInfoDaoFactory.getInstance().createInterface(); - private static PackageInfoDao packageInfoDao = PackageInfoDaoFactory.getInstance() - .createInterface(); - private static final ComponentDao componentDao = - ComponentDaoFactory.getInstance().createInterface(); - private static Logger logger = LoggerFactory.getLogger(ToscaNamespaceMigration.class); - private static int status = 0; - - - public static void main(String[] args) { - //CassandraConnectionInitializer.setCassandraConnectionPropertiesToSystem(); - - Collection<VspDetails> vspList = vspInfoDao.list(new VspDetails()); - - List<PackageInfo> packagesList = - packageInfoDao.listByCategory(null, null); - - for (VspDetails vspInfo : vspList) { - printMessage("Performing migration on vsp " + vspInfo.getName() + " and version " + vspInfo - .getVersion().toString() + "\n"); - performMigration(vspInfo); - } - - System.exit(status); - } - - private static void performMigration(VspDetails vspDetails) { - - changeComponentNamePrefix(vspDetails); - - try { - changeNamespaceInServiceTemplates(vspDetails); - } catch (Exception e) { - printMessage( - "Could not perform migration for service templates on vsp " + vspDetails.getName()); - status = -1; - } - - if (vspDetails.getVersion().isFinal()) { - changeNamespaceInPackage(vspDetails); - } - - } - - private static void changeComponentNamePrefix(VspDetails vspDetails){ - Collection<ComponentEntity> componentsList = - componentDao.list(new ComponentEntity(vspDetails.getId(), vspDetails.getVersion(), null)); - - if(CollectionUtils.isEmpty(componentsList)){ - printMessage("No component namespace migration was performed on vsp with id" + vspDetails - .getId() + " and version " + vspDetails.getVersion().toString() + " since it has no " + - "components"); - return; - } - - for(ComponentEntity component : componentsList){ - String compositionData = component.getCompositionData(); - - if(Objects.isNull(compositionData)){ - continue; - } - - ComponentData componentData = - JsonUtil.json2Object(compositionData, ComponentData.class); - componentData.setName(componentData.getName().replace("com.att.d2", "org.openecomp")); - component.setCompositionData(JsonUtil.object2Json(componentData)); - - componentDao.update(component); - } - - printMessage("Component namespace migration was performed on vsp with id" + vspDetails - .getId() + " and version " + vspDetails.getVersion().toString()); - } - - private static void changeNamespaceInServiceTemplates(VspDetails vspDetails) throws IOException { - String vspId = vspDetails.getId(); - Version version = vspDetails.getVersion(); - ToscaServiceModel serviceModel; - ToscaServiceModel enrichedServiceModel; - - serviceModel = - serviceModelDao.getServiceModel(vspId, version); - enrichedServiceModel = - enrichedServiceModelDao.getServiceModel(vspId, version); - - printMessage("Working on vsp_service_template table in DB \n"); - changeNamespaceInServiceModel(serviceModel); - printMessage("Finished Working on vsp_service_template table in DB \n"); - - printMessage("Working on vsp_enriched_service_template table in DB \n"); - changeNamespaceInServiceModel(enrichedServiceModel); - printMessage("Finished Working on vsp_enriched_service_template table in DB \n"); - - serviceModelDao.storeServiceModel(vspId, version, serviceModel); - enrichedServiceModelDao.storeServiceModel(vspId, version, enrichedServiceModel); - } - - private static void changeNamespaceInServiceModel(ToscaServiceModel serviceModel) { - Map<String, ServiceTemplate> changedServiceTemplates = new HashMap<>(); - Map<String, ServiceTemplate> serviceTemplates = serviceModel.getServiceTemplates(); - - for (Map.Entry<String, ServiceTemplate> serviceTemplateEntry : serviceTemplates.entrySet()) { - printMessage( - "Changing namespace for Service Template " + serviceTemplateEntry.getKey() + "\n"); - - ServiceTemplate serviceTemplate = serviceTemplateEntry.getValue(); - String fileAsJson = JsonUtil.object2Json(serviceTemplate); - String replacedNamespace = fileAsJson.replace("org.openecomp.d2", "org.openecomp"); - ServiceTemplate newServiceTemplate; - try { - newServiceTemplate = - new ToscaExtensionYamlUtil().yamlToObject(replacedNamespace, ServiceTemplate.class); - } catch (Exception e) { - System.out.println("Found vsp with old-versioned tosca service template"); - Old1610ServiceTemplate oldServiceTemplate = - JsonUtil.json2Object(replacedNamespace, Old1610ServiceTemplate.class); - newServiceTemplate = mapOldSTToCurrentST(oldServiceTemplate); - - } - changedServiceTemplates.put( - serviceTemplateEntry.getKey(), newServiceTemplate); - } - - serviceModel.setServiceTemplates(changedServiceTemplates); - } - - private static org.openecomp.sdc.tosca.datatypes.model.ServiceTemplate mapOldSTToCurrentST( - Old1610ServiceTemplate oldServiceTemplate) { - org.openecomp.sdc.tosca.datatypes.model.ServiceTemplate - serviceTemplate = new org.openecomp.sdc.tosca.datatypes.model.ServiceTemplate(); - - serviceTemplate.setArtifact_types(oldServiceTemplate.getArtifact_types()); - serviceTemplate.setCapability_types(oldServiceTemplate.getCapability_types()); - serviceTemplate.setData_types(oldServiceTemplate.getData_types()); - serviceTemplate.setDescription(oldServiceTemplate.getDescription()); - serviceTemplate.setGroup_types(oldServiceTemplate.getGroup_types()); - serviceTemplate.setInterface_types(oldServiceTemplate.getInterface_types()); - serviceTemplate.setMetadata(oldServiceTemplate.getMetadata()); - serviceTemplate.setNode_types(oldServiceTemplate.getNode_types()); - serviceTemplate.setPolicy_types(oldServiceTemplate.getPolicy_types()); - serviceTemplate.setRelationship_types(oldServiceTemplate.getRelationship_types()); - serviceTemplate.setTopology_template(oldServiceTemplate.getTopology_template()); - - List<Map<String, Import>> imports = new ArrayList<>(); - for (Map.Entry<String, Import> importEntry : oldServiceTemplate.getImports().entrySet()) { - Map<String, Import> importMap = new HashMap<>(); - importMap.put(importEntry.getKey(), importEntry.getValue()); - imports.add(importMap); - } - serviceTemplate.setImports(imports); - - return serviceTemplate; - - } - - private static void changeNamespaceInPackage(VspDetails vspDetails) { - String vspId = vspDetails.getId(); - printMessage("Start updating CSAR file with new namespace in DB \n"); - - File translatedFile; - try { - translatedFile = - getTranslatedFile(vspId, vspDetails.getVersion()); - } catch (Exception e) { - printMessage("No translated file was found under vsp " + vspDetails.getName() + "\n"); - return; - } - try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ZipOutputStream zos = new ZipOutputStream(baos); - ZipFile zipFile = new ZipFile(translatedFile)) { - Enumeration<? extends ZipEntry> entries = zipFile.entries(); - - while (entries.hasMoreElements()) { - ZipEntry ze = entries.nextElement(); - InputStream zipEntryIs = zipFile.getInputStream(ze); - byte[] contentAsByte = IOUtils.toByteArray(zipEntryIs); - String fileContent = new String(contentAsByte); - - String replacedNamespace = fileContent.replace("org.openecomp.d2", "org.openecomp"); - - zos.putNextEntry(new ZipEntry(ze.getName())); - zos.write(replacedNamespace.getBytes()); - } - printMessage("Changed Tosca namesapce in package for vsp " + vspDetails.getName() + "\n"); - - - packageInfoDao.updateTranslatedContent( - vspId, vspDetails.getVersion(), ByteBuffer.wrap(baos.toByteArray())); - printMessage("Updated CSAR file with new namespace in DB \n"); - - } catch (Exception e) { - printMessage("Could not perform migration on csar"); - } - } - - private static void printMessage(String message) { - System.out.print(message); - logger.debug(message); - } - - private static File getTranslatedFile(String vspId, Version version) - throws IOException { - PackageInfo packageInfo = - packageInfoDao.get(new PackageInfo(vspId, version)); - ByteBuffer translatedFileBuffer = packageInfo == null ? null : packageInfo.getTranslatedFile(); - - File translatedFile = new File(VendorSoftwareProductConstants.VSP_PACKAGE_ZIP); - - FileOutputStream fos = new FileOutputStream(translatedFile); - fos.write(translatedFileBuffer.array()); - fos.close(); - - - return translatedFile; - } -} diff --git a/openecomp-be/lib/openecomp-migration-lib/src/main/resources/logback.xml b/openecomp-be/lib/openecomp-migration-lib/src/main/resources/logback.xml deleted file mode 100644 index 5596881329..0000000000 --- a/openecomp-be/lib/openecomp-migration-lib/src/main/resources/logback.xml +++ /dev/null @@ -1,280 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<configuration scan="true" scanPeriod="5 seconds"> - - <property scope="system" name="OPENECOMP-component-name" value="ASDC" /> - <property scope="system" name="OPENECOMP-subcomponent-name" value="ASDC-BE" /> - <property file="${config.home}/catalog-be/configuration.yaml" /> - <property scope="context" name="enable-all-log" value="false" /> - - <!--statusListener class="ch.qos.logback.core.status.OnConsoleStatusListener" /> - <include resource="asdc_debug_logback.xml"/--> - - <!-- value used by pattern field list (| - is inter-field separator, || - unavailable or not applicable field value) (m - mandatory, o- optional)--> - <!--timestamp(m)| requestID(m)| serviceInstanceID(o)| threadID(m)| physicalServerName(o)| serviceName(m)| userID(m)| logLevel(m)| severity(o)| serverIpAddress(m)| serverName(m)| clientIpAddress(o)| className(m)| timer(o)| detailedMessage(o)--> - <property name="default-log-pattern" - value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{uuid}|%X{serviceInstanceID}|%thread||${OPENECOMP-subcomponent-name}|%X{userId}|%level|%X{alarmSeverity}|%X{localAddr}|${beFqdn}|%X{remoteAddr}|%logger{35}|%X{timer}|ActivityType=<%M>, Desc=<%msg>%n" /> - - <property name="asdc-debug-log-pattern" value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%msg %n"/> - - <!-- All log --> - <if condition='property("enable-all-log").equalsIgnoreCase("true")'> - <then> - <appender name="ALL_ROLLING" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/all.log - </file> - - <rollingPolicy - class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/all.log.%i - </fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>10</maxIndex> - </rollingPolicy> - - <triggeringPolicy - class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>20MB</maxFileSize> - </triggeringPolicy> - <encoder> - <pattern>${default-log-pattern}</pattern> - </encoder> - </appender> - - <appender name="ASYNC_ALL" class="ch.qos.logback.classic.AsyncAppender"> - <appender-ref ref="ALL_ROLLING" /> - </appender> - </then> - </if> - - <!-- Error log --> - <appender name="ERROR_ROLLING" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/Error.log - </file> - - <!-- Audit messages filter - deny audit messages --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.OnMarkerEvaluator"> - <marker>AUDIT_MARKER</marker> - </evaluator> - <onMismatch>NEUTRAL</onMismatch> - <onMatch>DENY</onMatch> - </filter> - - <!-- Transaction messages filter - deny Transaction messages --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.OnMarkerEvaluator"> - <marker>TRANSACTION_MARKER</marker> - </evaluator> - <onMismatch>NEUTRAL</onMismatch> - <onMatch>DENY</onMatch> - </filter> - - <!-- deny all events with a level below INFO, that is TRACE and DEBUG --> - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>INFO</level> - </filter> - - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/Error.log.%i - </fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>10</maxIndex> - </rollingPolicy> - - <triggeringPolicy - class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>20MB</maxFileSize> - </triggeringPolicy> - <encoder> - <pattern>${default-log-pattern}</pattern> - </encoder> - </appender> - - <!-- Debug log --> - <appender name="DEBUG_ROLLING" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/debug.log - </file> - - <!-- No need to deny audit messages - they are INFO only, will be denied - anyway --> - <!-- Transaction messages filter - deny Transaction messages, there are - some DEBUG level messages among them --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.OnMarkerEvaluator"> - <marker>TRANSACTION_MARKER</marker> - </evaluator> - <onMismatch>NEUTRAL</onMismatch> - <onMatch>DENY</onMatch> - </filter> - - <!-- accept DEBUG and TRACE level --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.GEventEvaluator"> - <expression> - e.level.toInt() <= DEBUG.toInt() - </expression> - </evaluator> - <OnMismatch>DENY</OnMismatch> - <OnMatch>NEUTRAL</OnMatch> - </filter> - - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/debug.log.%i - </fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>10</maxIndex> - </rollingPolicy> - - <triggeringPolicy - class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>20MB</maxFileSize> - </triggeringPolicy> - <encoder> - <pattern>${default-log-pattern}</pattern> - </encoder> - </appender> - - <!-- Debug log --> - <appender name="PACKAGE_DEBUG_ROLLING" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/migration_debug.log - </file> - - <!-- No need to deny audit messages - they are INFO only, will be denied - anyway --> - <!-- Transaction messages filter - deny Transaction messages, there are - some DEBUG level messages among them --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.OnMarkerEvaluator"> - <marker>TRANSACTION_MARKER</marker> - </evaluator> - <onMismatch>NEUTRAL</onMismatch> - <onMatch>DENY</onMatch> - </filter> - - <!-- accept DEBUG and TRACE level --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.GEventEvaluator"> - <expression> - e.level.toInt() <= DEBUG.toInt() - </expression> - </evaluator> - <OnMismatch>DENY</OnMismatch> - <OnMatch>NEUTRAL</OnMatch> - </filter> - - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/debug_by_package.log.%i - </fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>10</maxIndex> - </rollingPolicy> - - <triggeringPolicy - class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>20MB</maxFileSize> - </triggeringPolicy> - <encoder> - <pattern>${asdc-debug-log-pattern}</pattern> - </encoder> - </appender> - - <!-- Audit log --> - <appender name="AUDIT_ROLLING" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - - <file>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/audit.log - </file> - - <!-- Audit messages filter - accept audit messages --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.OnMarkerEvaluator"> - <marker>AUDIT_MARKER</marker> - </evaluator> - <onMismatch>DENY</onMismatch> - <onMatch>ACCEPT</onMatch> - </filter> - - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/audit.log.%i - </fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>10</maxIndex> - </rollingPolicy> - - <triggeringPolicy - class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>20MB</maxFileSize> - </triggeringPolicy> - <encoder> - <pattern>${default-log-pattern}</pattern> - </encoder> - </appender> - - <!-- SdncTransaction log --> - <appender name="TRANSACTION_ROLLING" - class="ch.qos.logback.core.rolling.RollingFileAppender"> - - <file>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/transaction.log - </file> - - <!-- Transaction messages filter - accept audit messages --> - <filter class="ch.qos.logback.core.filter.EvaluatorFilter"> - <evaluator class="ch.qos.logback.classic.boolex.OnMarkerEvaluator"> - <marker>TRANSACTION_MARKER</marker> - </evaluator> - <onMismatch>DENY</onMismatch> - <onMatch>ACCEPT</onMatch> - </filter> - - <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> - <fileNamePattern>${log.home}/${OPENECOMP-component-name}/${OPENECOMP-subcomponent-name}/transaction.log.%i - </fileNamePattern> - <minIndex>1</minIndex> - <maxIndex>10</maxIndex> - </rollingPolicy> - - <triggeringPolicy - class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> - <maxFileSize>20MB</maxFileSize> - </triggeringPolicy> - <encoder> - <pattern>${default-log-pattern}</pattern> - </encoder> - </appender> - - <!-- Asynchronicity Configurations --> - <appender name="ASYNC_DEBUG" class="ch.qos.logback.classic.AsyncAppender"> - <appender-ref ref="DEBUG_ROLLING" /> - </appender> - - <appender name="ASYNC_TRANSACTION" class="ch.qos.logback.classic.AsyncAppender"> - <appender-ref ref="TRANSACTION_ROLLING" /> - </appender> - - <appender name="ASYNC_ERROR" class="ch.qos.logback.classic.AsyncAppender"> - <appender-ref ref="ERROR_ROLLING" /> - </appender> - - - <root level="INFO"> - <appender-ref ref="ASYNC_ERROR" /> - <appender-ref ref="ASYNC_DEBUG" /> - <appender-ref ref="AUDIT_ROLLING" /> - <appender-ref ref="ASYNC_TRANSACTION" /> - <if condition='property("enable-all-log").equalsIgnoreCase("true")'> - <then> - <appender-ref ref="ALL_ROLLING" /> - </then> - </if> - </root> - - <logger name="org.openecomp.sdc" level="INFO" /> - - <logger name="org.openecomp.sdc.migration" level="DEBUG" additivity ="false"> - <appender-ref ref="PACKAGE_DEBUG_ROLLING" /> - </logger> -</configuration> diff --git a/openecomp-be/lib/openecomp-migration-lib/src/main/resources/openecomp-migration-lib.zip b/openecomp-be/lib/openecomp-migration-lib/src/main/resources/openecomp-migration-lib.zip Binary files differdeleted file mode 100644 index 659a2381f7..0000000000 --- a/openecomp-be/lib/openecomp-migration-lib/src/main/resources/openecomp-migration-lib.zip +++ /dev/null diff --git a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/pom.xml b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/pom.xml index 0ae2578969..9a80aa9c24 100644 --- a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/pom.xml +++ b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/pom.xml @@ -78,7 +78,7 @@ <configuration> <useSystemClassLoader>false</useSystemClassLoader> <redirectTestOutputToFile>true</redirectTestOutputToFile> - <skipTests>true</skipTests> + <skipTests>true</skipTests> </configuration> </plugin> </plugins> diff --git a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-impl/pom.xml b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-impl/pom.xml index c183fd4263..e8c39065a9 100644 --- a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-impl/pom.xml +++ b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-impl/pom.xml @@ -56,10 +56,10 @@ <version>${mvn.surefire.version}</version> <configuration> <useSystemClassLoader>false</useSystemClassLoader> - <skipTests>true</skipTests> <systemPropertyVariables> <config.location>${project.basedir}/configuration</config.location> </systemPropertyVariables> + <skipTests>true</skipTests> </configuration> </plugin> </plugins> diff --git a/openecomp-be/lib/pom.xml b/openecomp-be/lib/pom.xml index 183f4187c1..44594f7488 100644 --- a/openecomp-be/lib/pom.xml +++ b/openecomp-be/lib/pom.xml @@ -29,7 +29,6 @@ <module>openecomp-sdc-enrichment-lib</module> <module>openecomp-logging-lib</module> <module>openecomp-healing-lib</module> - <module>openecomp-migration-lib</module> <module>openecomp-sdc-activity-log-lib</module> </modules> </project> diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ExportDataCommand.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ExportDataCommand.java new file mode 100644 index 0000000000..b92ba52000 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ExportDataCommand.java @@ -0,0 +1,62 @@ +package org.openecomp.core.tools.Commands; + +import com.amdocs.zusammen.datatypes.SessionContext; +import org.apache.commons.io.FileUtils; +import org.openecomp.core.tools.Commands.exportdata.ElementHandler; +import org.openecomp.core.tools.Commands.exportdata.ImportProperties; +import org.openecomp.core.tools.Commands.exportdata.ItemHandler; +import org.openecomp.core.tools.Commands.exportdata.VersionHandler; +import org.openecomp.core.tools.util.ZipUtils; +import org.openecomp.core.zusammen.impl.CassandraConnectionInitializer; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +import static java.nio.file.Files.createDirectories; +public class ExportDataCommand { + private static final Logger logger = LoggerFactory.getLogger(ExportDataCommand.class); + + public static void exportData(SessionContext context, String filterItem) { + try { + ImportProperties.initParams(); + CassandraConnectionInitializer.setCassandraConnectionPropertiesToSystem(); + Path rootDir = Paths.get(ImportProperties.ROOT_DIRECTORY); + initDir(rootDir); + if (filterItem != null) { + filterItem = filterItem.replaceAll("\\r", ""); + } + new ItemHandler().createItemsData(context, filterItem); + new VersionHandler().loadVersions(filterItem); + new ElementHandler().loadElements(filterItem); + zipPath(rootDir,filterItem); + FileUtils.forceDelete(rootDir.toFile()); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + ex.printStackTrace(); + } + + } + private static void zipPath(Path rootDir,String filterItem ) throws Exception{ + LocalDateTime date = LocalDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + String dateStr = date.format(formatter); + String zipFile = System.getProperty("user.home")+ File.separatorChar+"onboarding_import"+ dateStr + ".zip"; + ZipUtils.createZip(zipFile, rootDir,filterItem); + } + + + public static void initDir(Path rootDir ) throws IOException{ + if (Files.exists(rootDir)) { + FileUtils.forceDelete(rootDir.toFile()); + } + createDirectories(rootDir); + } + +}
\ No newline at end of file diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ImportCommand.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ImportCommand.java new file mode 100644 index 0000000000..57486b81e3 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ImportCommand.java @@ -0,0 +1,43 @@ +package org.openecomp.core.tools.Commands; + +import com.amdocs.zusammen.datatypes.SessionContext; +import org.apache.commons.io.FileUtils; +import org.openecomp.core.tools.Commands.importdata.TreeWalker; +import org.openecomp.core.tools.Commands.exportdata.ImportProperties; +import org.openecomp.core.tools.util.ZipUtils; +import org.openecomp.core.zusammen.impl.CassandraConnectionInitializer; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; + +public class ImportCommand { + + private static final Logger logger = LoggerFactory.getLogger(ImportCommand.class); + + + public static void importData(SessionContext context, String zippedFile, String filterItem) { + try { + ImportProperties.initParams(); + CassandraConnectionInitializer.setCassandraConnectionPropertiesToSystem(); + if (zippedFile == null){ + logger.error("Import must have a valid file as an input."); + } + zippedFile = zippedFile.replaceAll("\\r", ""); + if(filterItem != null) { + filterItem = filterItem.replaceAll("\\r", ""); + } + Path rootDir = Paths.get(ImportProperties.ROOT_DIRECTORY); + ExportDataCommand.initDir(rootDir); + ZipUtils.unzip(Paths.get(zippedFile), rootDir); + TreeWalker.walkFiles(context, rootDir, filterItem); + + FileUtils.forceDelete(rootDir.toFile()); // clear all unzip data at the end. + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + ex.printStackTrace(); + } + } + +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ResetOldVersion.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ResetOldVersion.java index 185e80030f..527efa5fc6 100644 --- a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ResetOldVersion.java +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/ResetOldVersion.java @@ -2,22 +2,16 @@ package org.openecomp.core.tools.Commands; import com.amdocs.zusammen.datatypes.Id; import com.amdocs.zusammen.datatypes.SessionContext; -import com.amdocs.zusammen.plugin.statestore.cassandra.dao.impl.ItemCassandraDao; import com.amdocs.zusammen.plugin.statestore.cassandra.dao.impl.VersionCassandraDao; -import org.openecomp.core.tools.loaders.VersionInfoCassandraLoader; -import org.openecomp.core.tools.loaders.zusammen.VspGeneralLoader; +import org.openecomp.core.tools.store.VersionInfoCassandraLoader; +import org.openecomp.core.tools.store.VspGeneralLoader; import org.openecomp.core.tools.store.ElementHandler; import org.openecomp.core.zusammen.impl.CassandraConnectionInitializer; -import org.openecomp.core.zusammen.plugin.dao.impl.CassandraElementRepository; import org.openecomp.core.zusammen.plugin.dao.types.ElementEntity; import org.openecomp.sdc.vendorsoftwareproduct.VendorSoftwareProductConstants; import org.openecomp.sdc.versioning.dao.types.VersionInfoEntity; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class ResetOldVersion { diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ElementHandler.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ElementHandler.java new file mode 100644 index 0000000000..ebce90fb4a --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ElementHandler.java @@ -0,0 +1,100 @@ +package org.openecomp.core.tools.Commands.exportdata; + + +import org.openecomp.core.tools.store.ElementCassandraLoader; +import org.openecomp.core.tools.store.zusammen.datatypes.ElementEntity; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +import static java.io.File.separator; +import static java.nio.file.Files.*; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.*; + +public class ElementHandler { + + private static final Logger logger = LoggerFactory.getLogger(ElementHandler.class); + + public ElementHandler() { + } + + public void loadElements(String filteredItem) { + ElementCassandraLoader elementCassandraLoader = new ElementCassandraLoader(); + elementCassandraLoader.list().forEach(elementEntity -> handleElementEntity(elementEntity,filteredItem)); + } + + private void handleElementEntity(ElementEntity elementEntity, String filteredItem) { + try { + String itemId = elementEntity.getItemId(); + if (filteredItem != null && !itemId.contains(filteredItem)){ + return; + } + String versionId = elementEntity.getVersionId(); + String space = elementEntity.getSpace(); + String namespace = elementEntity.getNamespace(); + String elementId = elementEntity.getElement_id(); + + String namespacePath = separator; + if (!isNull(namespace)){ + namespacePath = namespace.replace(ELEMENT_NAMESPACE_SPLITTER,separator)+separator; + } + Path elementDirectoryPath = Paths.get( ROOT_DIRECTORY + separator + itemId + + separator + versionId + separator + space + separator + namespacePath+ separator + elementId); + if (notExists(elementDirectoryPath)) { + createDirectories(elementDirectoryPath); + } + + String info = elementEntity.getInfo(); + if (!isNull(info)) { + Path infoFilePath = Paths.get(elementDirectoryPath.toString() + separator + ELEMENT_INFO_PREFIX + + elementId + JSON_POSTFIX); + write(infoFilePath, info.getBytes()); + } + + String relations = elementEntity.getRelations(); + if (!isNull(relations)) { + Path realtionsFilePath = Paths.get(elementDirectoryPath.toString() + separator + + ELEMENT_RELATION_PREFIX + elementId + JSON_POSTFIX); + write(realtionsFilePath, relations.getBytes()); + } + + ByteBuffer data = elementEntity.getData(); + if (!Objects.isNull(data)) { + Path dataFilePath = Paths.get(elementDirectoryPath.toString() + separator + + ELEMENT_DATA_PREFIX + elementId + JSON_POSTFIX); + write(dataFilePath, data.array()); + } + + ByteBuffer visualization = elementEntity.getVisualization(); + if (!Objects.isNull(visualization)) { + Path visualFilePath = Paths.get(elementDirectoryPath.toString() + separator + + ELEMENT_VISUALIZATION_PREFIX + elementId ); + write(visualFilePath, visualization.array()); + } + + ByteBuffer searchableData = elementEntity.getSearchableData(); + if (!Objects.isNull(searchableData)) { + Path searchableFilePath = Paths.get(elementDirectoryPath.toString() + separator + + ELEMENT_SEARCHABLE_PREFIX + elementId); + write(searchableFilePath, searchableData.array()); + } + + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + ex.printStackTrace(); + } + + } + + private boolean isNull(String inStr){ + if (Objects.isNull(inStr)){ + return true; + } + return inStr.trim().equalsIgnoreCase("null"); + } + +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ImportProperties.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ImportProperties.java new file mode 100644 index 0000000000..3b7566fd1b --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ImportProperties.java @@ -0,0 +1,20 @@ +package org.openecomp.core.tools.Commands.exportdata; + +import java.io.File; + +public class ImportProperties { + public static final String VERSION_FILE_PREFIX = "version_"; + public static final String VERSION_INFO_FILE_PREFIX = "version_info_"; + public static final String JSON_POSTFIX = ".json"; + + public static final String ELEMENT_INFO_PREFIX = "elem_info"; + public static final String ELEMENT_RELATION_PREFIX = "elem_relations"; + public static final String ELEMENT_DATA_PREFIX = "elem_data"; + public static final String ELEMENT_VISUALIZATION_PREFIX = "elem_visualization"; + public static final String ELEMENT_SEARCHABLE_PREFIX = "elem_searchableData"; + public static final String ELEMENT_NAMESPACE_SPLITTER = "/"; + public static String ROOT_DIRECTORY; + public static final void initParams(){ + ROOT_DIRECTORY = System.getProperty("user.home")+File.separator+ "onboarding_import"; + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ItemHandler.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ItemHandler.java new file mode 100644 index 0000000000..9eea182609 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/ItemHandler.java @@ -0,0 +1,62 @@ +package org.openecomp.core.tools.Commands.exportdata; + +import com.amdocs.zusammen.datatypes.SessionContext; +import com.amdocs.zusammen.datatypes.item.Item; +import com.amdocs.zusammen.plugin.statestore.cassandra.dao.ItemDao; +import com.amdocs.zusammen.plugin.statestore.cassandra.dao.ItemDaoFactory; +import org.openecomp.core.tools.Commands.ExportDataCommand; +import org.openecomp.core.utilities.json.JsonUtil; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import static java.nio.file.Files.*; +import static java.io.File.separator; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.*; + +public class ItemHandler { + private static final Logger logger = LoggerFactory.getLogger(ExportDataCommand.class); + + + public ItemHandler() { + ImportProperties.initParams(); + } + + public void createItemsData(SessionContext context, String filteredItem) throws URISyntaxException, IOException { + + List<Item> items = getItemDao(context).list(context); + items.parallelStream().forEach(item -> createItemDirectoryAndFiles(item,filteredItem)); + + } + + private final void createItemDirectoryAndFiles(Item item,String filteredItem) { + try { + String itemId = item.getId().getValue(); + if (filteredItem != null && !itemId.contains(filteredItem)){ + return; + } + Path itemPath = Paths.get( ImportProperties.ROOT_DIRECTORY + separator + itemId); + Path itemFilePath = Paths.get( ImportProperties.ROOT_DIRECTORY + separator + + itemId + separator + itemId + JSON_POSTFIX); + if (notExists(itemPath)) { + createDirectories(itemPath); + } + String itemJson = JsonUtil.object2Json(item); + write(itemFilePath, itemJson.getBytes()); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + ex.printStackTrace(); + } + + } + + + private ItemDao getItemDao(SessionContext context) { + return ItemDaoFactory.getInstance().createInterface(context); + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/VersionHandler.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/VersionHandler.java new file mode 100644 index 0000000000..b8d82958c9 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/exportdata/VersionHandler.java @@ -0,0 +1,74 @@ +package org.openecomp.core.tools.Commands.exportdata; + +import org.openecomp.core.tools.Commands.ExportDataCommand; +import org.openecomp.core.tools.store.VersionInfoCassandraLoader; +import org.openecomp.core.tools.store.VersionCassandraLoader; + +import org.openecomp.core.tools.store.zusammen.datatypes.VersionEntity; +import org.openecomp.core.utilities.json.JsonUtil; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; +import org.openecomp.sdc.versioning.dao.types.VersionInfoEntity; + +import static java.io.File.separator; +import static java.nio.file.Files.*; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.*; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.ROOT_DIRECTORY; + +import java.nio.file.Path; +import java.nio.file.Paths; + +public class VersionHandler { + private static final Logger logger = LoggerFactory.getLogger(ExportDataCommand.class); + + public VersionHandler() { + } + + public void loadVersions(String filteredItem) { + VersionCassandraLoader versionCassandraLoader = new VersionCassandraLoader(); + versionCassandraLoader.list().forEach(versionEntity -> handleVersionEntity(versionEntity,filteredItem)); + VersionInfoCassandraLoader versionInfoCassandraLoader = new VersionInfoCassandraLoader(); + versionInfoCassandraLoader.list().forEach(versionInfoEntity -> handleVersionInfo(versionInfoEntity,filteredItem)); + } + + private void handleVersionEntity(VersionEntity versionEntity, String filteredItem) { + try { + String itemId = versionEntity.getItemId(); + if (filteredItem != null && !itemId.contains(filteredItem)){ + return; + } + String versionId = versionEntity.getVersionId(); + String space = versionEntity.getSpace(); + Path versionDirectoryPath = Paths.get( ROOT_DIRECTORY + separator + itemId + + separator + versionId + separator + space); + Path versionFilePath = Paths.get(versionDirectoryPath.toString() + separator + VERSION_FILE_PREFIX + + versionId + JSON_POSTFIX); + if (notExists(versionDirectoryPath)) { + createDirectories(versionDirectoryPath); + } + String versionJson = JsonUtil.object2Json(versionEntity); + write(versionFilePath, versionJson.getBytes()); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + ex.printStackTrace(); + } + + } + + private void handleVersionInfo(VersionInfoEntity versionInfoEntity, String filteredItem) { + try { + String itemId = versionInfoEntity.getEntityId(); + Path itemDirectory = Paths.get( ROOT_DIRECTORY + separator + itemId); + Path versionInfoFilePath = Paths.get(itemDirectory.toString() + separator + VERSION_INFO_FILE_PREFIX + + itemId + JSON_POSTFIX); + if (exists(itemDirectory)) { + String versionJson = JsonUtil.object2Json(versionInfoEntity); + write(versionInfoFilePath, versionJson.getBytes()); + } + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + ex.printStackTrace(); + } + + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/ElementImport.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/ElementImport.java new file mode 100644 index 0000000000..7ba830906c --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/ElementImport.java @@ -0,0 +1,116 @@ +package org.openecomp.core.tools.Commands.importdata; + +import com.amdocs.zusammen.datatypes.SessionContext; +import org.openecomp.core.tools.store.ElementCassandraLoader; +import org.openecomp.core.tools.store.ElementNamespaceHandler; +import org.openecomp.core.tools.store.VersionCassandraLoader; +import org.openecomp.core.tools.store.zusammen.datatypes.ElementEntity; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.io.File.separator; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.*; + +public class ElementImport { + private static final Logger logger = LoggerFactory.getLogger(ElementImport.class); + private ElementCassandraLoader elementCassandraLoader = new ElementCassandraLoader(); + private ElementNamespaceHandler cassandraElementRepository = new ElementNamespaceHandler(); + private VersionCassandraLoader versionCassandraLoader = new VersionCassandraLoader(); + + public void loadPath(SessionContext sessionContext, Path elementDir, String elementId, String[] pathObjects) { + try { + // load info file + ElementEntity elementEntity = new ElementEntity(); + Path infoFilePath = Paths.get(elementDir.toString() + separator + ELEMENT_INFO_PREFIX + + elementId + JSON_POSTFIX); + if (Files.exists(infoFilePath)) { + String info = new String(Files.readAllBytes(infoFilePath)); + elementEntity.setInfo(info); + } + + // load relation file + Path realtionsFilePath = Paths.get(elementDir.toString() + separator + + ELEMENT_RELATION_PREFIX + elementId + JSON_POSTFIX); + if (Files.exists(realtionsFilePath)) { + String relations = new String(Files.readAllBytes(realtionsFilePath)); + elementEntity.setRelations(relations); + } + + //load entity data + Path dataFilePath = Paths.get(elementDir.toString() + separator + + ELEMENT_DATA_PREFIX + elementId + JSON_POSTFIX); + if (Files.exists(dataFilePath)) { + byte[] bytes = Files.readAllBytes(dataFilePath); + ByteBuffer data = ByteBuffer.wrap(bytes); + elementEntity.setData(data); + } + + //load visualization + Path visualFilePath = Paths.get(elementDir.toString() + separator + + ELEMENT_VISUALIZATION_PREFIX + elementId ); + if (Files.exists(visualFilePath)) { + byte[] bytes = Files.readAllBytes(visualFilePath); + ByteBuffer visualization = ByteBuffer.wrap(bytes); + elementEntity.setVisualization(visualization); + } + + //load searchable + Path searchableFilePath = Paths.get(elementDir.toString() + separator + + ELEMENT_SEARCHABLE_PREFIX + elementId ); + if (Files.exists(searchableFilePath)) { + byte[] bytes = Files.readAllBytes(searchableFilePath); + ByteBuffer searchable = ByteBuffer.wrap(bytes); + elementEntity.setSearchableData(searchable); + } + + elementEntity.setSpace(pathObjects[2]); + elementEntity.setItemId(pathObjects[0]); + elementEntity.setVersionId(pathObjects[1]); + elementEntity.setElement_id(pathObjects[pathObjects.length - 1]); + elementEntity.setNamespace(getNameSpace(pathObjects)); + elementEntity.setParentId(getParentId(pathObjects)); + elementEntity.setSubElementIds(getAllSubElementsIds(elementDir)); + elementCassandraLoader.createEntity(elementEntity); + cassandraElementRepository.createElementNamespace(elementEntity); + versionCassandraLoader.insertElementToVersion(elementEntity); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + } + } + + private String getParentId(String[] pathObjects) { + if (pathObjects.length <= 4) { + return null; + } + return pathObjects[pathObjects.length - 2]; + } + + private Set<String> getAllSubElementsIds(Path root) throws IOException { + try (Stream<Path> walk = Files.walk(root)) { + return walk.filter(path -> Files.isDirectory(path)) + .map(path -> path.toFile().getName() ).collect(Collectors.toSet()); + } + } + + private String getNameSpace(String[] pathObjects) { + if (pathObjects.length <= 4) { + return null; + } + if (pathObjects.length == 5) { + return pathObjects[3]; + } + return Arrays.stream(pathObjects, 3, pathObjects.length - 1) + .reduce("", (s1, s2) -> s1 + File.separator + s2); + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/ItemImport.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/ItemImport.java new file mode 100644 index 0000000000..c7042743c2 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/ItemImport.java @@ -0,0 +1,42 @@ +package org.openecomp.core.tools.Commands.importdata; + +import com.amdocs.zusammen.datatypes.SessionContext; +import com.amdocs.zusammen.datatypes.item.Item; +import com.amdocs.zusammen.plugin.statestore.cassandra.dao.ItemDao; +import com.amdocs.zusammen.plugin.statestore.cassandra.dao.ItemDaoFactory; +import org.openecomp.core.utilities.json.JsonUtil; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.JSON_POSTFIX; + +public class ItemImport { + private static final Logger logger = LoggerFactory.getLogger(ItemImport.class); + + public void loadPath(SessionContext sessionContext, Path itemDir, String itemName) { + try { + Path itemPath = Paths.get(itemDir.toString() + File.separator + itemName + JSON_POSTFIX); + if (!Files.exists(itemPath)) { + return; + } + String itemJson = new String(Files.readAllBytes(itemPath)); + if (itemJson == null || itemJson.trim().isEmpty()) { + return; + } + Item item = JsonUtil.json2Object(itemJson, Item.class); + ItemDao itemDao = getItemDao(sessionContext); + itemDao.create(sessionContext, item.getId(), item.getInfo(), item.getCreationTime()); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + } + } + + private ItemDao getItemDao(SessionContext context) { + return ItemDaoFactory.getInstance().createInterface(context); + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/TreeWalker.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/TreeWalker.java new file mode 100644 index 0000000000..020b2f3c47 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/TreeWalker.java @@ -0,0 +1,51 @@ +package org.openecomp.core.tools.Commands.importdata; + +import com.amdocs.zusammen.datatypes.SessionContext; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +public class TreeWalker { + private static final Logger logger = LoggerFactory.getLogger(TreeWalker.class); + + public static final void walkFiles(SessionContext sessionContext,Path rootDir, String filterItem) throws IOException { + try (Stream<Path> walk = Files.walk(rootDir)) { + walk.parallel().filter(path -> Files.isDirectory(path)). + forEach(path -> handlePath(sessionContext,path, rootDir, filterItem)); + } + } + + private static final void handlePath(SessionContext sessionContext, Path path, Path root,String filterItem) { + String logicalPath = path.toString().replace(root.toString()+File.separator, ""); + String[] splitted = logicalPath.split(File.separator); + if(filterItem != null && splitted.length > 0 && !splitted[0].contains(filterItem)){ + return; + } + switch (splitted.length) { + case 0: + //root - ignore + break; + case 1: // handle Item + new ItemImport().loadPath(sessionContext,path,splitted[splitted.length -1]); + new VersionInfoImport().loadPath(sessionContext,path,splitted[splitted.length -1]); + break; + case 2: + //ignore this level + break; + case 3: // handle version + new VersionImport().loadPath(sessionContext,path,splitted[splitted.length -2]); + break; + default: + //handle elements + new ElementImport().loadPath(sessionContext,path,splitted[splitted.length -1],splitted); + break; + } + + } + +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/VersionImport.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/VersionImport.java new file mode 100644 index 0000000000..8f6c67d212 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/VersionImport.java @@ -0,0 +1,41 @@ +package org.openecomp.core.tools.Commands.importdata; + +import com.amdocs.zusammen.datatypes.SessionContext; +import org.openecomp.core.tools.store.VersionCassandraLoader; +import org.openecomp.core.tools.store.zusammen.datatypes.VersionEntity; +import org.openecomp.core.utilities.json.JsonUtil; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static java.io.File.separator; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.JSON_POSTFIX; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.VERSION_FILE_PREFIX; + +public class VersionImport { + private static final Logger logger = LoggerFactory.getLogger(VersionImport.class); + + public void loadPath(SessionContext sessionContext, Path versionDir , String versionId){ + try { + Path versionPath = Paths.get(versionDir.toString() + separator + VERSION_FILE_PREFIX + + versionId + JSON_POSTFIX); + if (!Files.exists(versionPath)) { + return; + } + String versionJson = new String(Files.readAllBytes(versionPath)); + if (versionJson == null || versionJson.trim().isEmpty()) { + return; + } + VersionEntity versionEntity = JsonUtil.json2Object(versionJson, VersionEntity.class); + VersionCassandraLoader versionCassandraLoader = new VersionCassandraLoader(); + versionCassandraLoader.insertVersion(versionEntity); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + } + } + + +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/VersionInfoImport.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/VersionInfoImport.java new file mode 100644 index 0000000000..5da7407c9f --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/Commands/importdata/VersionInfoImport.java @@ -0,0 +1,41 @@ +package org.openecomp.core.tools.Commands.importdata; + +import com.amdocs.zusammen.datatypes.SessionContext; +import org.openecomp.core.tools.store.VersionInfoCassandraLoader; +import org.openecomp.core.utilities.json.JsonUtil; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; +import org.openecomp.sdc.versioning.dao.types.VersionInfoEntity; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static java.io.File.separator; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.JSON_POSTFIX; +import static org.openecomp.core.tools.Commands.exportdata.ImportProperties.VERSION_INFO_FILE_PREFIX; + +public class VersionInfoImport { + private static final Logger logger = LoggerFactory.getLogger(VersionInfoImport.class); + + public void loadPath(SessionContext sessionContext, Path itemPath, String itemId) { + try { + Path versionInfoFilePath = Paths.get(itemPath.toString() + separator + VERSION_INFO_FILE_PREFIX + + itemId + JSON_POSTFIX); + if (!Files.exists(versionInfoFilePath)) { + return; + } + String versionInfoJson = new String(Files.readAllBytes(versionInfoFilePath)); + if (versionInfoJson == null || versionInfoJson.trim().isEmpty()) { + return; + } + VersionInfoEntity versionInfoEntity = JsonUtil.json2Object(versionInfoJson, VersionInfoEntity.class); + VersionInfoCassandraLoader versionInfoCassandraLoader = new VersionInfoCassandraLoader(); + versionInfoCassandraLoader.insertVersionInfo(versionInfoEntity); + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + } + } + + +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/main/ZusammenMainTool.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/main/ZusammenMainTool.java index 7bc4bb23fb..926566864f 100644 --- a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/main/ZusammenMainTool.java +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/main/ZusammenMainTool.java @@ -2,6 +2,8 @@ package org.openecomp.core.tools.main; import com.amdocs.zusammen.datatypes.SessionContext; import com.amdocs.zusammen.datatypes.UserInfo; +import org.openecomp.core.tools.Commands.ImportCommand; +import org.openecomp.core.tools.Commands.ExportDataCommand; import org.openecomp.core.tools.Commands.ResetOldVersion; import org.openecomp.core.tools.util.ToolsUtil; import org.openecomp.sdc.logging.api.Logger; @@ -23,8 +25,8 @@ public class ZusammenMainTool { String command = ToolsUtil.getParam("c",args); if(command == null){ printMessage(logger, - "parameter -c is mandatory. script usage: zusammenMainTool.sh -c {command name} " + - "[additional arguments depending on the command] "); + "parameter -c is mandatory. script usage: zusammenMainTool.sh -c {command name} " + + "[additional arguments depending on the command] "); System.exit(-1); } Instant startTime = Instant.now(); @@ -39,8 +41,10 @@ public class ZusammenMainTool { ResetOldVersion.reset(context,ToolsUtil.getParam("v",args)); break; case EXPORT: + ExportDataCommand.exportData(context,ToolsUtil.getParam("i",args)); break; case IMPORT: + ImportCommand.importData(context, ToolsUtil.getParam("f",args),ToolsUtil.getParam("i",args)); break; } @@ -52,9 +56,9 @@ public class ZusammenMainTool { printMessage(logger, - "Zusammen tools command:[] finished . Total run time was : " + minutesPart + ":" + - secondsPart - + " minutes"); + "Zusammen tools command:[] finished . Total run time was : " + minutesPart + ":" + + secondsPart + + " minutes"); System.exit(status); } diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/ElementCassandraLoader.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/ElementCassandraLoader.java new file mode 100644 index 0000000000..1c7e185e18 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/ElementCassandraLoader.java @@ -0,0 +1,74 @@ +/*- + * ============LICENSE_START======================================================= + * SDC + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.core.tools.store; + +import com.datastax.driver.mapping.Mapper; +import com.datastax.driver.mapping.Result; +import com.datastax.driver.mapping.annotations.Accessor; +import com.datastax.driver.mapping.annotations.Query; +import com.datastax.driver.mapping.annotations.QueryParameters; +import org.openecomp.core.nosqldb.api.NoSqlDb; +import org.openecomp.core.nosqldb.factory.NoSqlDbFactory; +import org.openecomp.core.tools.store.zusammen.datatypes.ElementEntity; + +import java.nio.ByteBuffer; +import java.util.Set; + +public class ElementCassandraLoader { + + private static NoSqlDb noSqlDb = NoSqlDbFactory.getInstance().createInterface(); + private static Mapper<ElementEntity> mapper = noSqlDb.getMappingManager().mapper(ElementEntity.class); + private static ElementAccessor accessor = noSqlDb.getMappingManager().createAccessor(ElementAccessor.class); + private String[] columns = {"space", "item_id", "version_id", "element_id", "data", "info", "namespace", "parent_id", + "relations", "searchable_data", "sub_element_ids"}; + + + public void createEntity(ElementEntity elementEntity) { + accessor.insertElement(elementEntity.getSpace(), + elementEntity.getItemId(), + elementEntity.getVersionId(), + elementEntity.getElement_id(), + elementEntity.getData(), + elementEntity.getInfo(), + elementEntity.getNamespace(), + elementEntity.getParentId(), + elementEntity.getRelations(), + elementEntity.getSearchableData(), + elementEntity.getSubElementIds()); + } + + public Result<ElementEntity> list() { + return accessor.getAll(); + } + + @Accessor + interface ElementAccessor { + + @Query("insert into zusammen_dox.element (space,item_id,version_id,element_id,data,info,namespace,parent_id,relations,searchable_data,sub_element_ids) values (?,?,?,?,?,?,?,?,?,?,?)") + void insertElement(String space, String itemId, String versionId, String elementId, ByteBuffer data, String info, String namespaceStr, + String parentId, String relations, ByteBuffer searchable, Set<String> subElementsIds); + + + @Query("select * from zusammen_dox.element ") + @QueryParameters(fetchSize = 100) + Result<ElementEntity> getAll(); + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/ElementNamespaceHandler.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/ElementNamespaceHandler.java new file mode 100644 index 0000000000..f1ed970b4d --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/ElementNamespaceHandler.java @@ -0,0 +1,27 @@ +package org.openecomp.core.tools.store; + + +import com.datastax.driver.mapping.annotations.Accessor; +import com.datastax.driver.mapping.annotations.Param; +import com.datastax.driver.mapping.annotations.Query; +import org.openecomp.core.nosqldb.api.NoSqlDb; +import org.openecomp.core.nosqldb.factory.NoSqlDbFactory; +import org.openecomp.core.tools.store.zusammen.datatypes.ElementEntity; + +public class ElementNamespaceHandler { + + private static NoSqlDb nnoSqlDb = NoSqlDbFactory.getInstance().createInterface(); + private static ElementNamespaceAccessor accessor = nnoSqlDb.getMappingManager().createAccessor(ElementNamespaceAccessor.class); + + public void createElementNamespace(ElementEntity elementEntity) { + accessor.create(elementEntity.getSpace(),elementEntity.getItemId(),elementEntity.getElement_id(),elementEntity.getNamespace()); + } + + @Accessor + interface ElementNamespaceAccessor { + @Query("UPDATE zusammen_dox.element_namespace SET namespace=:ns WHERE space=:space AND item_id=:item AND element_id=:id ") + void create(@Param("space") String space, @Param("item") String item, @Param("id") String id, @Param("ns") String ns); + } + + +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VersionCassandraLoader.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VersionCassandraLoader.java new file mode 100644 index 0000000000..851e92dcca --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VersionCassandraLoader.java @@ -0,0 +1,78 @@ +/*- + * ============LICENSE_START======================================================= + * SDC + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.core.tools.store; + +import com.datastax.driver.mapping.Mapper; +import com.datastax.driver.mapping.Result; +import com.datastax.driver.mapping.annotations.Accessor; +import com.datastax.driver.mapping.annotations.Query; +import com.datastax.driver.mapping.annotations.QueryParameters; +import com.google.common.collect.Sets; +import org.openecomp.core.nosqldb.api.NoSqlDb; +import org.openecomp.core.nosqldb.factory.NoSqlDbFactory; +import org.openecomp.core.tools.store.zusammen.datatypes.ElementEntity; +import org.openecomp.core.tools.store.zusammen.datatypes.VersionEntity; + +import java.util.Date; +import java.util.Set; + +public class VersionCassandraLoader { + + private static NoSqlDb noSqlDb = NoSqlDbFactory.getInstance().createInterface(); + private static Mapper<VersionEntity> mapper = noSqlDb.getMappingManager().mapper(VersionEntity.class); + private static VersionAccessor accessor = noSqlDb.getMappingManager().createAccessor(VersionAccessor.class); + + public void insertElementToVersion(ElementEntity elementEntity) { + accessor.addElements(Sets.newHashSet(elementEntity.getElement_id()), elementEntity.getSpace(), elementEntity.getItemId(), elementEntity.getVersionId()); + } + + public void insertVersion(VersionEntity versionEntity) { + accessor.insertVersion( versionEntity.getSpace(), + versionEntity.getItemId(), + versionEntity.getVersionId(), + versionEntity.getBaseVersionId(), + versionEntity.getCreationTime(), + versionEntity.getInfo(), + versionEntity.getModificationTime(), + versionEntity.getRelations()); + } + + + public Result<VersionEntity> list() { + return accessor.getAll(); + } + + @Accessor + interface VersionAccessor { + + @Query("UPDATE zusammen_dox.version_elements SET element_ids=element_ids+? " + + "WHERE space=? AND item_id=? AND version_id=?") + void addElements(Set<String> elementIds, String space, String itemId, String versionId); + + @Query("insert into zusammen_dox.version (space,item_id,version_id,base_version_id,creation_time,info,modification_time,relations) values (?,?,?,?,?,?,?,?)") + void insertVersion(String space, String itemId, String versionId, String baseVersionId, Date createTime, String info, Date modificationTime, String relations); + + + @Query("select * from zusammen_dox.version ") + @QueryParameters(fetchSize = 400) + Result<VersionEntity> getAll(); + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VersionInfoCassandraLoader.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VersionInfoCassandraLoader.java new file mode 100644 index 0000000000..d7ec93962c --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VersionInfoCassandraLoader.java @@ -0,0 +1,62 @@ +/*- + * ============LICENSE_START======================================================= + * SDC + * ================================================================================ + * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.core.tools.store; + +import com.datastax.driver.mapping.Mapper; +import com.datastax.driver.mapping.Result; +import com.datastax.driver.mapping.annotations.Accessor; +import com.datastax.driver.mapping.annotations.Query; +import com.datastax.driver.mapping.annotations.QueryParameters; +import org.openecomp.core.nosqldb.api.NoSqlDb; +import org.openecomp.core.nosqldb.factory.NoSqlDbFactory; +import org.openecomp.sdc.versioning.dao.VersionInfoDao; +import org.openecomp.sdc.versioning.dao.VersionInfoDaoFactory; +import org.openecomp.sdc.versioning.dao.types.VersionInfoEntity; + +import java.util.Collection; + +public class VersionInfoCassandraLoader { + + private static NoSqlDb noSqlDb = NoSqlDbFactory.getInstance().createInterface(); + private static Mapper<VersionInfoEntity> mapper = + noSqlDb.getMappingManager().mapper(VersionInfoEntity.class); + private static VersionInfoAccessor accessor = + noSqlDb.getMappingManager().createAccessor(VersionInfoAccessor.class); + private static VersionInfoDao versionInfoDao = + VersionInfoDaoFactory.getInstance().createInterface(); + + public void insertVersionInfo(VersionInfoEntity versionInfoEntity) { + versionInfoDao.create(versionInfoEntity); + } + + public Collection<VersionInfoEntity> list() { + return accessor.getAll().all(); + } + + @Accessor + interface VersionInfoAccessor { + @Query("select * from dox.version_info ") + @QueryParameters(fetchSize = 400) + Result<VersionInfoEntity> getAll(); + } + + +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VspGeneralLoader.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VspGeneralLoader.java new file mode 100644 index 0000000000..54a3dd9ada --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/VspGeneralLoader.java @@ -0,0 +1,40 @@ +package org.openecomp.core.tools.store; + +import com.amdocs.zusammen.datatypes.Id; +import com.amdocs.zusammen.datatypes.SessionContext; +import com.amdocs.zusammen.plugin.statestore.cassandra.dao.types.ElementEntityContext; +import org.openecomp.core.zusammen.plugin.dao.impl.CassandraElementRepository; +import org.openecomp.core.zusammen.plugin.dao.types.ElementEntity; +import org.openecomp.sdc.vendorsoftwareproduct.dao.impl.zusammen.StructureElement; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class VspGeneralLoader { + public static Map<String, ElementEntity> load(SessionContext context, + Map<String, List<String>> vspItemVersionsMap) { + Map<String, ElementEntity> elementEntityMap = new HashMap<>(); + System.setProperty("cassandra.dox.keystore", "zusammen_dox"); + CassandraElementRepository cassandraElementRepository = new CassandraElementRepository(); + for (Map.Entry<String, List<String>> entry : vspItemVersionsMap.entrySet()) { + + for (String version : entry.getValue()) { + + Optional<ElementEntity> result = + cassandraElementRepository.get(context, new ElementEntityContext( + context.getUser().getUserName(), + new Id(entry.getKey()), + new Id(version)), + new ElementEntity(new Id(StructureElement.General.name()))); + if (result.isPresent()) { + elementEntityMap.put(context.getUser().getUserName() + "_" + entry.getKey() + + "_" + version, result.get()); + } + } + } + + return elementEntityMap; + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/zusammen/datatypes/ElementEntity.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/zusammen/datatypes/ElementEntity.java new file mode 100644 index 0000000000..f4f450db70 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/zusammen/datatypes/ElementEntity.java @@ -0,0 +1,168 @@ +package org.openecomp.core.tools.store.zusammen.datatypes; + +import com.datastax.driver.mapping.annotations.Column; +import com.datastax.driver.mapping.annotations.PartitionKey; +import com.datastax.driver.mapping.annotations.Table; + +import java.nio.ByteBuffer; +import java.util.Set; + +/** + * CREATE TABLE zusammen_dox.element ( + * space text, + * item_id text, + * version_id text, + * element_id text, + * data blob, + * info text, + * namespace text, + * parent_id text, + * relations text, + * searchable_data blob, + * sub_element_ids set<text>, + * visualization blob, + * PRIMARY KEY ((space, item_id, version_id, element_id)) + * ) + */ +@Table( + keyspace = "zusammen_dox", + name = "version" +) +public class ElementEntity { + @Column( name = "space" ) + @PartitionKey(0) + private String space; + + @Column( name = "item_id" ) + @PartitionKey(1) + private String itemId; + + @Column( name = "version_id" ) + @PartitionKey(2) + private String versionId; + + @Column(name = "element_id") + @PartitionKey(3) + private String element_id; + + @Column(name = "data") + private ByteBuffer data; + + @Column(name = "info") + private String info; + + @Column(name = "namespace") + private String namespace; + + @Column(name = "parent_id") + private String parentId; + + @Column(name = "relations") + private String relations; + + @Column(name = "searchable_data") + private ByteBuffer searchableData; + + @Column(name = "sub_element_ids") + private Set<String> subElementIds; + + + @Column(name = "visualization") + private ByteBuffer visualization; + + public String getSpace() { + return space; + } + + public void setSpace(String space) { + this.space = space; + } + + public String getItemId() { + return itemId; + } + + public void setItemId(String itemId) { + this.itemId = itemId; + } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + + public String getElement_id() { + return element_id; + } + + public void setElement_id(String element_id) { + this.element_id = element_id; + } + + public ByteBuffer getData() { + return data; + } + + public void setData(ByteBuffer data) { + this.data = data; + } + + public String getInfo() { + return info; + } + + public void setInfo(String info) { + this.info = info; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getParentId() { + return parentId; + } + + public void setParentId(String parentId) { + this.parentId = parentId; + } + + public String getRelations() { + return relations; + } + + public void setRelations(String relations) { + this.relations = relations; + } + + public ByteBuffer getSearchableData() { + return searchableData; + } + + public void setSearchableData(ByteBuffer searchableData) { + this.searchableData = searchableData; + } + + public Set<String> getSubElementIds() { + return subElementIds; + } + + public void setSubElementIds(Set<String> subElementIds) { + this.subElementIds = subElementIds; + } + + public ByteBuffer getVisualization() { + return visualization; + } + + public void setVisualization(ByteBuffer visualization) { + this.visualization = visualization; + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/zusammen/datatypes/VersionEntity.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/zusammen/datatypes/VersionEntity.java new file mode 100644 index 0000000000..c7280c8a9d --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/store/zusammen/datatypes/VersionEntity.java @@ -0,0 +1,128 @@ +package org.openecomp.core.tools.store.zusammen.datatypes; + +import com.datastax.driver.mapping.annotations.*; + +import java.util.Date; + +/** + * CREATE TABLE zusammen_dox.version ( + * space text, + * item_id text, + * version_id text, + * base_version_id text, + * creation_time timestamp, + * info text, + * modification_time timestamp, + * relations text, + * PRIMARY KEY ((space, item_id), version_id) + * ) WITH CLUSTERING ORDER BY (version_id ASC) + * AND bloom_filter_fp_chance = 0.01 + * AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' + * AND comment = '' + * AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'} + * AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'} + * AND dclocal_read_repair_chance = 0.1 + * AND default_time_to_live = 0 + * AND gc_grace_seconds = 864000 + * AND max_index_interval = 2048 + * AND memtable_flush_period_in_ms = 0 + * AND min_index_interval = 128 + * AND read_repair_chance = 0.0 + * AND speculative_retry = '99.0PERCENTILE'; + */ +@Table( + keyspace = "zusammen_dox", + name = "version" +) +public class VersionEntity { + @Column(name = "space") + @PartitionKey(0) + private String space; + + @Column(name = "item_id") + @PartitionKey(1) + private String itemId; + + @Column(name = "version_id") + @PartitionKey(2) + private String versionId; + + @Column(name = "base_version_id") + private String baseVersionId; + + @Column(name = "creation_time") + private Date creationTime; + + @Column(name = "info") + private String info; + + @Column(name = "modification_time") + private Date modificationTime; + + @Column(name = "relations") + private String relations; + + public String getSpace() { + return space; + } + + public void setSpace(String space) { + this.space = space; + } + + public String getItemId() { + return itemId; + } + + public void setItemId(String itemId) { + this.itemId = itemId; + } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + + public String getBaseVersionId() { + return baseVersionId; + } + + public void setBaseVersionId(String baseVersionId) { + this.baseVersionId = baseVersionId; + } + + public Date getCreationTime() { + return creationTime; + } + + public void setCreationTime(Date creationTime) { + this.creationTime = creationTime; + } + + public String getInfo() { + return info; + } + + public void setInfo(String info) { + this.info = info; + } + + public Date getModificationTime() { + return modificationTime; + } + + public void setModificationTime(Date modificationTime) { + this.modificationTime = modificationTime; + } + + public String getRelations() { + return relations; + } + + public void setRelations(String relations) { + this.relations = relations; + } +} diff --git a/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/util/ZipUtils.java b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/util/ZipUtils.java new file mode 100644 index 0000000000..6447f85ea7 --- /dev/null +++ b/openecomp-be/tools/zusammen-tools/src/main/java/org/openecomp/core/tools/util/ZipUtils.java @@ -0,0 +1,89 @@ +package org.openecomp.core.tools.util; + +import com.google.common.io.ByteStreams; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Scanner; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +public class ZipUtils { + public static void createZip(String zipFileName, Path dir, String filterItem) throws Exception { + File dirObj = dir.toFile(); + try ( + FileOutputStream fileOutputStream = new FileOutputStream(zipFileName); + ZipOutputStream out = new ZipOutputStream(fileOutputStream)) { + addDir(dirObj, out, dir.toString(), filterItem); + } + } + + public static final String cleanStr(String inFilterStr) { + if (Objects.isNull(inFilterStr)) { + return inFilterStr; + } + Scanner scan = new Scanner(inFilterStr); + while (scan.hasNextLine()) { + inFilterStr = scan.nextLine().replaceAll("[^a-zA-Z0-9]", ""); + } + return inFilterStr; + } + + static void addDir(File dirObj, ZipOutputStream out, String root, String filterItem) throws IOException { + File[] files = dirObj.listFiles(); + filterItem = cleanStr(filterItem); + + for (int i = 0; i < files.length; i++) { + if (files[i].isDirectory()) { + addDir(files[i], out, root, filterItem); + continue; + } + try (FileInputStream in = new FileInputStream((files[i].getAbsolutePath()))) { + String filePath = files[i].getAbsolutePath().replace(root + File.separator, ""); + if (filterItem == null || filePath.contains(filterItem)) { + out.putNextEntry(new ZipEntry(filePath)); + try { + ByteStreams.copy(in, out); + + } finally { + out.closeEntry(); + } + } + + } + } + } + + public static void unzip(Path zipFile, Path outputFolder) throws IOException { + if (zipFile == null || outputFolder == null) { + return; + } + if (!Files.exists(outputFolder)) { + Files.createDirectories(outputFolder); + } + + try (FileInputStream fileInputStream = new FileInputStream(zipFile.toFile()); + ZipInputStream zis = new ZipInputStream(fileInputStream)) { + ZipEntry ze = zis.getNextEntry(); + while (ze != null) { + String fileName = ze.getName(); + File newFile = new File(outputFolder.toString() + File.separator + fileName); + new File(newFile.getParent()).mkdirs(); + try (FileOutputStream fos = new FileOutputStream(newFile)) { + ByteStreams.copy(zis, fos); + } + ze = zis.getNextEntry(); + } + + zis.closeEntry(); + } + + } +} + diff --git a/openecomp-be/tools/zusammen-tools/src/main/resources/zusammenMainTool.sh b/openecomp-be/tools/zusammen-tools/src/main/resources/zusammenMainTool.sh index c274d84424..da043b130d 100644 --- a/openecomp-be/tools/zusammen-tools/src/main/resources/zusammenMainTool.sh +++ b/openecomp-be/tools/zusammen-tools/src/main/resources/zusammenMainTool.sh @@ -10,7 +10,5 @@ # change exist package and service templates in db -java -Dlog.home=/apps/jetty/base/be/logs -Dconfiguration.yaml=/apps/jetty/base/be/config/catalog-be/configuration.yaml -jar openecomp-zusammen-tools-1.0-SNAPSHOT.jar org.openecomp.core.tools.main.ZusammenMainTool $1 $2 $3 $4 - -STATUS="${?}" echo "${STATUS}" echo "All log messages for the zusammenMainTool -migration process are in /apps/jetty/base/be/logs/ASDC/ASDC-BE/zusammen_tool_debug.log" +java -Dlog.home=/apps/jetty/base/be/logs -Dconfiguration.yaml=/apps/jetty/base/be/config/catalog-be/configuration.yaml -jar openecomp-zusammen-tools-1.0-SNAPSHOT.jar org.openecomp.core.tools.main.ZusammenMainTool $1 $2 $3 $4 $5 $6 +STATUS="${?}" echo "${STATUS}" echo "All log messages for the zusammenMainTool migration process are in /apps/jetty/base/be/logs/ASDC/ASDC-BE/zusammen_tool_debug.log" |