From c82aebcde26e34c4151531b4d7a8f6e7689734ba Mon Sep 17 00:00:00 2001 From: "andre.schmid" Date: Fri, 14 May 2021 20:38:45 +0100 Subject: Add models imports endpoint and persistence structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create the structure to persist the model imports. Changed create model API, allowing to create a model along its TOSCA descriptor import structure. Introduced an endpoint to update the imports of a model. Change-Id: Ic775ef544051c29c721cacc20b37c2fb20338be9 Issue-ID: SDC-3614 Signed-off-by: André Schmid --- .../org/openecomp/sdc/be/dao/api/ActionStatus.java | 2 +- .../api/exception/CassandraDaoInitException.java | 27 ++++++ .../CassandraDaoInitExceptionProvider.java | 36 +++++++ .../sdc/be/dao/cassandra/CassandraClient.java | 3 +- .../dao/cassandra/ToscaImportByModelAccessor.java | 33 +++++++ .../cassandra/ToscaModelImportCassandraDao.java | 106 +++++++++++++++++++++ .../be/dao/cassandra/schema/SdcSchemaBuilder.java | 24 +++-- .../sdc/be/dao/cassandra/schema/Table.java | 4 +- .../tables/ToscaImportByModelTableDescription.java | 78 +++++++++++++++ 9 files changed, 302 insertions(+), 11 deletions(-) create mode 100644 catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitException.java create mode 100644 catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitExceptionProvider.java create mode 100644 catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaImportByModelAccessor.java create mode 100644 catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaModelImportCassandraDao.java create mode 100644 catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/tables/ToscaImportByModelTableDescription.java (limited to 'catalog-dao/src/main/java') diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/ActionStatus.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/ActionStatus.java index 1ebaeab54f..ed448c18d4 100644 --- a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/ActionStatus.java +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/ActionStatus.java @@ -52,7 +52,7 @@ public enum ActionStatus { // Category related COMPONENT_MISSING_CATEGORY, COMPONENT_INVALID_CATEGORY, COMPONENT_ELEMENT_INVALID_NAME_FORMAT, COMPONENT_ELEMENT_INVALID_NAME_LENGTH, COMPONENT_CATEGORY_ALREADY_EXISTS, COMPONENT_CATEGORY_NOT_FOUND, COMPONENT_SUB_CATEGORY_NOT_FOUND_FOR_CATEGORY, COMPONENT_SUB_CATEGORY_EXISTS_FOR_CATEGORY, COMPONENT_GROUPING_EXISTS_FOR_SUB_CATEGORY, // Model related - MODEL_ALREADY_EXISTS, + MODEL_ALREADY_EXISTS, INVALID_MODEL, MODEL_IMPORTS_IS_EMPTY, COULD_NOT_READ_MODEL_IMPORTS, // Service API URL INVALID_SERVICE_API_URL, // Property related diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitException.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitException.java new file mode 100644 index 0000000000..b1ee5f0ae1 --- /dev/null +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitException.java @@ -0,0 +1,27 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.dao.api.exception; + +public class CassandraDaoInitException extends RuntimeException { + + public CassandraDaoInitException(final String message) { + super(message); + } +} diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitExceptionProvider.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitExceptionProvider.java new file mode 100644 index 0000000000..4fec8fbdd9 --- /dev/null +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/api/exception/CassandraDaoInitExceptionProvider.java @@ -0,0 +1,36 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.dao.api.exception; + +import java.util.function.Supplier; +import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus; + +public class CassandraDaoInitExceptionProvider { + + private CassandraDaoInitExceptionProvider() { + + } + + public static Supplier keySpaceConnectError(final String keyspace, final CassandraOperationStatus cassandraOperationStatus) { + var errorMsg = String.format("Could not connect to keyspace '%s'. Operation status was '%s'", keyspace, cassandraOperationStatus); + return () -> new CassandraDaoInitException(errorMsg); + } + +} diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java index 624f9b44f2..25a6b49bba 100644 --- a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/CassandraClient.java @@ -34,6 +34,7 @@ import java.util.List; import javax.annotation.PreDestroy; import org.apache.commons.lang3.tuple.ImmutablePair; import org.openecomp.sdc.be.config.ConfigurationManager; +import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode; import org.openecomp.sdc.common.log.wrappers.Logger; import org.springframework.stereotype.Component; @@ -155,7 +156,7 @@ public class CassandraClient { Mapper mapper = manager.mapper(clazz); mapper.save(entity); } catch (Exception e) { - logger.debug("Failed to save entity [{}], error :", entity, e); + logger.error(EcompLoggerErrorCode.DATA_ERROR, CassandraClient.class.getName(), "Failed to save entity [{}], error :", entity, e); return CassandraOperationStatus.GENERAL_ERROR; } return CassandraOperationStatus.OK; diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaImportByModelAccessor.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaImportByModelAccessor.java new file mode 100644 index 0000000000..eb54bd7fca --- /dev/null +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaImportByModelAccessor.java @@ -0,0 +1,33 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.dao.cassandra; + +import com.datastax.driver.mapping.Result; +import com.datastax.driver.mapping.annotations.Accessor; +import com.datastax.driver.mapping.annotations.Param; +import com.datastax.driver.mapping.annotations.Query; +import org.openecomp.sdc.be.data.model.ToscaImportByModel; + +@Accessor +public interface ToscaImportByModelAccessor { + + @Query("SELECT * FROM sdcartifact.tosca_import_by_model WHERE model_id = :modelId") + Result findAllByModel(@Param("modelId") String modelId); +} diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaModelImportCassandraDao.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaModelImportCassandraDao.java new file mode 100644 index 0000000000..a8b1ec635d --- /dev/null +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/ToscaModelImportCassandraDao.java @@ -0,0 +1,106 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.dao.cassandra; + +import static java.util.function.Predicate.not; + +import com.datastax.driver.core.Session; +import com.datastax.driver.mapping.Mapper; +import com.datastax.driver.mapping.MappingManager; +import fj.data.Either; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.PostConstruct; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.openecomp.sdc.be.dao.api.exception.CassandraDaoInitException; +import org.openecomp.sdc.be.dao.api.exception.CassandraDaoInitExceptionProvider; +import org.openecomp.sdc.be.data.model.ToscaImportByModel; +import org.openecomp.sdc.be.resources.data.auditing.AuditingTypesConstants; +import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode; +import org.openecomp.sdc.common.log.wrappers.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component("tosca-model-import-cassandra-dao") +public class ToscaModelImportCassandraDao extends CassandraDao { + + private static final Logger LOGGER = Logger.getLogger(ToscaModelImportCassandraDao.class.getName()); + + private ToscaImportByModelAccessor toscaImportByModelAccessor; + private Mapper toscaImportByModelMapper; + + @Autowired + public ToscaModelImportCassandraDao(final CassandraClient cassandraClient) { + super(cassandraClient); + } + + /** + * For test purposes. + * + * @param toscaImportByModelAccessor the sdcartifact.tosca_import_by_model accessor + */ + ToscaModelImportCassandraDao(final ToscaImportByModelAccessor toscaImportByModelAccessor, + final Mapper toscaImportByModelMapper) { + super(null); + this.toscaImportByModelAccessor = toscaImportByModelAccessor; + this.toscaImportByModelMapper = toscaImportByModelMapper; + } + + @PostConstruct + public void init() { + final var keyspace = AuditingTypesConstants.ARTIFACT_KEYSPACE; + if (!client.isConnected()) { + LOGGER.error(EcompLoggerErrorCode.SCHEMA_ERROR, ToscaModelImportCassandraDao.class.getName(), "Cassandra client isn't connected"); + return; + } + final Either, CassandraOperationStatus> connectionResult = client.connect(keyspace); + if (connectionResult.isRight()) { + final CassandraDaoInitException exception = + CassandraDaoInitExceptionProvider.keySpaceConnectError(keyspace, connectionResult.right().value()).get(); + LOGGER.error(EcompLoggerErrorCode.SCHEMA_ERROR, ToscaModelImportCassandraDao.class.getName(), exception.getMessage()); + throw exception; + } + session = connectionResult.left().value().getLeft(); + manager = connectionResult.left().value().getRight(); + toscaImportByModelMapper = manager.mapper(ToscaImportByModel.class); + toscaImportByModelAccessor = manager.createAccessor(ToscaImportByModelAccessor.class); + LOGGER.info("{} successfully initialized", ToscaModelImportCassandraDao.class.getName()); + } + + public void importAll(final String modelId, final List toscaImportByModelList) { + final List importOfModelList = toscaImportByModelList.stream() + .filter(toscaImportByModel -> modelId.equals(toscaImportByModel.getModelId())) + .collect(Collectors.toList()); + final List actualImportOfModelList = toscaImportByModelAccessor.findAllByModel(modelId).all(); + final List removedImportList = actualImportOfModelList.stream() + .filter(not(importOfModelList::contains)) + .collect(Collectors.toList()); + + importOfModelList.forEach(toscaImportByModelMapper::save); + removedImportList.forEach(toscaImportByModel -> + toscaImportByModelMapper.delete(toscaImportByModel.getModelId(), toscaImportByModel.getFullPath()) + ); + } + + public List findAllByModel(final String modelId) { + return toscaImportByModelAccessor.findAllByModel(modelId).all(); + } + +} diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/SdcSchemaBuilder.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/SdcSchemaBuilder.java index 0c6bb453ae..c0c12aa0b2 100644 --- a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/SdcSchemaBuilder.java +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/SdcSchemaBuilder.java @@ -31,10 +31,12 @@ import com.datastax.driver.core.schemabuilder.Alter; import com.datastax.driver.core.schemabuilder.Create; import com.datastax.driver.core.schemabuilder.SchemaBuilder; import com.datastax.driver.core.schemabuilder.SchemaStatement; +import com.datastax.oss.driver.shaded.guava.common.base.Function; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -112,17 +114,17 @@ public class SdcSchemaBuilder { * the method creats all the tables and indexes thet do not already exist * * @param iTableDescriptions: a list of table description we want to create - * @param keyspaceMetadate: the current tables that exist in the cassandra under this keyspace + * @param keyspaceMetadata: the current tables that exist in the cassandra under this keyspace * @param session: the session object used for the execution of the query. * @param existingTablesMetadata the current tables columns that exist in the cassandra under this keyspace */ - private static void createTables(List iTableDescriptions, Map> keyspaceMetadate, Session session, + private static void createTables(List iTableDescriptions, Map> keyspaceMetadata, Session session, Map> existingTablesMetadata) { for (ITableDescription tableDescription : iTableDescriptions) { String tableName = tableDescription.getTableName().toLowerCase(); Map> columnDescription = tableDescription.getColumnDescription(); log.info("creating tables:{}.", tableName); - if (keyspaceMetadate == null || !keyspaceMetadate.keySet().contains(tableName)) { + if (keyspaceMetadata == null || !keyspaceMetadata.containsKey(tableName)) { Create create = SchemaBuilder.createTable(tableDescription.getKeyspace(), tableDescription.getTableName()); for (ImmutablePair key : tableDescription.primaryKeys()) { create.addPartitionKey(key.getLeft(), key.getRight()); @@ -132,9 +134,15 @@ public class SdcSchemaBuilder { create.addClusteringColumn(key.getLeft(), key.getRight()); } } - for (Map.Entry> entry : columnDescription.entrySet()) { - create.addColumn(entry.getKey(), entry.getValue().getLeft()); - } + final Function, Boolean> notPrimaryKeyFilter = (Entry entry) -> { + if (entry == null) { + return true; + } + return tableDescription.primaryKeys().stream().noneMatch(primaryKeyPair -> primaryKeyPair.getLeft().equals(entry.getKey())); + }; + columnDescription.entrySet().stream() + .filter(notPrimaryKeyFilter::apply) + .forEach(entry -> create.addColumn(entry.getKey(), entry.getValue().getLeft())); log.trace("exacuting :{}", create); session.execute(create); log.info("table:{} created successfully.", tableName); @@ -142,8 +150,8 @@ public class SdcSchemaBuilder { log.info("table:{} already exists, skipping.", tableName); alterTable(session, existingTablesMetadata, tableDescription, tableName, columnDescription); } - log.info("keyspacemetadata:{}", keyspaceMetadate); - List indexNames = (keyspaceMetadate != null && keyspaceMetadate.get(tableName) != null ? keyspaceMetadate.get(tableName) + log.info("keyspacemetadata:{}", keyspaceMetadata); + List indexNames = (keyspaceMetadata != null && keyspaceMetadata.get(tableName) != null ? keyspaceMetadata.get(tableName) : new ArrayList<>()); log.info("table:{} creating indexes.", tableName); for (Map.Entry> description : columnDescription.entrySet()) { diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/Table.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/Table.java index 4c3c8fab2d..710a9a2921 100644 --- a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/Table.java +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/Table.java @@ -37,6 +37,7 @@ import org.openecomp.sdc.be.dao.cassandra.schema.tables.MigrationTasksTableDescr import org.openecomp.sdc.be.dao.cassandra.schema.tables.OperationalEnvironmentsTableDescription; import org.openecomp.sdc.be.dao.cassandra.schema.tables.ResAdminEventTableDescription; import org.openecomp.sdc.be.dao.cassandra.schema.tables.SdcSchemaFilesTableDescription; +import org.openecomp.sdc.be.dao.cassandra.schema.tables.ToscaImportByModelTableDescription; import org.openecomp.sdc.be.dao.cassandra.schema.tables.UserAccessEventTableDescription; import org.openecomp.sdc.be.dao.cassandra.schema.tables.UserAdminEventTableDescription; @@ -61,7 +62,8 @@ public enum Table { SDC_REPO(new MigrationTasksTableDescription()), SDC_OPERATIONAL_ENVIRONMENT(new OperationalEnvironmentsTableDescription()), AUDIT_ECOMP_OPERATIONAL_ENVIRONMENT(new EcompOperationalEnvironmentEventTableDesc()), - FEATURE_TOGGLE_STATE(new FeatureToggleEventTableDesc()); + FEATURE_TOGGLE_STATE(new FeatureToggleEventTableDesc()), + TOSCA_IMPORT_BY_MODEL(new ToscaImportByModelTableDescription()); // @formatter:on ITableDescription tableDescription; diff --git a/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/tables/ToscaImportByModelTableDescription.java b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/tables/ToscaImportByModelTableDescription.java new file mode 100644 index 0000000000..5b67136285 --- /dev/null +++ b/catalog-dao/src/main/java/org/openecomp/sdc/be/dao/cassandra/schema/tables/ToscaImportByModelTableDescription.java @@ -0,0 +1,78 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2021 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.dao.cassandra.schema.tables; + +import com.datastax.driver.core.DataType; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.openecomp.sdc.be.dao.cassandra.schema.ITableDescription; +import org.openecomp.sdc.be.resources.data.auditing.AuditingTypesConstants; + +public class ToscaImportByModelTableDescription implements ITableDescription { + + private static final String MODEL_ID = "model_id"; + private static final String FULL_PATH = "full_path"; + + @Override + public List> primaryKeys() { + return List.of( + new ImmutablePair<>(MODEL_ID, DataType.varchar()), + new ImmutablePair<>(FULL_PATH, DataType.varchar()) + ); + } + + @Override + public List> clusteringKeys() { + return Collections.emptyList(); + } + + @Override + public Map> getColumnDescription() { + return Stream.of(SdcSchemaFilesFieldsDescription.values()) + .collect(Collectors.toMap(SdcSchemaFilesFieldsDescription::getName, field -> new ImmutablePair<>(field.type, field.indexed))); + } + + @Override + public String getKeyspace() { + return AuditingTypesConstants.ARTIFACT_KEYSPACE; + } + + @Override + public String getTableName() { + return "tosca_import_by_model"; + } + + @Getter + @AllArgsConstructor + enum SdcSchemaFilesFieldsDescription { + MODEL_ID("model_id", DataType.varchar(), true), + CONTENT("content", DataType.varchar(), false); + + private final String name; + private final DataType type; + private final boolean indexed; + } +} -- cgit 1.2.3-korg