diff options
3 files changed, 260 insertions, 161 deletions
diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java index 241148b531..2f79a46827 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java @@ -21,38 +21,10 @@ package org.openecomp.sdc.be.tosca; -import static org.openecomp.sdc.be.tosca.ComponentCache.MergeStrategy.overwriteIfSameVersions; - import fj.F; import fj.data.Either; -import io.vavr.Tuple2; -import io.vavr.control.Try; import io.vavr.control.Option; -import java.io.BufferedOutputStream; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; +import io.vavr.control.Try; import lombok.Getter; import lombok.Setter; import org.apache.commons.codec.binary.Base64; @@ -61,11 +33,9 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.WordUtils; import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.commons.lang3.tuple.Triple; import org.onap.sdc.tosca.services.YamlUtil; import org.openecomp.sdc.be.components.impl.ImportUtils; -import org.openecomp.sdc.be.components.impl.ImportUtils.Constants; import org.openecomp.sdc.be.components.impl.exceptions.ByResponseFormatComponentException; import org.openecomp.sdc.be.config.ArtifactConfigManager; import org.openecomp.sdc.be.config.ArtifactConfiguration; @@ -88,12 +58,11 @@ import org.openecomp.sdc.be.model.LifecycleStateEnum; import org.openecomp.sdc.be.model.Resource; import org.openecomp.sdc.be.model.Service; import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade; -import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter; import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus; import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter; import org.openecomp.sdc.be.plugins.CsarEntryGenerator; import org.openecomp.sdc.be.resources.data.DAOArtifactData; -import org.openecomp.sdc.be.tosca.model.ToscaTemplate; +import org.openecomp.sdc.be.tosca.ZipIO.ZipWriter; import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil; import org.openecomp.sdc.be.utils.TypeUtils.ToscaTagNamesEnum; import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum; @@ -110,7 +79,37 @@ import org.openecomp.sdc.exception.ResponseFormat; import org.springframework.beans.factory.annotation.Autowired; import org.yaml.snakeyaml.Yaml; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import static org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter.isAtomicComponent; +import static org.openecomp.sdc.be.tosca.ComponentCache.MergeStrategy.overwriteIfSameVersions; import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.fromEither; +import static org.openecomp.sdc.be.tosca.ToscaExportHandler.getInterfaceFilename; +import static org.openecomp.sdc.be.tosca.ZipIO.writeTry; /** * @author tg851x @@ -272,7 +271,7 @@ public class CsarUtils { zip.write(mainYaml); //US798487 - Abstraction of complex types - if (!ModelConverter.isAtomicComponent(component)){ + if (!isAtomicComponent(component)){ log.debug("Component {} is complex - generating abstract type for it..", component.getName()); writeComponentInterface(component, zip, fileName, false); } @@ -447,7 +446,7 @@ public class CsarUtils { private Either<ZipOutputStream, ResponseFormat> getZipOutputStreamResponseFormatEither( ZipOutputStream zip, List<Triple<String, String, Component>> dependencies - ) throws IOException { + ) { ComponentCache innerComponentsCache = ComponentCache .overwritable(overwriteIfSameVersions()) @@ -476,48 +475,67 @@ public class CsarUtils { } //add inner components to CSAR - return addInnerComponentsToCSAR(zip, innerComponentsCache); + ZipWriter writer = ZipWriter.live(zip); + return addInnerComponentsToCSAR(innerComponentsCache).run(writer) + .map(void0 -> Either.<ZipOutputStream, ResponseFormat>left(zip)) + .recover(WithResponseFormat.class, e -> { + log.debug("#addInnerComponentsToCSAR Error while writing zip: ", e); + return Either.right(e.asResponseFormat(componentsUtils)); + }).get(); } return null; } - private Either<ZipOutputStream, ResponseFormat> addInnerComponentsToCSAR( - ZipOutputStream zip, - ComponentCache innerComponentsCache - ) throws IOException { - for (ImmutableTriple<String, String, Component> ict : innerComponentsCache.iterable()) { - Component innerComponent = ict.getRight(); - - String icFileName = ict.getMiddle(); - // add component to zip - Either<Tuple2<byte[], ZipEntry>, ResponseFormat> zipEntry = toZipEntry(ict); - // TODO: this should not be done, we should instead compose this either further, - // but in order to keep this refactoring small, we'll stop here. - if (zipEntry.isRight()) { - return Either.right(zipEntry.right().value()); - } - Tuple2<byte[], ZipEntry> value = zipEntry.left().value(); - zip.putNextEntry(value._2); - zip.write(value._1); - // add component interface to zip - if (!ModelConverter.isAtomicComponent(innerComponent)) { - writeComponentInterface(innerComponent, zip, icFileName, true); - } + private ZipIO addInnerComponentsToCSAR(ComponentCache innerComponentsCache) { + return ZipIO.writeAll(innerComponentsCache.all().map(e -> { + ZipIO writeEntryData = writeTry( + DEFINITIONS_PATH + e.fileName, + () -> fetchEntryData(e.id, e.component) + ); + ZipIO writeInnerComponent = writeTry( + DEFINITIONS_PATH + getInterfaceFilename(e.fileName), + () -> exportComponentInterface(e.component) + ).writeIf(isAtomicComponent(e.component)); + + return ZipIO.both(writeEntryData, writeInnerComponent); + })); + } + + private Try<byte[]> fetchEntryData(String cassandraId, Component childComponent) { + return fromEither(getEntryData(cassandraId, childComponent), + status -> new EntryDataFetchingFailure(cassandraId, status) + ); + } + + private Try<byte[]> exportComponentInterface(final Component component) { + return fromEither(toscaExportUtils + .exportComponentInterface(component, true) + .left().map(ToscaRepresentation::getMainYaml), + ToscaErrorException::new + ); + } + + public static abstract class WithResponseFormat extends Exception { + abstract ResponseFormat asResponseFormat(ComponentsUtils cu); + + public WithResponseFormat(String message) { + super(message); } - return null; } - private Either<Tuple2<byte[], ZipEntry>, ResponseFormat> toZipEntry( - ImmutableTriple<String, String, Component> cachedEntry - ) { - String cassandraId = cachedEntry.getLeft(); - String fileName = cachedEntry.getMiddle(); - Component innerComponent = cachedEntry.getRight(); - return getEntryData(cassandraId, innerComponent) - .right().map(status -> { - log.debug("Failed adding to zip component {}, error {}", cassandraId, status); - return componentsUtils.getResponseFormat(status); - }).left().map(content -> new Tuple2<>(content, new ZipEntry(DEFINITIONS_PATH + fileName))); + public static class EntryDataFetchingFailure extends WithResponseFormat { + + private final ActionStatus status; + + EntryDataFetchingFailure(String cassandraId, ActionStatus status) { + super("Failed fetching entry data for " + cassandraId + ", error: " + status); + this.status = status; + } + + @Override + public ResponseFormat asResponseFormat(ComponentsUtils cu) { + return cu.getResponseFormat(status); + } } /** @@ -604,7 +622,7 @@ public class CsarUtils { //if not atomic - insert inner components as well // TODO: This could potentially create a StackOverflowException if the call stack // happens to be too large. Tail-recursive optimization should be used here. - if (!ModelConverter.isAtomicComponent(componentRI)) { + if (!isAtomicComponent(componentRI)) { addInnerComponentsToCache(componentCache, componentRI); } }); @@ -646,33 +664,38 @@ public class CsarUtils { boolean isAssociatedComponent ) { // TODO: This should not be done but we need this to keep the refactoring small enough to be easily reviewable - return writeComponentInterface(component, fileName, isAssociatedComponent, ZipWriter.live(zip)) - .map(void0 -> Either.<ZipOutputStream, ResponseFormat>left(zip)) - .recover(th -> { - log.error("#writeComponentInterface - zip writing failed with error: ", th); - return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR)); - }).get(); + return writeComponentInterface(component, fileName, isAssociatedComponent) + .run(ZipWriter.live(zip)) + .map(void0 -> Either.<ZipOutputStream, ResponseFormat>left(zip)) + .recover(th -> { + log.error("#writeComponentInterface - zip writing failed with error: ", th); + return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR)); + }).get(); } - private Try<Void> writeComponentInterface( + private ZipIO writeComponentInterface( Component component, String fileName, - boolean isAssociatedComponent, - ZipWriter zw + boolean isAssociatedComponent ) { Either<byte[], ToscaError> yml = toscaExportUtils .exportComponentInterface(component, isAssociatedComponent) .left().map(ToscaRepresentation::getMainYaml); - return fromEither(yml, ToscaErrorException::new) - .flatMap(zw.write(DEFINITIONS_PATH + ToscaExportHandler.getInterfaceFilename(fileName))); + return ZipIO.writeTry(DEFINITIONS_PATH + getInterfaceFilename(fileName), + fromEither(yml, ToscaErrorException::new)); } - public static class ToscaErrorException extends Exception { + public static class ToscaErrorException extends WithResponseFormat { ToscaErrorException(ToscaError error) { super("Error while exporting component's interface (toscaError:" + error + ")"); } + + @Override + public ResponseFormat asResponseFormat(ComponentsUtils cu) { + return cu.getResponseFormat(ActionStatus.GENERAL_ERROR); + } } private Either<byte[], ActionStatus> getEntryData(String cassandraId, Component childComponent) { diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ZipIO.java b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ZipIO.java new file mode 100644 index 0000000000..916aecfb33 --- /dev/null +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ZipIO.java @@ -0,0 +1,154 @@ +/*- + * ============LICENSE_START======================================================= + * SDC + * ================================================================================ + * Copyright (C) 2020 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.tosca; + +import io.vavr.collection.Stream; +import io.vavr.control.Try; + +import java.util.function.Supplier; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * ZipIO abstracts a writing operation in a Zip file + */ +public interface ZipIO { + + /** + * Run the operation and perform any IO required + * + * @param zw The writer used to write in the zip file + */ + Try<Void> run(ZipWriter zw); + + /** No-Op */ + // It's ok to return a null as an operation results in a Try<Void> + ZipIO None = zw -> Try.success(null); + + /** + * Combines two {@link org.openecomp.sdc.be.tosca.ZipIO} into one. The resulting operation performs each + * underlying operation sequentially. If the first operation fails, the second one won't be executed. + * + * @param left The first operation to run + * @param right The second operation to run + */ + static ZipIO both(ZipIO left, ZipIO right) { + return zw -> left.run(zw).flatMap(v -> right.run(zw)); + } + + /** + * Builds an operation resulting in a failure + * + * @param th The resulting failure + */ + static ZipIO error(Throwable th) { + return zw -> Try.failure(th); + } + + /** + * Builds an operation adding an entry in a zip file + * + * @param name The entry name + * @param payload The entry's payload + */ + static ZipIO write(String name, Supplier<byte[]> payload) { + return zw -> zw.write(name, payload.get()); + } + + /** + * Alias for {@link org.openecomp.sdc.be.tosca.ZipIO}.writeTry + */ + static ZipIO writeTry(String name, Try<byte[]> payload) { + return writeTry(name, () -> payload); + } + + /** + * Builds an operation resulting from a computation potentially resulting in a payload. + * If the payload cannot be retrieved, the operation results in an error. + * + * @param name The entry's name + * @param payload The entry's payload if it can be successfully retrieved + */ + static ZipIO writeTry(String name, Supplier<Try<byte[]>> payload) { + return zw -> payload.get() + .map(bs -> write(name, () -> bs)) + .getOrElseGet(ZipIO::error) + .run(zw); + } + + /** + * Combine all operations resulting from a {@link io.vavr.collection.Stream} + * + * @param zs The {@link io.vavr.collection.Stream} outputting the operations + */ + static ZipIO writeAll(Stream<ZipIO> zs) { + return zw -> zs.foldLeft( + Try.success(null), + (acc, zio) -> acc.flatMap(void0 -> zio.run(zw)) + ); + } + + /** + * Alias for {@link org.openecomp.sdc.be.tosca.ZipIO}.both + */ + default ZipIO and(ZipIO zw) { + return both(this, zw); + } + + /** + * Builds an operation only if the given predicate is true, otherwise returns a no-op. + * + * @param p The predicate + */ + default ZipIO writeIf(boolean p) { + return p ? this : None; + } + + /** + * ZipWriter abstracts the Zip file writing logic. + */ + interface ZipWriter { + + /** + * Writes an entry provided with its name and its payload + * + * @param entryName The entry's name to use in the zip file + * @param payload The payload to write for this entry + */ + Try<Void> write(String entryName, byte[] payload); + + /** + * Builds a {@link org.openecomp.sdc.be.tosca.ZipIO.ZipWriter} that outputs the data + * on a {@link java.util.zip.ZipOutputStream} + * + * @param zos the target {@link java.util.zip.ZipOutputStream} + */ + static ZipWriter live(ZipOutputStream zos) { + return (entryName, payload) -> Try.of(() -> { + zos.putNextEntry(new ZipEntry(entryName)); + zos.write(payload); + // We can return null as a Void is expected; + return null; + }); + } + } +} + diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ZipWriter.java b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ZipWriter.java deleted file mode 100644 index 916895fff4..0000000000 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ZipWriter.java +++ /dev/null @@ -1,78 +0,0 @@ -/*- - * ============LICENSE_START======================================================= - * SDC - * ================================================================================ - * Copyright (C) 2020 AT&T Intellectual Property. All rights reserved. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END========================================================= - */ - -package org.openecomp.sdc.be.tosca; - -import fj.data.Either; -import io.vavr.control.Try; -import java.util.function.Function; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** - * ZipWriter abstracts the Zip file writing logic. - */ -public interface ZipWriter { - - /** - * Writes an entry provided with its name and its payload - * - * @param entryName The entry's name to use in the zip file - * @param payload The payload to write for this entry - */ - Try<Void> write(String entryName, byte[] payload); - - /** - * Writes an entry provided with its name and its payload - * - * @param entryName The entry's name to use in the zip file - * @param payload The payload to write for this entry - */ - default Try<Void> write(String entryName, String payload) { - return write(entryName, payload.getBytes()); - } - - /** - * Alias for {@link org.openecomp.sdc.be.tosca.ZipWriter} - * - * @param entryName The entry's name to use in the zip file - */ - default Function<String, Try<Void>> writeString(String entryName) { - return payload -> write(entryName, payload.getBytes()); - } - - default Function<byte[], Try<Void>> write(String entryName) { - return payload -> write(entryName, payload); - } - - /** - * Builds a ZipWriter that outputs the data on a {@link java.util.zip.ZipOutputStream} - * @param zos the target {@link java.util.zip.ZipOutputStream} - */ - static ZipWriter live(ZipOutputStream zos) { - return (entryName, payload) -> Try.of(() -> { - zos.putNextEntry(new ZipEntry(entryName)); - zos.write(payload); - // We can return null as a Void is expected; - return null; - }); - } -} - |