diff options
Diffstat (limited to 'participant')
23 files changed, 1655 insertions, 0 deletions
diff --git a/participant/participant-impl/participant-impl-kserve/pom.xml b/participant/participant-impl/participant-impl-kserve/pom.xml new file mode 100755 index 000000000..81305006a --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/pom.xml @@ -0,0 +1,67 @@ +<!-- + ============LICENSE_START======================================================= + Copyright (C) 2023 Nordix Foundation. + ================================================================================ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + SPDX-License-Identifier: Apache-2.0 + ============LICENSE_END========================================================= +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.onap.policy.clamp.participant</groupId> + <artifactId>policy-clamp-participant-impl</artifactId> + <version>6.4.1-SNAPSHOT</version> + </parent> + + <artifactId>policy-clamp-participant-impl-kserve</artifactId> + <name>${project.artifactId}</name> + <description>Kserve participant, that performs operations related to Kserve inference service deployment</description> + <build> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>repackage</goal> + </goals> + <phase>package</phase> + </execution> + </executions> + </plugin> + </plugins> + </build> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.cloud</groupId> + <artifactId>spring-cloud-dependencies</artifactId> + <version>2021.0.5</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.cloud</groupId> + <artifactId>spring-cloud-starter-kubernetes-client</artifactId> + </dependency> + </dependencies> +</project> diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/Application.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/Application.java new file mode 100755 index 000000000..c5299e235 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/Application.java @@ -0,0 +1,61 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.util.Config; +import java.io.IOException; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; + +/** + * Starter. + */ +// @formatter:off +@SpringBootApplication +@ComponentScan({"org.onap.policy.clamp.acm.participant.kserve", "org.onap.policy.clamp.acm.participant.intermediary"}) +@ConfigurationPropertiesScan("org.onap.policy.clamp.acm.participant.kserve.parameters") +//@formatter:on +public class Application { + + /** + * Main class. + * + * @param args args + */ + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } + + /** + * Default Api Client bean creation. + * + * @return ApiClient + * @throws IOException exception + */ + @Bean + public ApiClient defaultApiClient() throws IOException { + return Config.fromCluster(); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/MicrometerConfig.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/MicrometerConfig.java new file mode 100755 index 000000000..92dc5d3d6 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/MicrometerConfig.java @@ -0,0 +1,46 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.config; + +import io.micrometer.core.aop.TimedAspect; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MicrometerConfig { + + /** + * Load up the metrics registry. + */ + @Bean + public InitializingBean forcePrometheusPostProcessor(BeanPostProcessor meterRegistryPostProcessor, + MeterRegistry registry) { + return () -> meterRegistryPostProcessor.postProcessAfterInitialization(registry, ""); + } + + @Bean + public TimedAspect timedAspect(MeterRegistry registry) { + return new TimedAspect(registry); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/ParticipantConfig.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/ParticipantConfig.java new file mode 100755 index 000000000..1a671343a --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/ParticipantConfig.java @@ -0,0 +1,43 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.config; + +import org.onap.policy.clamp.acm.participant.intermediary.api.ParticipantIntermediaryApi; +import org.onap.policy.clamp.acm.participant.kserve.handler.AutomationCompositionElementHandler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ParticipantConfig { + + /** + * Register AutomationCompositionElementListener. + * + * @param intermediaryApi the ParticipantIntermediaryApi + * @param acElementHandler the AutomationComposition Element Handler + */ + @Autowired + public void registerAutomationCompositionElementListener(ParticipantIntermediaryApi intermediaryApi, + AutomationCompositionElementHandler acElementHandler) { + intermediaryApi.registerAutomationCompositionElementListener(acElementHandler); + acElementHandler.setIntermediaryApi(intermediaryApi); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/SecurityConfig.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/SecurityConfig.java new file mode 100755 index 000000000..f9dbc08fc --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/config/SecurityConfig.java @@ -0,0 +1,48 @@ +/*- + * ========================LICENSE_START================================= + * Copyright (C) 2023 Nordix Foundation. All rights reserved. + * ====================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================LICENSE_END=================================== + */ + +package org.onap.policy.clamp.acm.participant.kserve.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.web.SecurityFilterChain; + +@Configuration +public class SecurityConfig { + + /** + * Return the configuration of how access to this module's REST end points is secured. + * + * @param http the HTTP security settings + * @return the HTTP security settings + */ + @Bean + public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception { + http + .httpBasic() + .and() + .authorizeHttpRequests() + .anyRequest() + .authenticated() + .and() + .csrf() + .disable(); + return http.build(); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/exception/KserveException.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/exception/KserveException.java new file mode 100755 index 000000000..045c80c21 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/exception/KserveException.java @@ -0,0 +1,39 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.exception; + +import javax.ws.rs.core.Response; +import org.onap.policy.models.base.PfModelException; + +public class KserveException extends PfModelException { + + public KserveException(String message) { + super(Response.Status.BAD_GATEWAY, message); + } + + public KserveException(int statusCode, String message, Exception e) { + super(Response.Status.fromStatusCode(statusCode), message, e); + } + + public KserveException(String message, Exception originalException) { + super(Response.Status.BAD_GATEWAY, message, originalException); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/handler/AutomationCompositionElementHandler.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/handler/AutomationCompositionElementHandler.java new file mode 100755 index 000000000..cae70565f --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/handler/AutomationCompositionElementHandler.java @@ -0,0 +1,167 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.handler; + +import io.kubernetes.client.openapi.ApiException; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import javax.validation.Validation; +import javax.validation.ValidationException; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.apache.http.HttpStatus; +import org.onap.policy.clamp.acm.participant.intermediary.api.AutomationCompositionElementListener; +import org.onap.policy.clamp.acm.participant.intermediary.api.ParticipantIntermediaryApi; +import org.onap.policy.clamp.acm.participant.kserve.exception.KserveException; +import org.onap.policy.clamp.acm.participant.kserve.k8s.InferenceServiceValidator; +import org.onap.policy.clamp.acm.participant.kserve.k8s.KserveClient; +import org.onap.policy.clamp.acm.participant.kserve.models.ConfigurationEntity; +import org.onap.policy.clamp.acm.participant.kserve.models.KserveInferenceEntity; +import org.onap.policy.clamp.models.acm.concepts.AcElementDeploy; +import org.onap.policy.clamp.models.acm.concepts.DeployState; +import org.onap.policy.clamp.models.acm.concepts.LockState; +import org.onap.policy.common.utils.coder.Coder; +import org.onap.policy.common.utils.coder.CoderException; +import org.onap.policy.common.utils.coder.StandardCoder; +import org.onap.policy.models.base.PfModelException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +/** + * This class handles implementation of automationCompositionElement updates. + */ +@Component +@RequiredArgsConstructor +public class AutomationCompositionElementHandler implements AutomationCompositionElementListener { + + private static final Coder CODER = new StandardCoder(); + + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + @Setter + private ParticipantIntermediaryApi intermediaryApi; + + private final KserveClient kserveClient; + + @Getter + private static final Map<UUID, ConfigurationEntity> configRequestMap = new HashMap<>(); + + + private static class ThreadConfig { + + private int uninitializedToPassiveTimeout = 60; + private int statusCheckInterval = 30; + } + + @Override + public void undeploy(UUID automationCompositionId, UUID automationCompositionElementId) { + var configurationEntity = configRequestMap.get(automationCompositionElementId); + if (configurationEntity != null) { + try { + for (KserveInferenceEntity kserveInferenceEntity : configurationEntity.getKserveInferenceEntities()) { + kserveClient.undeployInferenceService(kserveInferenceEntity.getNamespace(), + kserveInferenceEntity.getName()); + } + configRequestMap.remove(automationCompositionElementId); + intermediaryApi.updateAutomationCompositionElementState(automationCompositionId, + automationCompositionElementId, DeployState.UNDEPLOYED, LockState.NONE); + } catch (IOException | ApiException exception) { + LOGGER.warn("Deletion of Inference service failed", exception); + } + } + } + + /** + * Callback method to handle an update on an automation composition element. + * + * @param automationCompositionId the ID of the automation composition + * @param element the information on the automation composition element + * @param properties properties Map + */ + @Override + public void deploy(UUID automationCompositionId, AcElementDeploy element, Map<String, Object> properties) + throws PfModelException { + try { + var configurationEntity = CODER.convert(properties, ConfigurationEntity.class); + var violations = Validation.buildDefaultValidatorFactory().getValidator().validate(configurationEntity); + if (violations.isEmpty()) { + boolean isAllInferenceSvcDeployed = true; + var config = CODER.convert(properties, ThreadConfig.class); + for (KserveInferenceEntity kserveInferenceEntity : configurationEntity.getKserveInferenceEntities()) { + kserveClient.deployInferenceService(kserveInferenceEntity.getNamespace(), + kserveInferenceEntity.getPayload()); + + if (!checkInferenceServiceStatus(kserveInferenceEntity.getName(), + kserveInferenceEntity.getNamespace(), config.uninitializedToPassiveTimeout, + config.statusCheckInterval)) { + isAllInferenceSvcDeployed = false; + break; + } + } + if (isAllInferenceSvcDeployed) { + configRequestMap.put(element.getId(), configurationEntity); + intermediaryApi.updateAutomationCompositionElementState(automationCompositionId, element.getId(), + DeployState.DEPLOYED, LockState.LOCKED); + } else { + LOGGER.error("Inference Service deployment failed"); + } + } else { + LOGGER.error("Violations found in the config request parameters: {}", violations); + throw new ValidationException("Constraint violations in the config request"); + } + } catch (ValidationException | ExecutionException | InterruptedException | CoderException e) { + throw new KserveException(HttpStatus.SC_BAD_REQUEST, "Invalid Configuration", e); + } catch (IOException | ApiException e) { + throw new KserveException(HttpStatus.SC_BAD_REQUEST, "Failed to configure the inference service", e); + } + } + + /** + * Check the status of Inference Service. + * + * @param inferenceServiceName name of the inference service + * @param namespace kubernetes namespace + * @param timeout Inference service time check + * @param statusCheckInterval Status check time interval + * @return status of the inference service + * @throws ExecutionException Exception on execution + * @throws InterruptedException Exception on inference service status check + */ + public boolean checkInferenceServiceStatus(String inferenceServiceName, String namespace, int timeout, + int statusCheckInterval) throws ExecutionException, InterruptedException { + // Invoke runnable thread to check pod status + Future<String> result = executor.submit( + new InferenceServiceValidator(inferenceServiceName, namespace, timeout, statusCheckInterval, + kserveClient), "Done"); + return (!result.get().isEmpty()) && result.isDone(); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/k8s/InferenceServiceValidator.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/k8s/InferenceServiceValidator.java new file mode 100755 index 000000000..65c4c7e27 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/k8s/InferenceServiceValidator.java @@ -0,0 +1,102 @@ +/*- + * ========================LICENSE_START================================= + * Copyright (C) 2023 Nordix Foundation. All rights reserved. + * ====================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ========================LICENSE_END=================================== + */ + +package org.onap.policy.clamp.acm.participant.kserve.k8s; + +import io.kubernetes.client.openapi.ApiException; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import lombok.SneakyThrows; +import org.onap.policy.clamp.acm.participant.kserve.exception.KserveException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +public class InferenceServiceValidator implements Runnable { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final KserveClient kserveClient; + + private final int statusCheckInterval; + + //Timeout for the thread to exit. + private final int timeout; + + private final String inferenceServiceName; + + private final String namespace; + + /** + * Constructor for PodStatusValidator. + * + * @param inferenceServiceName name of the inference service + * @param namespace kubernetes namespace + * @param timeout timeout for the thread to exit + * @param statusCheckInterval Interval to check pod status + */ + public InferenceServiceValidator(String inferenceServiceName, String namespace, int timeout, + int statusCheckInterval, KserveClient kserveClient) { + this.inferenceServiceName = inferenceServiceName; + this.namespace = namespace; + this.timeout = timeout; + this.statusCheckInterval = statusCheckInterval; + this.kserveClient = kserveClient; + } + + + @SneakyThrows + @Override + public void run() { + logger.info("Polling the status of deployed Inference Service {} in namespace {}", inferenceServiceName, + namespace); + try { + verifyInferenceServiceStatus(); + } catch (KserveException | IOException e) { + throw new KserveException("Error verifying the status of the inference service. Exiting", e); + } + } + + /** + * Verify inference service status. + * @throws KserveException exception + * @throws IOException exception + * @throws InterruptedException exception + * @throws ApiException exception + */ + private void verifyInferenceServiceStatus() + throws KserveException, IOException, InterruptedException, ApiException { + var isVerified = false; + long endTime = System.currentTimeMillis() + (timeout * 1000L); + + while (!isVerified && System.currentTimeMillis() < endTime) { + var output = kserveClient.getInferenceServiceStatus(namespace, inferenceServiceName); + isVerified = output.equalsIgnoreCase(Boolean.TRUE.toString()); + if (!isVerified) { + logger.info("Waiting for the inference service {} to be active ", inferenceServiceName); + // Recheck status of pods in specific intervals. + Thread.sleep(statusCheckInterval * 1000L); + } else { + logger.info("Inference Service {} is Ready to use ", inferenceServiceName); + } + } + if (!isVerified) { + throw new KserveException("Time out Exception verifying the status of the inference service"); + } + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/k8s/KserveClient.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/k8s/KserveClient.java new file mode 100755 index 000000000..b5e7892eb --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/k8s/KserveClient.java @@ -0,0 +1,112 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.k8s; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CustomObjectsApi; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import javax.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import okhttp3.Response; +import org.onap.policy.clamp.acm.participant.kserve.parameters.CustomResourceDefinitionParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class KserveClient { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final ApiClient apiClient; + + private final CustomResourceDefinitionParameters crdParams; + + CustomObjectsApi customObjectsApi; + + @PostConstruct + void initialize() { + apiClient.setDebugging(logger.isDebugEnabled()); + customObjectsApi = new CustomObjectsApi(apiClient); + } + + /** + * Deploy the inference service. + * + * @param namespace k8s namespace + * @param jsonContent k8s payload + * @throws ApiException exception + */ + public boolean deployInferenceService(String namespace, String jsonContent) throws ApiException, IOException { + var httpCall = customObjectsApi.createNamespacedCustomObjectCall(crdParams.getGroup(), crdParams.getVersion(), + namespace, crdParams.getPlural(), jsonContent.getBytes(), null, null, null, null); + try (Response httpResponse = httpCall.execute()) { + logger.debug("Response of inference service deploy in namespace {} is {}", namespace, httpResponse); + return httpResponse.isSuccessful(); + } + } + + /** + * Undeploy inference service. + * + * @param namespace k8s namespace + * @param inferenceServiceName name of the inference service + * @throws ApiException exception + */ + public boolean undeployInferenceService(String namespace, String inferenceServiceName) + throws ApiException, IOException { + var httpCall = customObjectsApi.deleteNamespacedCustomObjectCall(crdParams.getGroup(), crdParams.getVersion(), + namespace, crdParams.getPlural(), inferenceServiceName, crdParams.getGracePeriod(), false, null, null, + null, null); + try (Response httpResponse = httpCall.execute()) { + logger.debug("Response of inference service undeploy in namespace {} is {}", namespace, httpResponse); + return httpResponse.isSuccessful(); + } + } + + /** + * Get the status of Inference service. + * + * @param namespace k8s namespace + * @param inferenceServiceName name of the inference service + * @return State of the inference service + * @throws ApiException exception on k8s client + * @throws IOException exception + */ + public String getInferenceServiceStatus(String namespace, String inferenceServiceName) + throws ApiException, IOException { + var httpCall = + customObjectsApi.getNamespacedCustomObjectCall(crdParams.getGroup(), crdParams.getVersion(), namespace, + crdParams.getPlural(), inferenceServiceName, null); + try (Response httpResponse = httpCall.execute()) { + logger.debug("Response of getting inference service in {} is {}", namespace, httpResponse); + if (httpResponse.isSuccessful() && httpResponse.body() != null) { + JsonNode jsonNode = new ObjectMapper().readTree(httpResponse.body().string()); + return jsonNode.at("/status/conditions/2/status").asText(); + } + } + return Boolean.FALSE.toString(); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/models/ConfigurationEntity.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/models/ConfigurationEntity.java new file mode 100755 index 000000000..ab85b7c52 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/models/ConfigurationEntity.java @@ -0,0 +1,37 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.models; + +import java.util.List; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class ConfigurationEntity { + + @NotNull + @Valid + private List<KserveInferenceEntity> kserveInferenceEntities; + +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/models/KserveInferenceEntity.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/models/KserveInferenceEntity.java new file mode 100755 index 000000000..3717970c2 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/models/KserveInferenceEntity.java @@ -0,0 +1,49 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.models; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import javax.validation.constraints.NotNull; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.onap.policy.models.tosca.authorative.concepts.ToscaConceptIdentifier; + +@Data +@AllArgsConstructor +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) +public class KserveInferenceEntity { + + @NotNull + @JsonIgnore + private ToscaConceptIdentifier kserveInferenceEntityId; + + @NotNull + private String name; + + @NotNull + private String payload; + + @NotNull + private String namespace; + +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/parameters/CustomResourceDefinitionParameters.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/parameters/CustomResourceDefinitionParameters.java new file mode 100755 index 000000000..69a7cb466 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/parameters/CustomResourceDefinitionParameters.java @@ -0,0 +1,43 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.parameters; + +import javax.validation.constraints.NotNull; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Validated +@Data +@ConfigurationProperties(prefix = "customresourcedefinition") +public class CustomResourceDefinitionParameters { + + @NotNull + private String group; + + @NotNull + private String version; + + @NotNull + private String plural; + + private int gracePeriod = 10; +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/parameters/KserveParticipantParameters.java b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/parameters/KserveParticipantParameters.java new file mode 100755 index 000000000..81582e1f0 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/java/org/onap/policy/clamp/acm/participant/kserve/parameters/KserveParticipantParameters.java @@ -0,0 +1,44 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.parameters; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import lombok.Getter; +import lombok.Setter; +import org.onap.policy.clamp.acm.participant.intermediary.parameters.ParticipantIntermediaryParameters; +import org.onap.policy.clamp.acm.participant.intermediary.parameters.ParticipantParameters; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * Class to hold all parameters needed for the Kserve participant. + */ +@Validated +@Getter +@Setter +@ConfigurationProperties(prefix = "participant") +public class KserveParticipantParameters implements ParticipantParameters { + + @NotNull + @Valid + private ParticipantIntermediaryParameters intermediaryParameters; +} diff --git a/participant/participant-impl/participant-impl-kserve/src/main/resources/config/application.yaml b/participant/participant-impl/participant-impl-kserve/src/main/resources/config/application.yaml new file mode 100755 index 000000000..5ce1a7daf --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/main/resources/config/application.yaml @@ -0,0 +1,65 @@ +spring: + cloud: + kubernetes: + enabled: false + discovery: + enabled: false + security: + user: + name: participantUser + password: zb!XztG34 + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration + - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration + - org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration + - org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration + - io.kubernetes.client.spring.extended.manifests.config.KubernetesManifestsAutoConfiguration + - io.kubernetes.client.spring.extended.network.config.KubernetesLoadBalancerAutoConfiguration + +security: + enable-csrf: false + +participant: + intermediaryParameters: + reportingTimeIntervalMs: 120000 + description: Participant Description + participantId: 101c62b3-8918-41b9-a747-d21eb79c6c04 + clampAutomationCompositionTopics: + topicSources: + - topic: POLICY-ACRUNTIME-PARTICIPANT + servers: + - ${topicServer:localhost} + topicCommInfrastructure: dmaap + fetchTimeout: 15000 + topicSinks: + - topic: POLICY-ACRUNTIME-PARTICIPANT + servers: + - ${topicServer:localhost} + topicCommInfrastructure: dmaap + participantSupportedElementTypes: + - + typeName: org.onap.policy.clamp.acm.KserveAutomationCompositionElement + typeVersion: 1.0.1 + - + typeName: org.onap.policy.clamp.acm.AutomationCompositionElement + typeVersion: 1.0.0 + +customresourcedefinition: + group: serving.kserve.io + version: v1beta1 + plural: inferenceservices + grace-period: 10 + +management: + endpoints: + web: + base-path: / + exposure: + include: health, metrics, prometheus +server: + port: 8087 + servlet: + context-path: /onap/policy/clamp/acm/kserveparticipant + + diff --git a/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/handler/AcElementHandlerTest.java b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/handler/AcElementHandlerTest.java new file mode 100755 index 000000000..0743e0429 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/handler/AcElementHandlerTest.java @@ -0,0 +1,125 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.handler; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import io.kubernetes.client.openapi.ApiException; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.onap.policy.clamp.acm.participant.intermediary.api.ParticipantIntermediaryApi; +import org.onap.policy.clamp.acm.participant.kserve.exception.KserveException; +import org.onap.policy.clamp.acm.participant.kserve.k8s.KserveClient; +import org.onap.policy.clamp.acm.participant.kserve.utils.CommonTestData; +import org.onap.policy.clamp.acm.participant.kserve.utils.ToscaUtils; +import org.onap.policy.clamp.models.acm.concepts.AutomationCompositionElement; +import org.onap.policy.models.base.PfModelException; +import org.onap.policy.models.tosca.authorative.concepts.ToscaConceptIdentifier; +import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@ExtendWith(SpringExtension.class) +class AcElementHandlerTest { + + private final KserveClient kserveClient = mock(KserveClient.class); + + @InjectMocks + @Spy + private AutomationCompositionElementHandler automationCompositionElementHandler = + new AutomationCompositionElementHandler(kserveClient); + + @Mock + private ParticipantIntermediaryApi participantIntermediaryApi; + + @Mock + private ExecutorService executor; + @Mock + private Future<String> result; + + private final CommonTestData commonTestData = new CommonTestData(); + + private static ToscaServiceTemplate serviceTemplate; + private static final String KSERVE_AUTOMATION_COMPOSITION_ELEMENT = + "onap.policy.clamp.ac.element.KserveAutomationCompositionElement"; + + @BeforeAll + static void init() { + serviceTemplate = ToscaUtils.readAutomationCompositionFromTosca(); + } + + @BeforeEach + void startMocks() throws KserveException, ExecutionException, InterruptedException, IOException, ApiException { + doReturn(true).when(kserveClient).deployInferenceService(any(), any()); + doReturn(true).when(automationCompositionElementHandler) + .checkInferenceServiceStatus(any(), any(), anyInt(), anyInt()); + } + + @Test + void test_automationCompositionElementStateChange() throws PfModelException { + var automationCompositionId = commonTestData.getAutomationCompositionId(); + var element = commonTestData.getAutomationCompositionElement(); + var automationCompositionElementId = element.getId(); + + + var nodeTemplatesMap = serviceTemplate.getToscaTopologyTemplate().getNodeTemplates(); + automationCompositionElementHandler.deploy(commonTestData.getAutomationCompositionId(), element, + nodeTemplatesMap.get(KSERVE_AUTOMATION_COMPOSITION_ELEMENT).getProperties()); + + assertDoesNotThrow(() -> automationCompositionElementHandler.undeploy(automationCompositionId, + automationCompositionElementId)); + + } + + @Test + void test_AutomationCompositionElementUpdate() { + var element = commonTestData.getAutomationCompositionElement(); + + var nodeTemplatesMap = serviceTemplate.getToscaTopologyTemplate().getNodeTemplates(); + assertDoesNotThrow( + () -> automationCompositionElementHandler.deploy(commonTestData.getAutomationCompositionId(), element, + nodeTemplatesMap.get(KSERVE_AUTOMATION_COMPOSITION_ELEMENT).getProperties())); + } + + @Test + void test_checkInferenceServiceStatus() throws ExecutionException, InterruptedException { + doReturn(result).when(executor).submit(any(Runnable.class), any()); + doReturn("Done").when(result).get(); + doReturn(true).when(result).isDone(); + ToscaConceptIdentifier automationCompositionId = new ToscaConceptIdentifier(); + AutomationCompositionElement element = new AutomationCompositionElement(); + assertDoesNotThrow( + () -> automationCompositionElementHandler.checkInferenceServiceStatus("sklearn-iris", "kserve-test", 1, + 1)); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/k8s/InferenceServiceValidatorTest.java b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/k8s/InferenceServiceValidatorTest.java new file mode 100755 index 000000000..3ef89a7c1 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/k8s/InferenceServiceValidatorTest.java @@ -0,0 +1,78 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.k8s; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; + +import io.kubernetes.client.openapi.ApiException; +import java.io.IOException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.onap.policy.clamp.acm.participant.kserve.exception.KserveException; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; + + +@ExtendWith(SpringExtension.class) +class InferenceServiceValidatorTest { + + private static int TIMEOUT = 2; + private static int STATUS_CHECK_INTERVAL = 1; + + @MockBean + private KserveClient kserveClient; + + String inferenceSvcName = "inference-test"; + String namespace = "test"; + + @Test + void test_runningPodState() throws IOException, ApiException { + doReturn("True").when(kserveClient).getInferenceServiceStatus(any(), any()); + var inferenceServiceValidator = + new InferenceServiceValidator(inferenceSvcName, namespace, TIMEOUT, STATUS_CHECK_INTERVAL, + kserveClient); + assertDoesNotThrow(inferenceServiceValidator::run); + } + + @Test + void test_EmptyPodState() throws IOException, ApiException { + doReturn("").when(kserveClient).getInferenceServiceStatus(any(), any()); + var inferenceServiceValidator = + new InferenceServiceValidator("", namespace, TIMEOUT, STATUS_CHECK_INTERVAL, + kserveClient); + assertThatThrownBy(inferenceServiceValidator::run).isInstanceOf(KserveException.class) + .hasMessage("Error verifying the status of the inference service. Exiting"); + } + + @Test + void test_PodFailureState() throws IOException, ApiException { + doReturn("False").when(kserveClient).getInferenceServiceStatus(any(), any()); + var inferenceServiceValidator = + new InferenceServiceValidator(inferenceSvcName, namespace, TIMEOUT, STATUS_CHECK_INTERVAL, + kserveClient); + assertThatThrownBy(inferenceServiceValidator::run).isInstanceOf(KserveException.class) + .hasMessage("Error verifying the status of the inference service. Exiting"); + } + +} diff --git a/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/k8s/KserveClientTest.java b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/k8s/KserveClientTest.java new file mode 100755 index 000000000..c9be352db --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/k8s/KserveClientTest.java @@ -0,0 +1,196 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.k8s; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CustomObjectsApi; +import java.io.IOException; +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.apache.http.HttpStatus; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles("test") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class KserveClientTest { + + @SpyBean + private KserveClient kserveClient; + + String namespace = "kserve-test"; + + String inferenceServiceName = "sklearn-iris"; + + final Call remoteCall = mock(Call.class); + + final CustomObjectsApi customObjectsApi = mock(CustomObjectsApi.class); + + @MockBean + ApiClient apiClient; + + @BeforeAll + void initialize() { + kserveClient.customObjectsApi = customObjectsApi; + } + + + @Test + void test_deployInferenceServiceValidResponse() throws IOException, ApiException { + String jsonContent = + "{\"apiVersion\": \"serving.kserve.io/v1beta1\",\"kind\": \"InferenceService\",\"metadata\": " + + "{\"name\": \"" + inferenceServiceName + + "\"},\"spec\": {\"predictor\": {\"model\":{\"modelFormat\": " + + "{\"name\": \"sklearn\"},\"storageUri\": " + + "\"gs://kfserving-examples/models/sklearn/1.0/model\"}}}}"; + + var response = getResponse(HttpStatus.SC_OK); + when(remoteCall.execute()).thenReturn(response); + when(customObjectsApi.createNamespacedCustomObjectCall(any(), any(), any(), any(), any(), any(), any(), any(), + any())).thenReturn(remoteCall); + assertTrue(kserveClient.deployInferenceService(namespace, jsonContent)); + } + + @Test + void test_deployInferenceServiceInvalidResponse() throws IOException, ApiException { + String jsonContent = + "{\"apiVersion\": \"serving.kserve.io/v1beta1\",\"kind\": \"InferenceService\",\"metadata\": " + + "{\"name\": \"" + inferenceServiceName + + "\"},\"spec\": {\"predictor\": {\"model\":{\"modelFormat\": " + + "{\"name\": \"sklearn\"},\"storageUri\": " + + "\"gs://kfserving-examples/models/sklearn/1.0/model\"}}}}"; + + var response = getResponse(HttpStatus.SC_BAD_REQUEST); + when(remoteCall.execute()).thenReturn(response); + when(customObjectsApi.createNamespacedCustomObjectCall(any(), any(), any(), any(), any(), any(), any(), any(), + any())).thenReturn(remoteCall); + assertFalse(kserveClient.deployInferenceService(namespace, jsonContent)); + } + + @Test + void test_deployInvalidInferenceService() throws IOException, ApiException { + doThrow(new ApiException("Error in deploying the service")).when(kserveClient) + .deployInferenceService(any(), any()); + assertThatThrownBy(() -> kserveClient.deployInferenceService(namespace, "")).isInstanceOf(ApiException.class); + } + + @Test + void test_undeployInferenceServiceValidResponse() throws IOException, ApiException { + + var response = getResponse(HttpStatus.SC_OK); + when(remoteCall.execute()).thenReturn(response); + when(customObjectsApi.deleteNamespacedCustomObjectCall(any(), any(), any(), any(), any(), any(), any(), any(), + any(), any(), any())).thenReturn(remoteCall); + assertTrue(kserveClient.undeployInferenceService(namespace, inferenceServiceName)); + } + + @Test + void test_undeployInferenceServiceInvalidResponse() throws IOException, ApiException { + + var response = getResponse(HttpStatus.SC_BAD_REQUEST); + when(remoteCall.execute()).thenReturn(response); + when(customObjectsApi.deleteNamespacedCustomObjectCall(any(), any(), any(), any(), any(), any(), any(), any(), + any(), any(), any())).thenReturn(remoteCall); + assertFalse(kserveClient.undeployInferenceService(namespace, inferenceServiceName)); + } + + @Test + void test_getInferenceServiceStatusValidResponse() throws IOException, ApiException { + + var response = getResponse(HttpStatus.SC_OK, getInferenceServiceResponseBody("True")); + when(remoteCall.execute()).thenReturn(response); + when(customObjectsApi.getNamespacedCustomObjectCall(any(), any(), any(), any(), any(), any())).thenReturn( + remoteCall); + assertEquals("True", kserveClient.getInferenceServiceStatus(namespace, inferenceServiceName)); + } + + @Test + void test_getInferenceServiceStatusFalseResponse() throws IOException, ApiException { + + var response = getResponse(HttpStatus.SC_OK, getInferenceServiceResponseBody("False")); + when(remoteCall.execute()).thenReturn(response); + when(customObjectsApi.getNamespacedCustomObjectCall(any(), any(), any(), any(), any(), any())).thenReturn( + remoteCall); + assertEquals("False", kserveClient.getInferenceServiceStatus(namespace, inferenceServiceName)); + } + + @Test + void test_getInferenceServiceStatusInvalidResponse() throws IOException, ApiException { + + var response = getResponse(HttpStatus.SC_BAD_REQUEST, ""); + when(remoteCall.execute()).thenReturn(response); + when(customObjectsApi.getNamespacedCustomObjectCall(any(), any(), any(), any(), any(), any())).thenReturn( + remoteCall); + assertEquals("false", kserveClient.getInferenceServiceStatus(namespace, inferenceServiceName)); + } + + Response getResponse(int code) { + return getResponse(code, "{}"); + } + + Response getResponse(int code, String body) { + return new Response.Builder().request(new Request.Builder().url("http://test").build()) + .protocol(Protocol.HTTP_1_1).code(code).message("") + .body(ResponseBody.create(body, MediaType.parse("application/json"))).build(); + } + + String getInferenceServiceResponseBody(String status) { + return "{ \"apiVersion\": \"serving.kserve.io/v1beta1\", \"kind\": \"InferenceService\", \"spec\": " + + "{ \"predictor\": { \"model\": { \"modelFormat\": { \"name\": \"sklearn\" }, \"name\": \"\", " + + "\"resources\": {}, \"storageUri\": \"gs://kfserving-examples/models/sklearn/1.0/model\" } } " + + "}, \"status\": { \"address\": { \"url\": \"http://sklearn-iris.kserve-test.svc.cluster.local\"" + + " }, \"components\": { \"predictor\": { \"latestCreatedRevision\": \"1\", \"url\": " + + "\"http://sklearn-iris-predictor-default-kserve-test.example.com\" } }, \"conditions\": [ " + + "{ \"lastTransitionTime\": \"2023-02-15T13:39:16Z\", \"status\": \"" + status + + "\", \"type\": " + + "\"IngressReady\" }, { \"lastTransitionTime\": \"2023-02-15T13:39:16Z\", \"status\": " + "\"" + + status + "\", \"type\": \"PredictorReady\" }, { \"lastTransitionTime\": " + + "\"2023-02-15T13:39:16Z\", \"status\": \"" + status + "\", \"type\": \"Ready\" } ], " + + "\"modelStatus\": { \"copies\": { \"failedCopies\": 0, \"totalCopies\": 1 }, \"states\": " + + "{ \"activeModelState\": \"Loaded\", \"targetModelState\": \"Loaded\" }, \"transitionStatus\":" + + "\"UpToDate\" }, \"observedGeneration\": 1, \"url\": " + + " \"http://sklearn-iris-kserve-test.example.com\" } }"; + } + +} diff --git a/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/rest/ActuatorControllerTest.java b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/rest/ActuatorControllerTest.java new file mode 100755 index 000000000..0cabac793 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/rest/ActuatorControllerTest.java @@ -0,0 +1,97 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2021-2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.kubernetes.client.openapi.ApiClient; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.core.Response; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.onap.policy.clamp.acm.participant.kserve.utils.CommonActuatorController; +import org.springframework.boot.test.autoconfigure.actuate.metrics.AutoConfigureMetrics; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@AutoConfigureMetrics +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT) +@ActiveProfiles("test") +class ActuatorControllerTest extends CommonActuatorController { + + private static final String HEALTH_ENDPOINT = "health"; + private static final String METRICS_ENDPOINT = "metrics"; + private static final String PROMETHEUS_ENDPOINT = "prometheus"; + + @MockBean + ApiClient apiClient; + + @LocalServerPort + private int randomServerPort; + + @BeforeEach + public void setUpPort() { + super.setHttpPrefix(randomServerPort); + } + + @Test + void testGetHealth_Unauthorized() throws Exception { + assertUnauthorizedActGet(HEALTH_ENDPOINT); + } + + @Test + void testGetMetrics_Unauthorized() throws Exception { + assertUnauthorizedActGet(METRICS_ENDPOINT); + } + + @Test + void testGetPrometheus_Unauthorized() throws Exception { + assertUnauthorizedActGet(PROMETHEUS_ENDPOINT); + } + + @Test + void testGetHealth() throws Exception { + Invocation.Builder invocationBuilder = super.sendActRequest(HEALTH_ENDPOINT); + Response rawresp = invocationBuilder.buildGet().invoke(); + assertEquals(Response.Status.OK.getStatusCode(), rawresp.getStatus()); + } + + @Test + void testGetMetrics() throws Exception { + Invocation.Builder invocationBuilder = super.sendActRequest(METRICS_ENDPOINT); + Response rawresp = invocationBuilder.buildGet().invoke(); + assertEquals(Response.Status.OK.getStatusCode(), rawresp.getStatus()); + } + + @Test + void testGePrometheus() throws Exception { + Invocation.Builder invocationBuilder = super.sendActRequest(PROMETHEUS_ENDPOINT); + Response rawresp = invocationBuilder.buildGet().invoke(); + assertEquals(Response.Status.OK.getStatusCode(), rawresp.getStatus()); + } + +} diff --git a/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/CommonActuatorController.java b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/CommonActuatorController.java new file mode 100755 index 000000000..b83a13897 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/CommonActuatorController.java @@ -0,0 +1,107 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.utils; + +import static org.junit.Assert.assertEquals; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; +import org.onap.policy.common.gson.GsonMessageBodyHandler; +import org.onap.policy.common.utils.network.NetworkUtil; + +/** + * Class to perform Rest unit tests. + */ +public class CommonActuatorController { + + public static final String SELF = NetworkUtil.getHostname(); + public static final String CONTEXT_PATH = "onap/policy/clamp/acm/kserveparticipant/"; + + private static String httpPrefix; + + /** + * Sends a request to an actuator endpoint. + * + * @param endpoint the target endpoint + * @return a request builder + */ + protected Invocation.Builder sendActRequest(final String endpoint) { + return sendFqeRequest(httpPrefix + CONTEXT_PATH + endpoint, true); + } + + /** + * Sends a request to an actuator endpoint, without any authorization header. + * + * @param endpoint the target endpoint + * @return a request builder + */ + protected Invocation.Builder sendNoAuthActRequest(final String endpoint) { + return sendFqeRequest(httpPrefix + CONTEXT_PATH + endpoint, false); + } + + /** + * Sends a request to a fully qualified endpoint. + * + * @param fullyQualifiedEndpoint the fully qualified target endpoint + * @param includeAuth if authorization header should be included + * @return a request builder + */ + protected Invocation.Builder sendFqeRequest(final String fullyQualifiedEndpoint, boolean includeAuth) { + final Client client = ClientBuilder.newBuilder().build(); + + client.property(ClientProperties.METAINF_SERVICES_LOOKUP_DISABLE, "true"); + client.register(GsonMessageBodyHandler.class); + + if (includeAuth) { + client.register(HttpAuthenticationFeature.basic("participantUser", "zb!XztG34")); + } + + final WebTarget webTarget = client.target(fullyQualifiedEndpoint); + + return webTarget.request(MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN); + } + + /** + * Assert that GET call to actuator endpoint is Unauthorized. + * + * @param endPoint the endpoint + */ + protected void assertUnauthorizedActGet(final String endPoint) { + Response rawresp = sendNoAuthActRequest(endPoint).buildGet().invoke(); + assertEquals(Response.Status.UNAUTHORIZED.getStatusCode(), rawresp.getStatus()); + } + + /** + * Set Up httpPrefix. + * + * @param port the port + */ + protected void setHttpPrefix(int port) { + httpPrefix = "http://" + SELF + ":" + port + "/"; + } + +} diff --git a/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/CommonTestData.java b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/CommonTestData.java new file mode 100755 index 000000000..076738ac4 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/CommonTestData.java @@ -0,0 +1,60 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * Modifications Copyright (C) 2022 AT&T Intellectual Property. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.utils; + +import java.util.List; +import java.util.UUID; +import org.onap.policy.clamp.models.acm.concepts.AcElementDeploy; +import org.onap.policy.clamp.models.acm.concepts.AutomationCompositionElement; +import org.onap.policy.clamp.models.acm.concepts.AutomationCompositionOrderedState; +import org.onap.policy.clamp.models.acm.messages.rest.instantiation.DeployOrder; +import org.onap.policy.models.tosca.authorative.concepts.ToscaConceptIdentifier; + +public class CommonTestData { + + private static final String TEST_KEY_NAME = "org.onap.domain.database.KserveAutomationCompositionElement"; + + private static final List<UUID> AC_ID_LIST = List.of(UUID.randomUUID(), UUID.randomUUID()); + + /** + * Get a automationComposition Element. + * + * @return automationCompositionElement object + */ + public AcElementDeploy getAutomationCompositionElement() { + var element = new AcElementDeploy(); + element.setId(UUID.randomUUID()); + element.setDefinition(new ToscaConceptIdentifier(TEST_KEY_NAME, "1.0.1")); + element.setOrderedState(DeployOrder.DEPLOY); + return element; + } + + /** + * Get automation composition id. + * + * @return ToscaConceptIdentifier automationCompositionId + */ + public UUID getAutomationCompositionId() { + return AC_ID_LIST.get(0); + } + +} diff --git a/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/ToscaUtils.java b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/ToscaUtils.java new file mode 100755 index 000000000..e2b00e4c9 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/java/org/onap/policy/clamp/acm/participant/kserve/utils/ToscaUtils.java @@ -0,0 +1,51 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.clamp.acm.participant.kserve.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.onap.policy.common.utils.coder.YamlJsonTranslator; +import org.onap.policy.common.utils.resources.ResourceUtils; +import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate; + +/** + * Util class for Test scope. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class ToscaUtils { + + private static final YamlJsonTranslator yamlTranslator = new YamlJsonTranslator(); + private static final String TOSCA_TEMPLATE_YAML = "clamp/acm/test/participant-kserve.yaml"; + + + /** + * Read a service template yaml. + * @return ToscaServiceTemplate + */ + public static ToscaServiceTemplate readAutomationCompositionFromTosca() { + return serializeAutomationCompositionYaml(); + } + + private static ToscaServiceTemplate serializeAutomationCompositionYaml() { + String automationCompositionString = ResourceUtils.getResourceAsString(ToscaUtils.TOSCA_TEMPLATE_YAML); + return yamlTranslator.fromYaml(automationCompositionString, ToscaServiceTemplate.class); + } +} diff --git a/participant/participant-impl/participant-impl-kserve/src/test/resources/application-test.yaml b/participant/participant-impl/participant-impl-kserve/src/test/resources/application-test.yaml new file mode 100755 index 000000000..8452ed071 --- /dev/null +++ b/participant/participant-impl/participant-impl-kserve/src/test/resources/application-test.yaml @@ -0,0 +1,17 @@ +participant: + intermediaryParameters: + reportingTimeIntervalMs: 120000 + description: Participant Description + participantId: 101c62b3-8918-41b9-a747-d21eb79c6c04 + clampAutomationCompositionTopics: + topicSources: + - topic: POLICY-ACRUNTIME-PARTICIPANT + servers: + - ${topicServer:localhost} + topicCommInfrastructure: dmaap + fetchTimeout: 15000 + topicSinks: + - topic: POLICY-ACRUNTIME-PARTICIPANT + servers: + - ${topicServer:localhost} + topicCommInfrastructure: dmaap
\ No newline at end of file diff --git a/participant/participant-impl/pom.xml b/participant/participant-impl/pom.xml index aaac69f97..3df049273 100644..100755 --- a/participant/participant-impl/pom.xml +++ b/participant/participant-impl/pom.xml @@ -41,6 +41,7 @@ <module>participant-impl-kubernetes</module> <module>participant-impl-http</module> <module>participant-impl-a1pms</module> + <module>participant-impl-kserve</module> </modules> <dependencies> |