diff options
author | ramverma <ram.krishna.verma@ericsson.com> | 2018-09-21 11:09:58 +0100 |
---|---|---|
committer | ramverma <ram.krishna.verma@ericsson.com> | 2018-09-21 17:40:10 +0100 |
commit | 6931309e2969dbf7be71777f048dcab4830d1c05 (patch) | |
tree | 4c1b51e9cc2bbce0a28460a21564380c3bac818a | |
parent | b15571184fabee50986f38f40577b87281e8765c (diff) |
Adding retry logic for SDC Client in Distribution
* SdcReceptionHandler in distribution tries to initialize & start the
SDC Client when distribution framework comes up. If by any chance the
SDC service is not up then currently the SdcReceptionHandler throws an
exception to the distribution activator and the distribution framework
goes down. And the health check fails.
* This review fixes it by adding a retry mechanism in
SdcReceptionHandler for all lifecycle methods of SdcClient (init, start,
stop). After failure the same operation is retried again with some
delay. The delay is passed as parameter from configuration json file.
The minimum default value for the delay is kept as 30 seconds.
* Adding SdcClientHandler timer task for performing the retries
asynchronously.
Change-Id: Ibb6d936fcf4872c82f87e2cd04a00583b81c92ff
Issue-ID: POLICY-1035
Signed-off-by: ramverma <ram.krishna.verma@ericsson.com>
7 files changed, 187 insertions, 80 deletions
diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java new file mode 100644 index 00000000..f69fb155 --- /dev/null +++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java @@ -0,0 +1,76 @@ +/*- + * ============LICENSE_START======================================================= + * Copyright (C) 2018 Ericsson. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.policy.distribution.reception.handling.sdc; + +import java.util.Timer; +import java.util.TimerTask; + +/** + * This class implements TimerTask for calling life cycle methods of SdcClient iteratively after specified interval + * until the operation is successful. + * + * @author Ram Krishna Verma (ram.krishna.verma@ericsson.com) + */ +public class SdcClientHandler extends TimerTask { + + private SdcReceptionHandler sdcReceptionHandler; + private SdcClientOperationType operationType; + private Timer timer; + + public enum SdcClientOperationType { + START, STOP + } + + /** + * Constructs an instance of {@link SdcClientHandler} class. + * + * @param sdcReceptionHandler the sdcReceptionHandler + */ + public SdcClientHandler(final SdcReceptionHandler sdcReceptionHandler, final SdcClientOperationType operationType, + final long retryDelay) { + this.sdcReceptionHandler = sdcReceptionHandler; + this.operationType = operationType; + timer = new Timer(false); + timer.scheduleAtFixedRate(this, 0, retryDelay * 1000L); + } + + /** + * {@inheritDoc}. + */ + @Override + public void run() { + if (SdcClientOperationType.START.equals(operationType)) { + sdcReceptionHandler.initializeSdcClient(); + sdcReceptionHandler.startSdcClient(); + } else { + sdcReceptionHandler.stopSdcClient(); + } + } + + /** + * {@inheritDoc}. + */ + @Override + public boolean cancel() { + timer.cancel(); + return true; + } +} diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java index 443235b7..2ada9fcd 100644 --- a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java +++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java @@ -30,10 +30,9 @@ import org.onap.policy.common.logging.flexlogger.FlexLogger; import org.onap.policy.common.logging.flexlogger.Logger; import org.onap.policy.common.parameters.ParameterService; import org.onap.policy.distribution.model.Csar; -import org.onap.policy.distribution.reception.decoding.PluginInitializationException; -import org.onap.policy.distribution.reception.decoding.PluginTerminationException; import org.onap.policy.distribution.reception.decoding.PolicyDecodingException; import org.onap.policy.distribution.reception.handling.AbstractReceptionHandler; +import org.onap.policy.distribution.reception.handling.sdc.SdcClientHandler.SdcClientOperationType; import org.onap.policy.distribution.reception.handling.sdc.exceptions.ArtifactDownloadException; import org.onap.policy.distribution.reception.statistics.DistributionStatisticsManager; import org.onap.sdc.api.IDistributionClient; @@ -54,38 +53,34 @@ import org.onap.sdc.utils.DistributionStatusEnum; public class SdcReceptionHandler extends AbstractReceptionHandler implements INotificationCallback { private static final Logger LOGGER = FlexLogger.getLogger(SdcReceptionHandler.class); + private static final String SECONDS = "Seconds"; private SdcReceptionHandlerStatus sdcReceptionHandlerStatus = SdcReceptionHandlerStatus.STOPPED; - private SdcReceptionHandlerConfigurationParameterGroup handlerParameters; private IDistributionClient distributionClient; private SdcConfiguration sdcConfig; private volatile int nbOfNotificationsOngoing = 0; + private int retryDelay; + private SdcClientHandler sdcClientHandler; private enum DistributionStatusType { DOWNLOAD, DEPLOY } @Override - protected void initializeReception(final String parameterGroupName) throws PluginInitializationException { - handlerParameters = ParameterService.get(parameterGroupName); - initializeSdcClient(); - startSdcClient(); + protected void initializeReception(final String parameterGroupName) { + final SdcReceptionHandlerConfigurationParameterGroup handlerParameters = + ParameterService.get(parameterGroupName); + retryDelay = handlerParameters.getRetryDelay() < 30 ? 30 : handlerParameters.getRetryDelay(); + sdcConfig = new SdcConfiguration(handlerParameters); + distributionClient = createSdcDistributionClient(); + sdcClientHandler = new SdcClientHandler(this, SdcClientOperationType.START, retryDelay); } @Override - public void destroy() throws PluginTerminationException { - LOGGER.debug("Going to stop the SDC Client..."); + public void destroy() { if (distributionClient != null) { - final IDistributionClientResult clientResult = distributionClient.stop(); - if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) { - final String message = - "SDC client stop failed with reason:" + clientResult.getDistributionMessageResult(); - LOGGER.error(message); - throw new PluginTerminationException(message); - } + sdcClientHandler = new SdcClientHandler(this, SdcClientOperationType.STOP, retryDelay); } - changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED); - LOGGER.debug("SDC Client is stopped successfully"); } @Override @@ -115,6 +110,8 @@ public class SdcReceptionHandler extends AbstractReceptionHandler implements INo ++nbOfNotificationsOngoing; sdcReceptionHandlerStatus = newStatus; break; + default: + break; } } @@ -130,48 +127,61 @@ public class SdcReceptionHandler extends AbstractReceptionHandler implements INo /** * Method to initialize the SDC client. * - * @throws PluginInitializationException if the initialization of SDC Client fails */ - private void initializeSdcClient() throws PluginInitializationException { + protected void initializeSdcClient() { LOGGER.debug("Initializing the SDC Client..."); if (sdcReceptionHandlerStatus != SdcReceptionHandlerStatus.STOPPED) { - final String message = "The SDC Client is already initialized"; - LOGGER.error(message); - throw new PluginInitializationException(message); + LOGGER.error("The SDC Client is already initialized"); + return; } - sdcConfig = new SdcConfiguration(handlerParameters); - distributionClient = createSdcDistributionClient(); final IDistributionClientResult clientResult = distributionClient.init(sdcConfig, this); if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) { - changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED); - final String message = - "SDC client initialization failed with reason:" + clientResult.getDistributionMessageResult(); - LOGGER.error(message); - throw new PluginInitializationException(message); + LOGGER.error("SDC client initialization failed with reason:" + clientResult.getDistributionMessageResult() + + ". Initialization will be retried after " + retryDelay + SECONDS); + return; } LOGGER.debug("SDC Client is initialized successfully"); - this.changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.INIT); + changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.INIT); } /** * Method to start the SDC client. * - * @param configParameter the configuration parameters - * @throws PluginInitializationException if the start of SDC Client fails */ - private void startSdcClient() throws PluginInitializationException { + protected void startSdcClient() { LOGGER.debug("Going to start the SDC Client..."); + if (sdcReceptionHandlerStatus != SdcReceptionHandlerStatus.INIT) { + LOGGER.error("The SDC Client is not initialized"); + return; + } final IDistributionClientResult clientResult = distributionClient.start(); if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) { - changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED); - final String message = "SDC client start failed with reason:" + clientResult.getDistributionMessageResult(); - LOGGER.error(message); - throw new PluginInitializationException(message); + LOGGER.error("SDC client start failed with reason:" + clientResult.getDistributionMessageResult() + + ". Start will be retried after " + retryDelay + SECONDS); + return; } LOGGER.debug("SDC Client is started successfully"); - this.changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.IDLE); + changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.IDLE); + sdcClientHandler.cancel(); + } + + /** + * Method to stop the SDC client. + * + */ + protected void stopSdcClient() { + LOGGER.debug("Going to stop the SDC Client..."); + final IDistributionClientResult clientResult = distributionClient.stop(); + if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) { + LOGGER.error("SDC client stop failed with reason:" + clientResult.getDistributionMessageResult() + + ". Stop will be retried after " + retryDelay + SECONDS); + return; + } + LOGGER.debug("SDC Client is stopped successfully"); + changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED); + sdcClientHandler.cancel(); } /** diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java index 34eaf33e..65305c1c 100644 --- a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java +++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java @@ -44,6 +44,7 @@ public class SdcReceptionHandlerConfigurationParameterBuilder { private String keystorePassword; private List<String> messageBusAddress; private List<String> artifactTypes; + private int retryDelay; /** * Set activeserverTlsAuth to this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance. @@ -197,6 +198,16 @@ public class SdcReceptionHandlerConfigurationParameterBuilder { } /** + * Set retryDelay to this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance. + * + * @param retryDelay the retryDelay + */ + public SdcReceptionHandlerConfigurationParameterBuilder setRetryDelay(final int retryDelay) { + this.retryDelay = retryDelay; + return this; + } + + /** * Returns the active server TlsAuth of this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance. * * @return the activeserverTlsAuth @@ -332,6 +343,15 @@ public class SdcReceptionHandlerConfigurationParameterBuilder { return artifactTypes; } + /** + * Returns the retryDelay of this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance. + * + * @return the retryDelay + */ + public int getRetryDelay() { + return retryDelay; + } + } diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java index 0140fb53..bdc9767d 100644 --- a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java +++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java @@ -21,15 +21,15 @@ package org.onap.policy.distribution.reception.handling.sdc; import java.util.List; + import org.onap.policy.common.parameters.GroupValidationResult; import org.onap.policy.common.parameters.ValidationStatus; import org.onap.policy.common.utils.validation.ParameterValidationUtils; import org.onap.policy.distribution.reception.parameters.ReceptionHandlerConfigurationParameterGroup; /** - * This class handles reading, parsing and validating of the Policy SDC Service Distribution - * parameters from Json format, which strictly adheres to the interface:IConfiguration, defined by - * SDC SDK. + * This class handles reading, parsing and validating of the Policy SDC Service Distribution parameters from Json + * format, which strictly adheres to the interface:IConfiguration, defined by SDC SDK. */ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHandlerConfigurationParameterGroup { @@ -39,6 +39,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan private String password; private int pollingInterval; private int pollingTimeout; + private int retryDelay; private String consumerId; private List<String> artifactTypes; private String consumerGroup; @@ -50,8 +51,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan private boolean isUseHttpsWithDmaap; /** - * The constructor for instantiating {@link SdcReceptionHandlerConfigurationParameterGroup} - * class. + * The constructor for instantiating {@link SdcReceptionHandlerConfigurationParameterGroup} class. * * @param builder the SDC configuration builder */ @@ -63,6 +63,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan password = builder.getPassword(); pollingInterval = builder.getPollingInterval(); pollingTimeout = builder.getPollingTimeout(); + retryDelay = builder.getRetryDelay(); consumerId = builder.getConsumerId(); artifactTypes = builder.getArtifactTypes(); consumerGroup = builder.getConsumerGroup(); @@ -99,6 +100,10 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan return pollingTimeout; } + public int getRetryDelay() { + return retryDelay; + } + public String getConsumerId() { return consumerId; } @@ -149,6 +154,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan validateStringElement(validationResult, keyStorePassword, "keyStorePassword"); validateIntElement(validationResult, pollingInterval, "pollingInterval"); validateIntElement(validationResult, pollingTimeout, "pollingTimeout"); + validateIntElement(validationResult, retryDelay, "retryDelay"); validateStringListElement(validationResult, messageBusAddress, "messageBusAddress"); validateStringListElement(validationResult, artifactTypes, "artifactTypes"); return validationResult; diff --git a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java index bf42476f..557a7683 100644 --- a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java +++ b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java @@ -51,7 +51,6 @@ import org.onap.policy.distribution.forwarding.PolicyForwarder; import org.onap.policy.distribution.forwarding.parameters.PolicyForwarderParameters; import org.onap.policy.distribution.model.Csar; import org.onap.policy.distribution.reception.decoding.PluginInitializationException; -import org.onap.policy.distribution.reception.decoding.PluginTerminationException; import org.onap.policy.distribution.reception.decoding.PolicyDecoder; import org.onap.policy.distribution.reception.handling.AbstractReceptionHandler; import org.onap.policy.distribution.reception.handling.PluginHandler; @@ -97,9 +96,14 @@ public class TestSdcReceptionHandler { * Setup for the test cases. * * @throws IOException if it occurs + * @throws SecurityException if it occurs + * @throws NoSuchFieldException if it occurs + * @throws IllegalAccessException if it occurs + * @throws IllegalArgumentException if it occurs */ @Before - public final void init() throws IOException { + public final void init() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, + IllegalAccessException { DistributionStatisticsManager.resetAllStatistics(); final Gson gson = new GsonBuilder().create(); pssdConfigParameters = gson.fromJson(new FileReader("src/test/resources/handling-sdc.json"), @@ -140,45 +144,34 @@ public class TestSdcReceptionHandler { public final void testInitializeSdcClient() { try { sypHandler.initializeReception(pssdConfigParameters.getName()); - } catch (final PluginInitializationException exp) { + } catch (final Exception exp) { LOGGER.error(exp); fail("Test should not throw any exception"); } } @Test - public final void testInitializeSdcClient_Again() throws PluginInitializationException { - sypHandler.initializeReception(pssdConfigParameters.getName()); - try { - sypHandler.initializeReception(pssdConfigParameters.getName()); - fail("Test must throw an exception here"); - } catch (final Exception exp) { - assertTrue(exp.getMessage().startsWith("The SDC Client is already initialized")); - } - } - - @Test - public final void testInitializeSdcClient_Failure() throws PluginInitializationException { + public final void testInitializeSdcClient_Failure() { Mockito.when(successfulClientInitResult.getDistributionActionResult()) - .thenReturn(DistributionActionResultEnum.FAIL); + .thenReturn(DistributionActionResultEnum.FAIL).thenReturn(DistributionActionResultEnum.SUCCESS); try { sypHandler.initializeReception(pssdConfigParameters.getName()); - fail("Test must throw an exception here"); } catch (final Exception exp) { - assertTrue(exp.getMessage().startsWith("SDC client initialization failed with reason")); + LOGGER.error(exp); + fail("Test should not throw any exception"); } } @Test - public final void testStartSdcClient_Failure() throws PluginInitializationException { + public final void testStartSdcClient_Failure() { try { - Mockito.when(distributionClient.start()).thenReturn(failureClientInitResult); + Mockito.when(distributionClient.start()).thenReturn(failureClientInitResult) + .thenReturn(successfulClientInitResult); sypHandler.initializeReception(pssdConfigParameters.getName()); - - fail("Test must throw an exception here"); } catch (final Exception exp) { - assertTrue(exp.getMessage().startsWith("SDC client start failed with reason")); + LOGGER.error(exp); + fail("Test should not throw any exception"); } } @@ -187,7 +180,7 @@ public class TestSdcReceptionHandler { try { sypHandler.initializeReception(pssdConfigParameters.getName()); sypHandler.destroy(); - } catch (final PluginInitializationException | PluginTerminationException exp) { + } catch (final Exception exp) { LOGGER.error(exp); fail("Test should not throw any exception"); } @@ -195,28 +188,28 @@ public class TestSdcReceptionHandler { } @Test - public final void testStopSdcClientWithoutStart() { + public final void testStopSdcClient_Failure() throws PluginInitializationException { + + sypHandler.initializeReception(pssdConfigParameters.getName()); + Mockito.when(distributionClient.stop()).thenReturn(failureClientInitResult) + .thenReturn(successfulClientInitResult); try { sypHandler.destroy(); - } catch (final PluginTerminationException exp) { + } catch (final Exception exp) { LOGGER.error(exp); fail("Test should not throw any exception"); } - } @Test - public final void testStopSdcClient_Failure() throws PluginInitializationException { - - sypHandler.initializeReception(pssdConfigParameters.getName()); - Mockito.when(successfulClientInitResult.getDistributionActionResult()) - .thenReturn(DistributionActionResultEnum.FAIL); + public final void testStopSdcClientWithoutStart() { try { sypHandler.destroy(); - fail("Test must throw an exception here"); } catch (final Exception exp) { - assertTrue(exp.getMessage().startsWith("SDC client stop failed with reason")); + LOGGER.error(exp); + fail("Test should not throw any exception"); } + } @Test diff --git a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java index df72d45c..f9a65c54 100644 --- a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java +++ b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java @@ -92,8 +92,8 @@ public class TestSdcReceptionHandlerConfigurationParameterGroup { new SdcReceptionHandlerConfigurationParameterBuilder().setAsdcAddress("localhost") .setConsumerGroup("policy-group").setConsumerId("policy-id").setEnvironmentName("TEST") .setKeystorePassword("password").setKeystorePath("dummyPath").setPassword("policy") - .setPollingInterval(10).setPollingTimeout(20).setUser("policy").setUseHttpsWithDmaap(false) - .setActiveserverTlsAuth(false).setFilterinEmptyResources(true) + .setPollingInterval(10).setPollingTimeout(20).setRetryDelay(30).setUser("policy") + .setUseHttpsWithDmaap(false).setActiveserverTlsAuth(false).setFilterinEmptyResources(true) .setArtifactTypes(Arrays.asList("TOSCA_CSAR")).setMessageBusAddress(Arrays.asList("localhost")); final SdcReceptionHandlerConfigurationParameterGroup configParameters = new SdcReceptionHandlerConfigurationParameterGroup(builder); @@ -106,6 +106,7 @@ public class TestSdcReceptionHandlerConfigurationParameterGroup { assertEquals("policy", configParameters.getPassword()); assertEquals(10, configParameters.getPollingInterval()); assertEquals(20, configParameters.getPollingTimeout()); + assertEquals(30, configParameters.getRetryDelay()); assertEquals("policy-id", configParameters.getConsumerId()); assertEquals("policy-group", configParameters.getConsumerGroup()); assertEquals("TEST", configParameters.getEnvironmentName()); diff --git a/plugins/reception-plugins/src/test/resources/handling-sdc.json b/plugins/reception-plugins/src/test/resources/handling-sdc.json index 7e3758de..0bb936bf 100644 --- a/plugins/reception-plugins/src/test/resources/handling-sdc.json +++ b/plugins/reception-plugins/src/test/resources/handling-sdc.json @@ -10,6 +10,7 @@ "password": "policy", "pollingInterval":20, "pollingTimeout":30, + "retryDelay":30, "consumerId": "policy-id", "artifactTypes": [ "TOSCA_CSAR", |