Add CSITs for Tracing

Added custom start scripts for acm and participants
Added config for tracing to acm and participants
Added CSITs for Jaeger tracing of acm and participants

Issue-ID: POLICY-5062
Change-Id: I58c4dcc85d119d5dc3a74c39b786491f080bafdc
Signed-off-by: saul.gill <saul.gill@est.tech>

3 files changed, 67 insertions, 1 deletions

diff --git a/csit/resources/scripts/run-test.sh b/csit/resources/scripts/run-test.sh
index fb7d30bb..1e756f6d 100755
--- a/csit/resources/scripts/run-test.sh
+++ b/csit/resources/scripts/run-test.sh
@@ -41,6 +41,7 @@ DISTRIBUTION_IP=policy-distribution:${DEFAULT_PORT}
 
 POLICY_RUNTIME_ACM_IP=policy-clamp-runtime-acm:${DEFAULT_PORT}
 POLICY_PARTICIPANT_SIM_IP=policy-clamp-ac-sim-ppnt:${DEFAULT_PORT}
+JAEGER_IP=jaeger:16686
 
 KAFKA_IP=kafka:9092
 PROMETHEUS_IP=prometheus:9090
@@ -63,7 +64,8 @@ ROBOT_VARIABLES="-v DATA:${DATA}
 -v DROOLS_IP_2:${DROOLS_IP_2}
 -v TEMP_FOLDER:${DIST_TEMP_FOLDER}
 -v DISTRIBUTION_IP:${DISTRIBUTION_IP}
--v TEST_ENV:${TEST_ENV}"
+-v TEST_ENV:${TEST_ENV}
+-v JAEGER_IP:${JAEGER_IP}"
 
 export ROBOT_VARIABLES
 
diff --git a/csit/resources/tests/common-library.robot b/csit/resources/tests/common-library.robot
index bee80a90..a585bc97 100644
--- a/csit/resources/tests/common-library.robot
+++ b/csit/resources/tests/common-library.robot
@@ -115,6 +115,36 @@ GetMetrics
     Log  Received response from policy ${resp.text}
     RETURN  ${resp}
 
+VerifyTracingWorks
+    [Arguments]  ${domain}    ${service}
+    Log  Creating session http://${domain}
+    ${session}=  Create Session  jaeger  http://${domain}
+    ${resp}=  GET On Session  jaeger  /api/traces  params=service=${service}    expected_status=200
+    Log  Received response from jaeger ${resp.text}
+    RETURN  ${resp}
+
+VerifyKafkaInTraces
+    [Arguments]  ${domain}    ${service}
+    Log  Creating session http://${domain}
+    ${session}=  Create Session  jaeger  http://${domain}
+    ${tags}=    Create Dictionary    otel.library.name=io.opentelemetry.kafka-clients-2.6    messaging.system=kafka
+    ${tags_json}=    evaluate    json.dumps(${tags})    json
+    ${params}=    Create Dictionary    service=${service}    tags=${tags_json}    operation=policy-acruntime-participant send    lookback=1h    limit=10
+    ${resp}=  GET On Session  jaeger  /api/traces  params=${params}    expected_status=200
+    Log  Received response from jaeger ${resp.text}
+    RETURN  ${resp}
+
+VerifyHttpInTraces
+    [Arguments]  ${domain}    ${service}
+    Log  Creating session http://${domain}
+    ${session}=  Create Session  jaeger  http://${domain}
+    ${tags}=    Create Dictionary    uri=/v2/compositions/{compositionId}
+    ${tags_json}=    evaluate    json.dumps(${tags})    json
+    ${params}=    Create Dictionary    service=${service}    tags=${tags_json}    operation=http put /v2/compositions/{compositionId}    lookback=1h    limit=10
+    ${resp}=  GET On Session  jaeger  /api/traces  params=${params}    expected_status=200
+    Log  Received response from jaeger ${resp.text}
+    RETURN  ${resp}
+
 QueryPrometheus
     [Arguments]  ${query}
     ${params}=  Create Dictionary  query=${query}
diff --git a/csit/resources/tests/policy-clamp-test.robot b/csit/resources/tests/policy-clamp-test.robot
index ca7cf7d1..9d69520a 100644
--- a/csit/resources/tests/policy-clamp-test.robot
+++ b/csit/resources/tests/policy-clamp-test.robot
@@ -5,6 +5,7 @@ Library     OperatingSystem
 Library     String
 Library     json
 Library     yaml
+Resource    common-library.robot
 
 *** Test Cases ***
 HealthcheckAcm
@@ -159,6 +160,39 @@ DeployAutomationComposition
      Should Be Equal As Strings    ${resp.status_code}     202
      Wait Until Keyword Succeeds    10 min    5 sec    VerifyDeployStatus  ${compositionId}  ${instanceId}  DEPLOYED
 
+CheckTraces
+     [Documentation]    Verify that traces are being recorded in jaeger
+     Log    Verifying Jaeger traces
+     ${acmResp}=    VerifyTracingWorks    ${JAEGER_IP}    acm-r
+     ${httpResp}=    VerifyTracingWorks    ${JAEGER_IP}    http-ppnt
+     ${policyResp}=    VerifyTracingWorks    ${JAEGER_IP}    policy-ppnt
+     ${k8sResp}=    VerifyTracingWorks    ${JAEGER_IP}    k8s-ppnt
+     Should Not Be Empty    ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Log  Received spanID is ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${httpResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${policyResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${k8sResp.json()["data"][0]["spans"][0]["spanID"]}
+
+CheckKafkaPresentInTraces
+     [Documentation]    Verify that kafka traces are being recorded in jaeger
+     Log    Verifying Kafka Jaeger traces
+     ${acmResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    acm-r
+     ${httpResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    http-ppnt
+     ${policyResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    policy-ppnt
+     ${k8sResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    k8s-ppnt
+     Should Not Be Empty    ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Log  Received spanID is ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${httpResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${policyResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${k8sResp.json()["data"][0]["spans"][0]["spanID"]}
+
+CheckHttpPresentInAcmTraces
+     [Documentation]    Verify that http traces are being recorded in jaeger
+     Log    Verifying Http Jaeger traces
+     ${acmResp}=    VerifyHttpInTraces    ${JAEGER_IP}    acm-r
+     Should Not Be Empty    ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Log  Received spanID is ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+
 QueryPolicies
      [Documentation]    Verify the new policies deployed
      ${auth}=    Create List    policyadmin    zb!XztG34