Improve debugging support for CSITs

The following improvements are added:
- "docker ps" command added to the wait_for_port.sh script
  tp show status of the containers coming up and down
- "docker-compose logs" added to the end of the test runs
  to print out the standard output from the containers
- Add standard output logging to all containers
- Some cleanup and use of better names

Issue-ID: POLICY-4350
Change-Id: I3ae8c59dba64f9c267f310366bb4a2a6ffd5d418
Signed-off-by: liamfallon <liam.fallon@est.tech>

1 files changed, 219 insertions, 0 deletions

diff --git a/csit/config/clamp/logback-backend.xml b/csit/config/clamp/logback-backend.xml
new file mode 100644
index 00000000..cfa280c4
--- /dev/null
+++ b/csit/config/clamp/logback-backend.xml
@@ -0,0 +1,219 @@
+<included>
+    <jmxConfigurator />
+    <!-- Example evaluator filter applied against console appender -->
+    <property name="p_tim" value="%d{&quot;yyyy-MM-dd'T'HH:mm:ss.SSSXXX&quot;, UTC}" />
+    <property name="p_lvl" value="%level" />
+    <property name="p_log" value="%logger" />
+    <property name="p_mdc" value="%replace(%replace(%mdc){'\t','\\\\t'}){'\n', '\\\\n'}" />
+    <property name="p_msg" value="%replace(%replace(%msg){'\t', '\\\\t'}){'\n','\\\\n'}" />
+    <property name="p_exc" value="%replace(%replace(%rootException){'\t', '\\\\t'}){'\n','\\\\n'}" />
+    <property name="p_mak" value="%replace(%replace(%marker){'\t', '\\\\t'}){'\n','\\\\n'}" />
+    <property name="p_thr" value="%thread" />
+    <property name="defaultPattern"
+        value="%nopexception${p_tim}\t${p_thr}\t${p_lvl}\t${p_log}\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n" />
+    <property name="debugPattern" value="%nopexception${p_tim}|${p_lvl}|${p_mdc}|${p_exc}|%msg%n" />
+
+    <property name="errorPattern"
+        value="%X{EntryTimestamp}|%X{RequestID}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}||||%msg%n" />
+
+    <property name="auditPattern"
+        value="%X{EntryTimestamp}|%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX,UTC}|%X{RequestID}|%X{ServiceInstanceId}|%thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{Severity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||%marker|%mdc|||%msg%n" />
+
+    <property name="metricPattern"
+        value="%X{InvokeTimestamp}|%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX,UTC}|%X{RequestID}|%X{ServiceInstanceId}|%thread||%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{Severity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}||||%X{TargetVirtualEntity}|%marker|%mdc|||%msg%n" />
+
+
+    <!-- Example evaluator filter applied against console appender -->
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>INFO</level>
+        </filter>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{1024} - %msg%n
+            </pattern>
+        </encoder>
+    </appender>
+
+    <appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>ERROR</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+        <file>${logDirectory}/error.log</file>
+        <append>true</append>
+        <encoder>
+            <pattern>${errorPattern}</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${logDirectory}/error.%d{yyyy-MM-dd}.%i.log.zip
+            </fileNamePattern>
+            <maxFileSize>50MB</maxFileSize>
+            <maxHistory>20</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
+    </appender>
+    <appender name="asyncError" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>256</queueSize>
+        <appender-ref ref="ERROR" />
+    </appender>
+
+    <appender name="DEBUG" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${logDirectory}/debug.log</file>
+        <append>true</append>
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{1024} - %msg%n
+            </pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${logDirectory}/debug.%d{yyyy-MM-dd}.%i.log.zip
+            </fileNamePattern>
+            <maxFileSize>50MB</maxFileSize>
+            <maxHistory>20</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
+    </appender>
+    <appender name="asyncDebug" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>256</queueSize>
+        <appender-ref ref="DEBUG" />
+    </appender>
+
+    <appender name="AUDIT" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="org.onap.policy.common.utils.logging.LoggerMarkerFilter$AuditLoggerMarkerFilter" />
+        <file>${logDirectory}/audit.log</file>
+        <encoder>
+            <pattern>${auditPattern}</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <FileNamePattern>${logDirectory}/audit.%d{yyyy-MM-dd}.%i.log.zip
+            </FileNamePattern>
+            <maxFileSize>50MB</maxFileSize>
+            <maxHistory>20</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
+    </appender>
+    <appender name="asyncAudit" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>256</queueSize>
+        <appender-ref ref="AUDIT" />
+    </appender>
+
+    <appender name="METRIC" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="org.onap.policy.common.utils.logging.LoggerMarkerFilter$MetricLoggerMarkerFilter" />
+        <file>${logDirectory}/metric.log</file>
+        <encoder>
+            <pattern>${metricPattern}</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <FileNamePattern>${logDirectory}/metric.%d{yyyy-MM-dd}.%i.log.zip
+            </FileNamePattern>
+            <maxFileSize>50MB</maxFileSize>
+            <maxHistory>20</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
+    </appender>
+    <appender name="asyncMetric" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>256</queueSize>
+        <appender-ref ref="METRIC" />
+    </appender>
+
+    <!-- SECURITY related loggers -->
+    <appender name="SECURITY" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="org.onap.policy.common.utils.logging.LoggerMarkerFilter$SecurityLoggerMarkerFilter" />
+        <file>${logDirectory}/security.log</file>
+        <encoder>
+            <pattern>%X{EntryTimestamp}|%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX,UTC}|%X{RequestID}|%X{ServiceInstanceId}|%.20thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}|%msg%n
+            </pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <FileNamePattern>${logDirectory}/security.%d{yyyy-MM-dd}.%i.log.zip
+            </FileNamePattern>
+            <maxFileSize>50MB</maxFileSize>
+            <maxHistory>20</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
+    </appender>
+    <appender name="asyncSecurity" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>256</queueSize>
+        <appender-ref ref="SECURITY" />
+    </appender>
+
+    <!-- logback jms appenders & loggers definition starts here -->
+    <appender name="auditLogs" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+        </filter>
+        <file>${logDirectory}/Audits.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${logDirectory}/Audits-%d{yyyy-MM-dd}.%i.log.zip
+            </fileNamePattern>
+            <maxFileSize>50MB</maxFileSize>
+            <maxHistory>20</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>"%d [%thread] %-5level %logger{1024} - %msg%n"</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="perfLogs" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+        </filter>
+        <file>${logDirectory}/Perform.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${logDirectory}/Perform--%d{yyyy-MM-dd}.%i.log.zip
+            </fileNamePattern>
+            <maxFileSize>50MB</maxFileSize>
+            <maxHistory>20</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>"%d [%thread] %-5level %logger{1024} - %msg%n"</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="org.onap.aaf" level="DEBUG">
+        <appender-ref ref="DEBUG" />
+    </logger>
+    <logger name="org.hibernate" level="INFO">
+        <appender-ref ref="DEBUG" />
+    </logger>
+    <logger name="org.apache.camel.Tracing" level="INFO">
+        <appender-ref ref="DEBUG" />
+    </logger>
+
+    <logger name="org.apache" level="INFO">
+        <appender-ref ref="DEBUG" />
+    </logger>
+    <!-- Spring related loggers -->
+    <logger name="org.springframework" level="INFO">
+        <appender-ref ref="DEBUG" />
+    </logger>
+
+    <!-- CLAMP related loggers -->
+    <logger name="org.onap.policy.clamp" level="DEBUG">
+        <appender-ref ref="ERROR" />
+        <appender-ref ref="DEBUG" />
+    </logger>
+
+    <!-- logback internals logging -->
+    <logger name="ch.qos.logback.classic" level="INFO" />
+    <logger name="ch.qos.logback.core" level="INFO" />
+    <logger name="AuditRecord" level="INFO" additivity="false">
+        <appender-ref ref="auditLogs" />
+    </logger>
+    <logger name="AuditRecord_DirectCall" level="INFO" additivity="false">
+        <appender-ref ref="auditLogs" />
+    </logger>
+    <logger name="PerfTrackerRecord" level="INFO" additivity="false">
+        <appender-ref ref="perfLogs" />
+    </logger>
+    <!-- logback jms appenders & loggers definition ends here -->
+
+    <root level="DEBUG">
+        <appender-ref ref="asyncDebug" />
+        <appender-ref ref="asyncError" />
+        <appender-ref ref="asyncAudit" />
+        <appender-ref ref="asyncMetric" />
+        <appender-ref ref="asyncSecurity" />
+        <appender-ref ref="STDOUT" />
+    </root>
+</included>