Update doc/tools

Issue-ID: DOC-745

Signed-off-by: thmsdt <thomas.kulik@telekom.de>
Change-Id: I5ede64f51d5beb82d146c412a0915e1e59cd9779

4 files changed, 241 insertions, 11 deletions

diff --git a/tools/checkdocs.sh b/tools/checkdocs.sh
index 66cbd4c47..ca217d488 100755
--- a/tools/checkdocs.sh
+++ b/tools/checkdocs.sh
@@ -58,7 +58,7 @@
 ### SHORT: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d'
 ###
 
-script_version="1.7 (2021-04-12)"
+script_version="1.9 (2021-05-31)"
 
 # save command for the restart with logging enabled
 command=$0
@@ -366,8 +366,11 @@ do
         printf "\nconf.py files:\n"
         find ./$reponame -type f -name conf.py | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_confpy.log
 
-        printf "\nindex.rst files:\n"
-        find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst.log
+        printf "\nindex.rst files (all):\n"
+        find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst_all.log
+
+        printf "\nindex.rst files (docs root directory):\n"
+        find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | grep ']/docs/index.rst' | tee -a ${branch}_indexrst_docs_root.log
 
         printf "\nINFO.yaml files:\n"
         find ./$reponame -type f -name INFO.yaml | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_infoyaml.log
@@ -381,6 +384,12 @@ do
 
   done <${repolist}
 
+  # get (first) title for a rst file
+  drawline
+  python3 ../getrsttitle.py ${branch}_rstfiles.log | tee ${branch}_rstfiles_titles.log
+  drawline
+  python3 ../getrsttitle.py ${branch}_indexrst_docs_root.log | tee ${branch}_indexrst_docs_root_titles.log
+
   # examine repos
   drawline
   find . -type f -name values.yaml -print -exec grep "image:" {} \; | sed -r 's:^ +::' | tee ${branch}_dockerimagesfull.log
@@ -637,6 +646,7 @@ do
   # csv column #11: conf.py
   # csv column #12: tox.ini
   # csv column #13: index.rst
+  # csv column #14: first title in index.rst
   #
   # columns are filled with values from requested branch.
   # if data is not available values from master branch are used.
@@ -645,7 +655,7 @@ do
 
   readarray -t array < ./${repolist};
   i=0
-  csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst"
+  csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst,first title in index.rst"
   ((i++))
   for line in "${array[@]}"
   do
@@ -699,13 +709,16 @@ do
       docs="${docs},-"
     fi
 
-    # index.rst
+    # index.rst, first title in index.rst
+    indexrsttitle=""
     if [ -f ./${line}/docs/index.rst ] ; then
-      docs="${docs},index.rst"
+      indexrsttitle=$(cat ${branch}_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $2}');
+      docs="${docs},index.rst,${indexrsttitle}"
     elif [ -f ../master/${line}/docs/index.rst ] ; then
-      docs="${docs},(index.rst)"
+      indexrsttitle=$(cat ../master/master_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $2}');
+      docs="${docs},(index.rst),(${indexrsttitle})"
     else
-      docs="${docs},-"
+      docs="${docs},-,-"
     fi
 
     #echo "DBUG: docs=${docs}"
@@ -718,8 +731,8 @@ do
   unset docs
 
   #
-  # csv column #14: index.html@RTD accessibility check
-  # csv column #15: index.html url
+  # csv column #15: index.html@RTD accessibility check
+  # csv column #16: index.html url
   #
 
   readarray -t array < ./${branch}_repoclone.log;
@@ -832,7 +845,7 @@ do
   done
 
   #
-  # csv column #16: release notes
+  # csv column #17: release notes
   #
 
   readarray -t array < ../${repolist};
diff --git a/tools/getrsttitle.py b/tools/getrsttitle.py
new file mode 100644
index 000000000..e852d9f92
--- /dev/null
+++ b/tools/getrsttitle.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+### ===========================================================================
+### Licensed under the Apache License, Version 2.0 (the "License");
+### you may not use this file except in compliance with the License.
+### You may obtain a copy of the License at
+###
+###       http://www.apache.org/licenses/LICENSE-2.0
+###
+### Unless required by applicable law or agreed to in writing, software
+### distributed under the License is distributed on an "AS IS" BASIS,
+### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+### See the License for the specific language governing permissions and
+### limitations under the License.
+###
+### Copyright (C) 2021 Deutsche Telekom AG
+### ============LICENSE_END====================================================
+
+#
+# getrsttitle.py
+# AUTHOR(S):
+# Thomas Kulik, Deutsche Telekom AG, 2021
+# DESCRIPTION:
+# Processes a list of rst files and retrieves the first title for every single rst file.
+# Copy program to {branch} directory of cloned ONAP documentation and run it.
+# USAGE:
+# python3 getrsttitle.py filename
+#
+# Helpful resources:
+# https://regex101.com/r/YNYK2Q/1/
+# https://stackoverflow.com/questions/20312443/how-to-find-title-a-la-restructuredtext
+#
+
+import re
+import os.path
+import sys
+import argparse
+
+#
+# argument handling
+#
+
+parser = argparse.ArgumentParser(description='Processes a list of rst files and retrieves the first title for every single rst file.')
+parser.add_argument('filename')
+args = parser.parse_args()
+
+# regex to find title underlined with various characters
+#regex1 = r"(?:^|\n)(?!\=)([^\n\r]+)\r?\n(\=+)(?:\r?\n| *$)"
+#regex2 = r"(?:^|\n)(?!\-)([^\n\r]+)\r?\n(\-+)(?:\r?\n| *$)"
+#regex3 = r"(?:^|\n)(?!\~)([^\n\r]+)\r?\n(\~+)(?:\r?\n| *$)"
+#regex4 = r"(?:^|\n)(?!\#)([^\n\r]+)\r?\n(\#+)(?:\r?\n| *$)"
+#regex5 = r"(?:^|\n)(?!\*)([^\n\r]+)\r?\n(\*+)(?:\r?\n| *$)"
+
+# there is a problem with raw strings (r"...") in the regex search below
+# workaround: using \\ to mask special characters in regex
+regex_list = [
+    "(?:^|\\n)(?!\\=)([^\\n\\r]+)\\r?\\n(\\=+)(?:\\r?\\n| *$)",
+    "(?:^|\\n)(?!\\-)([^\\n\\r]+)\\r?\\n(\\-+)(?:\\r?\\n| *$)",
+    "(?:^|\\n)(?!\\~)([^\\n\\r]+)\\r?\\n(\\~+)(?:\\r?\\n| *$)",
+    "(?:^|\\n)(?!\\#)([^\\n\\r]+)\\r?\\n(\\#+)(?:\\r?\\n| *$)",
+    "(?:^|\\n)(?!\\*)([^\\n\\r]+)\\r?\\n(\\*+)(?:\\r?\\n| *$)",
+    ]
+
+# DBUG only
+#for regex in regex_list:
+#    print(repr(regex))
+
+#filename = './master_indexrst_docs_root.log'
+#filename = './master_rstfiles.log'
+
+if os.path.isfile(args.filename):
+    with open(args.filename) as fn:
+        # read first line
+        line = fn.readline()
+        #print("DBUG: line={}".format(line))
+        file_cnt = 0
+        while line:
+            rstfile         = "./" + re.sub('\[|\]', '', line).strip()
+            repository_tmp1 = re.sub('\].+$', '',line).strip()
+            repository      = re.sub('\[', '',repository_tmp1).strip()
+            project_tmp1    = re.sub('\].+$', '',line).strip()
+            project_tmp2    = re.sub('\/.+$', '',project_tmp1).strip()
+            project         = re.sub('\[', '',project_tmp2).strip()
+            #print("DBUG:       file #{} {}".format(file_cnt, rstfile))
+            #print("DBUG: repository #{} {}".format(file_cnt, repository))
+            #print("DBUG:    project #{} {}".format(file_cnt, project))
+            file_cnt += 1
+            if os.path.isfile(rstfile):
+                with open(rstfile, 'r') as content:
+                    content_rstfile = content.read()
+                    #print("DBUG: content_rstfile = \n{}".format(content_rstfile))
+                    regex_cnt = 0
+                    for regex in regex_list:
+                        regex_cnt += 1
+                        m = re.search(regex, content_rstfile, re.MULTILINE)
+                        #print("DBUG: using regex  " + repr(regex))
+                        #print("DBUG: using regex1 " + repr(regex1))
+                        #print("DBUG: regex_cnt = {}".format(regex_cnt))
+                        if m:
+                            match = m.group(1)
+                            #print ("DBUG: |REGEX| {} |REGEXCNT| {} |FILECNT| {} |FILE| {} |MATCH| {}".format(repr(regex), regex_cnt, file_cnt, rstfile, match))
+                            # end regex loop if we have a title
+                            break
+                        else:
+                            match = "NO-TITLE-FOUND"
+                            #print ("DBUG: NO-TITLE-FOUND")
+            else:
+                print ("ERR:  File {} does not exist".format(rstfile))
+
+            #print ("DBUG: |REGEX| {} |REGEXCNT| {} |FILECNT| {} |FILE| {} |MATCH| {}".format(repr(regex), regex_cnt, file_cnt, rstfile, match))
+            #print ("DBUG: file #{} '{}' '{}'".format(file_cnt, rstfile, match))
+
+            # clean up result and print
+            match_1 = match.replace(",", "") # remove ,
+            match_final = match_1.strip()    # remove \n
+            print ("{},{},{},{}".format(project.strip(), repository.strip(), line.strip(), match_final.strip()))
+
+            # read next line and loop
+            line = fn.readline()
+else:
+    print ("ERR:  File {} does not exist".format(args.filename))
+
+sys.exit()
+
+#
+# example code to show detailed regex matches and group content
+# to be used in a future version of this program
+#
+# matches = re.finditer(regex2, content, re.MULTILINE)
+# for matchNum, match in enumerate(matches, start=1):
+#     print ("Match {matchNum} was found at {start}-{end}: {match}".format(matchNum = matchNum, start = match.start(), end = match.end(), match = match.group()))
+#     print ("{match}".format(match = match.group()))
+#     for groupNum in range(0, len(match.groups())):
+#         groupNum = groupNum + 1
+#         print ("Group {groupNum} found at {start}-{end}: {group}".format(groupNum = groupNum, start = match.start(groupNum), end = match.end(groupNum), group = match.group(groupNum)))
+# print ("Test:" "{group}".format(group = match.group(1)))
+#
+
+#
+# example code for pandas
+# to be used in a future version of this program
+#
+# import pandas as pd
+# pd.set_option('display.max_rows', 500)
+# pd.set_option('display.max_columns', 500)
+# pd.set_option('display.width', 1000)
+#
+# table = pd.read_csv("master_table.csv")
+# print(table)
+#
\ No newline at end of file
diff --git a/tools/latestbranch.sh b/tools/latestbranch.sh
new file mode 100755
index 000000000..ae9dd1fc0
--- /dev/null
+++ b/tools/latestbranch.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# set -x
+
+repolist=$1
+source="git://cloud.onap.org/mirror"
+
+  #
+  # csv column #nn: latest branch
+  #
+
+  readarray -t array < ./${repolist};
+  i=0
+  csv[i]="${csv[i]},MASTER repo name"
+  ((i++))
+  for line in "${array[@]}"
+  do
+    reponame=$(echo $line | awk -F "|" '{print $1}');
+    csv[i]="${csv[i]},${reponame}"
+    ((i++))
+    
+    #echo " "
+    #echo ${reponame}
+  
+    git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' | sed -nr '/^master$|^amsterdam$|^beijing$|^casablanca$|^dublin$|^elalto$|^frankfurt$|^guilin$|^honolulu$|^istanbul$/Ip' | tail -2 | head -1
+    #git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' 
+
+  done
+  unset array
+  unset i
+  unset reponame
+
+exit
+
diff --git a/tools/wiki_lifecycle_state_210517.txt b/tools/wiki_lifecycle_state_210517.txt
new file mode 100644
index 000000000..af4aad8e4
--- /dev/null
+++ b/tools/wiki_lifecycle_state_210517.txt
@@ -0,0 +1,34 @@
+PROJECT;WIKI STATE;WIKI NAME
+aaf;unmaintained;Application Authorization Framework Project
+aai;mature;Active and Available Inventory Project
+appc;unmaintained;Application Controller Project
+ccsdk;mature;Common Controller SDK Project
+ci-management;incubation;CI-Management
+clamp;incubation;CLAMP Project
+cli;incubation;Command Line Interface Project
+cps;incubation;Configuration Persistence Service Project
+dcaegen2;mature;Data Collection Analytics and Events Project
+dmaap;mature;Data Movement as a Platform Project
+doc;incubation;Documentation Project
+externalapi;unmaintained;External API Framework Project
+holmes;incubation;Holmes Project
+integration;incubation;Integration Project
+logging-analytics;unmaintained;Logging Enhancements Project
+modeling;mature;Modeling Project
+msb;incubation;Microservices Bus Project
+multicloud;mature;Multi VIM/Cloud Project
+music;unmaintained;MUSIC Project
+oof;mature;Optimization Framework Project
+oom;incubation;ONAP Operations Manager Project
+policy;mature;Policy Framework Project
+portal;unmaintained;Portal Platform Project
+sdc;mature;Service Design & Creation Project
+sdnc;mature;Software Defined Network Controller Project
+so;mature;Service Orchestrator Project
+university;incubation;ONAP University Project
+usecase-ui;mature;Usecase UI Project
+vfc;mature;Virtual Function Controller Project
+vid;unmaintained;Virtual Infrastructure Deployment Project
+vnfrqts;incubation;VNF Requirements Project
+vnfsdk;incubation;VNF SDK Project
+vvp;incubation;VNF Validation Program Project
\ No newline at end of file