diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/checkdocs.sh | 35 | ||||
| -rw-r--r-- | tools/getrsttitle.py | 150 | ||||
| -rwxr-xr-x | tools/latestbranch.sh | 33 | ||||
| -rw-r--r-- | tools/wiki_lifecycle_state_210517.txt | 34 |
4 files changed, 241 insertions, 11 deletions
diff --git a/tools/checkdocs.sh b/tools/checkdocs.sh index 66cbd4c47..ca217d488 100755 --- a/tools/checkdocs.sh +++ b/tools/checkdocs.sh @@ -58,7 +58,7 @@ ### SHORT: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' ### -script_version="1.7 (2021-04-12)" +script_version="1.9 (2021-05-31)" # save command for the restart with logging enabled command=$0 @@ -366,8 +366,11 @@ do printf "\nconf.py files:\n" find ./$reponame -type f -name conf.py | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_confpy.log - printf "\nindex.rst files:\n" - find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst.log + printf "\nindex.rst files (all):\n" + find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst_all.log + + printf "\nindex.rst files (docs root directory):\n" + find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | grep ']/docs/index.rst' | tee -a ${branch}_indexrst_docs_root.log printf "\nINFO.yaml files:\n" find ./$reponame -type f -name INFO.yaml | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_infoyaml.log @@ -381,6 +384,12 @@ do done <${repolist} + # get (first) title for a rst file + drawline + python3 ../getrsttitle.py ${branch}_rstfiles.log | tee ${branch}_rstfiles_titles.log + drawline + python3 ../getrsttitle.py ${branch}_indexrst_docs_root.log | tee ${branch}_indexrst_docs_root_titles.log + # examine repos drawline find . -type f -name values.yaml -print -exec grep "image:" {} \; | sed -r 's:^ +::' | tee ${branch}_dockerimagesfull.log @@ -637,6 +646,7 @@ do # csv column #11: conf.py # csv column #12: tox.ini # csv column #13: index.rst + # csv column #14: first title in index.rst # # columns are filled with values from requested branch. # if data is not available values from master branch are used. @@ -645,7 +655,7 @@ do readarray -t array < ./${repolist}; i=0 - csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst" + csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst,first title in index.rst" ((i++)) for line in "${array[@]}" do @@ -699,13 +709,16 @@ do docs="${docs},-" fi - # index.rst + # index.rst, first title in index.rst + indexrsttitle="" if [ -f ./${line}/docs/index.rst ] ; then - docs="${docs},index.rst" + indexrsttitle=$(cat ${branch}_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $2}'); + docs="${docs},index.rst,${indexrsttitle}" elif [ -f ../master/${line}/docs/index.rst ] ; then - docs="${docs},(index.rst)" + indexrsttitle=$(cat ../master/master_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $2}'); + docs="${docs},(index.rst),(${indexrsttitle})" else - docs="${docs},-" + docs="${docs},-,-" fi #echo "DBUG: docs=${docs}" @@ -718,8 +731,8 @@ do unset docs # - # csv column #14: index.html@RTD accessibility check - # csv column #15: index.html url + # csv column #15: index.html@RTD accessibility check + # csv column #16: index.html url # readarray -t array < ./${branch}_repoclone.log; @@ -832,7 +845,7 @@ do done # - # csv column #16: release notes + # csv column #17: release notes # readarray -t array < ../${repolist}; diff --git a/tools/getrsttitle.py b/tools/getrsttitle.py new file mode 100644 index 000000000..e852d9f92 --- /dev/null +++ b/tools/getrsttitle.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +### =========================================================================== +### Licensed under the Apache License, Version 2.0 (the "License"); +### you may not use this file except in compliance with the License. +### You may obtain a copy of the License at +### +### http://www.apache.org/licenses/LICENSE-2.0 +### +### Unless required by applicable law or agreed to in writing, software +### distributed under the License is distributed on an "AS IS" BASIS, +### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +### See the License for the specific language governing permissions and +### limitations under the License. +### +### Copyright (C) 2021 Deutsche Telekom AG +### ============LICENSE_END==================================================== + +# +# getrsttitle.py +# AUTHOR(S): +# Thomas Kulik, Deutsche Telekom AG, 2021 +# DESCRIPTION: +# Processes a list of rst files and retrieves the first title for every single rst file. +# Copy program to {branch} directory of cloned ONAP documentation and run it. +# USAGE: +# python3 getrsttitle.py filename +# +# Helpful resources: +# https://regex101.com/r/YNYK2Q/1/ +# https://stackoverflow.com/questions/20312443/how-to-find-title-a-la-restructuredtext +# + +import re +import os.path +import sys +import argparse + +# +# argument handling +# + +parser = argparse.ArgumentParser(description='Processes a list of rst files and retrieves the first title for every single rst file.') +parser.add_argument('filename') +args = parser.parse_args() + +# regex to find title underlined with various characters +#regex1 = r"(?:^|\n)(?!\=)([^\n\r]+)\r?\n(\=+)(?:\r?\n| *$)" +#regex2 = r"(?:^|\n)(?!\-)([^\n\r]+)\r?\n(\-+)(?:\r?\n| *$)" +#regex3 = r"(?:^|\n)(?!\~)([^\n\r]+)\r?\n(\~+)(?:\r?\n| *$)" +#regex4 = r"(?:^|\n)(?!\#)([^\n\r]+)\r?\n(\#+)(?:\r?\n| *$)" +#regex5 = r"(?:^|\n)(?!\*)([^\n\r]+)\r?\n(\*+)(?:\r?\n| *$)" + +# there is a problem with raw strings (r"...") in the regex search below +# workaround: using \\ to mask special characters in regex +regex_list = [ + "(?:^|\\n)(?!\\=)([^\\n\\r]+)\\r?\\n(\\=+)(?:\\r?\\n| *$)", + "(?:^|\\n)(?!\\-)([^\\n\\r]+)\\r?\\n(\\-+)(?:\\r?\\n| *$)", + "(?:^|\\n)(?!\\~)([^\\n\\r]+)\\r?\\n(\\~+)(?:\\r?\\n| *$)", + "(?:^|\\n)(?!\\#)([^\\n\\r]+)\\r?\\n(\\#+)(?:\\r?\\n| *$)", + "(?:^|\\n)(?!\\*)([^\\n\\r]+)\\r?\\n(\\*+)(?:\\r?\\n| *$)", + ] + +# DBUG only +#for regex in regex_list: +# print(repr(regex)) + +#filename = './master_indexrst_docs_root.log' +#filename = './master_rstfiles.log' + +if os.path.isfile(args.filename): + with open(args.filename) as fn: + # read first line + line = fn.readline() + #print("DBUG: line={}".format(line)) + file_cnt = 0 + while line: + rstfile = "./" + re.sub('\[|\]', '', line).strip() + repository_tmp1 = re.sub('\].+$', '',line).strip() + repository = re.sub('\[', '',repository_tmp1).strip() + project_tmp1 = re.sub('\].+$', '',line).strip() + project_tmp2 = re.sub('\/.+$', '',project_tmp1).strip() + project = re.sub('\[', '',project_tmp2).strip() + #print("DBUG: file #{} {}".format(file_cnt, rstfile)) + #print("DBUG: repository #{} {}".format(file_cnt, repository)) + #print("DBUG: project #{} {}".format(file_cnt, project)) + file_cnt += 1 + if os.path.isfile(rstfile): + with open(rstfile, 'r') as content: + content_rstfile = content.read() + #print("DBUG: content_rstfile = \n{}".format(content_rstfile)) + regex_cnt = 0 + for regex in regex_list: + regex_cnt += 1 + m = re.search(regex, content_rstfile, re.MULTILINE) + #print("DBUG: using regex " + repr(regex)) + #print("DBUG: using regex1 " + repr(regex1)) + #print("DBUG: regex_cnt = {}".format(regex_cnt)) + if m: + match = m.group(1) + #print ("DBUG: |REGEX| {} |REGEXCNT| {} |FILECNT| {} |FILE| {} |MATCH| {}".format(repr(regex), regex_cnt, file_cnt, rstfile, match)) + # end regex loop if we have a title + break + else: + match = "NO-TITLE-FOUND" + #print ("DBUG: NO-TITLE-FOUND") + else: + print ("ERR: File {} does not exist".format(rstfile)) + + #print ("DBUG: |REGEX| {} |REGEXCNT| {} |FILECNT| {} |FILE| {} |MATCH| {}".format(repr(regex), regex_cnt, file_cnt, rstfile, match)) + #print ("DBUG: file #{} '{}' '{}'".format(file_cnt, rstfile, match)) + + # clean up result and print + match_1 = match.replace(",", "") # remove , + match_final = match_1.strip() # remove \n + print ("{},{},{},{}".format(project.strip(), repository.strip(), line.strip(), match_final.strip())) + + # read next line and loop + line = fn.readline() +else: + print ("ERR: File {} does not exist".format(args.filename)) + +sys.exit() + +# +# example code to show detailed regex matches and group content +# to be used in a future version of this program +# +# matches = re.finditer(regex2, content, re.MULTILINE) +# for matchNum, match in enumerate(matches, start=1): +# print ("Match {matchNum} was found at {start}-{end}: {match}".format(matchNum = matchNum, start = match.start(), end = match.end(), match = match.group())) +# print ("{match}".format(match = match.group())) +# for groupNum in range(0, len(match.groups())): +# groupNum = groupNum + 1 +# print ("Group {groupNum} found at {start}-{end}: {group}".format(groupNum = groupNum, start = match.start(groupNum), end = match.end(groupNum), group = match.group(groupNum))) +# print ("Test:" "{group}".format(group = match.group(1))) +# + +# +# example code for pandas +# to be used in a future version of this program +# +# import pandas as pd +# pd.set_option('display.max_rows', 500) +# pd.set_option('display.max_columns', 500) +# pd.set_option('display.width', 1000) +# +# table = pd.read_csv("master_table.csv") +# print(table) +#
\ No newline at end of file diff --git a/tools/latestbranch.sh b/tools/latestbranch.sh new file mode 100755 index 000000000..ae9dd1fc0 --- /dev/null +++ b/tools/latestbranch.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# set -x + +repolist=$1 +source="git://cloud.onap.org/mirror" + + # + # csv column #nn: latest branch + # + + readarray -t array < ./${repolist}; + i=0 + csv[i]="${csv[i]},MASTER repo name" + ((i++)) + for line in "${array[@]}" + do + reponame=$(echo $line | awk -F "|" '{print $1}'); + csv[i]="${csv[i]},${reponame}" + ((i++)) + + #echo " " + #echo ${reponame} + + git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' | sed -nr '/^master$|^amsterdam$|^beijing$|^casablanca$|^dublin$|^elalto$|^frankfurt$|^guilin$|^honolulu$|^istanbul$/Ip' | tail -2 | head -1 + #git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' + + done + unset array + unset i + unset reponame + +exit + diff --git a/tools/wiki_lifecycle_state_210517.txt b/tools/wiki_lifecycle_state_210517.txt new file mode 100644 index 000000000..af4aad8e4 --- /dev/null +++ b/tools/wiki_lifecycle_state_210517.txt @@ -0,0 +1,34 @@ +PROJECT;WIKI STATE;WIKI NAME +aaf;unmaintained;Application Authorization Framework Project +aai;mature;Active and Available Inventory Project +appc;unmaintained;Application Controller Project +ccsdk;mature;Common Controller SDK Project +ci-management;incubation;CI-Management +clamp;incubation;CLAMP Project +cli;incubation;Command Line Interface Project +cps;incubation;Configuration Persistence Service Project +dcaegen2;mature;Data Collection Analytics and Events Project +dmaap;mature;Data Movement as a Platform Project +doc;incubation;Documentation Project +externalapi;unmaintained;External API Framework Project +holmes;incubation;Holmes Project +integration;incubation;Integration Project +logging-analytics;unmaintained;Logging Enhancements Project +modeling;mature;Modeling Project +msb;incubation;Microservices Bus Project +multicloud;mature;Multi VIM/Cloud Project +music;unmaintained;MUSIC Project +oof;mature;Optimization Framework Project +oom;incubation;ONAP Operations Manager Project +policy;mature;Policy Framework Project +portal;unmaintained;Portal Platform Project +sdc;mature;Service Design & Creation Project +sdnc;mature;Software Defined Network Controller Project +so;mature;Service Orchestrator Project +university;incubation;ONAP University Project +usecase-ui;mature;Usecase UI Project +vfc;mature;Virtual Function Controller Project +vid;unmaintained;Virtual Infrastructure Deployment Project +vnfrqts;incubation;VNF Requirements Project +vnfsdk;incubation;VNF SDK Project +vvp;incubation;VNF Validation Program Project
\ No newline at end of file |