1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
#!/usr/bin/env python3
"""Fetch alerting and aggregation rules from provided urls into this chart."""
import textwrap
from os import makedirs
import requests
import yaml
from yaml.representer import SafeRepresenter
# https://stackoverflow.com/a/20863889/961092
class LiteralStr(str):
pass
def change_style(style, representer):
def new_representer(dumper, data):
scalar = representer(dumper, data)
scalar.style = style
return scalar
return new_representer
# Source files list
charts = [
{
'source': 'https://raw.githubusercontent.com/coreos/prometheus-operator/master/contrib/kube-prometheus/manifests/prometheus-rules.yaml',
'destination': '../templates/alertmanager/rules'
},
# don't uncomment until https://github.com/etcd-io/etcd/pull/10244 is merged
# {
# 'source': 'https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml',
# 'destination': '../templates/alertmanager/rules'
# },
]
# Additional conditions map
condition_map = {
'kube-apiserver.rules': ' .Values.kubeApiServer.enabled',
'kube-scheduler.rules': ' .Values.kubeScheduler.enabled',
'node.rules': ' .Values.nodeExporter.enabled',
'kubernetes-apps': ' .Values.kubeStateMetrics.enabled',
'etcd': ' .Values.kubeEtcd.enabled',
}
alert_condition_map = {
'KubeAPIDown': '.Values.kubeApiServer.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics
'KubeControllerManagerDown': '.Values.kubeControllerManager.enabled',
'KubeSchedulerDown': '.Values.kubeScheduler.enabled',
'KubeStateMetricsDown': '.Values.kubeStateMetrics.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics
'KubeletDown': '.Values.prometheusOperator.kubeletService.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics
'PrometheusOperatorDown': '.Values.prometheusOperator.enabled',
'NodeExporterDown': '.Values.nodeExporter.enabled',
'CoreDNSDown': '.Values.kubeDns.enabled',
}
replacement_map = {
'job="prometheus-operator"': {
'replacement': 'job="{{ $operatorJob }}"',
'init': '{{- $operatorJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "operator" }}'},
'job="prometheus-k8s"': {
'replacement': 'job="{{ $prometheusJob }}"',
'init': '{{- $prometheusJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "prometheus" }}'},
'job="alertmanager-main"': {
'replacement': 'job="{{ $alertmanagerJob }}"',
'init': '{{- $alertmanagerJob := printf "%s-%s" (include "prometheus-operator.fullname" .) "alertmanager" }}'},
}
# standard header
header = '''# Generated from '%(name)s' group from %(url)s
{{- if and .Values.defaultRules.create%(condition)s }}%(init_line)s
apiVersion: {{ printf "%%s/v1" (.Values.prometheusOperator.crdApiGroup | default "monitoring.coreos.com") }}
kind: PrometheusRule
metadata:
name: {{ printf "%%s-%%s" (include "prometheus-operator.fullname" .) "%(name)s" | trunc 63 | trimSuffix "-" }}
labels:
app: {{ template "prometheus-operator.name" . }}
{{ include "prometheus-operator.labels" . | indent 4 }}
{{- if .Values.defaultRules.labels }}
{{ toYaml .Values.defaultRules.labels | indent 4 }}
{{- end }}
{{- if .Values.defaultRules.annotations }}
annotations:
{{ toYaml .Values.defaultRules.annotations | indent 4 }}
{{- end }}
spec:
groups:
-'''
def init_yaml_styles():
represent_literal_str = change_style('|', SafeRepresenter.represent_str)
yaml.add_representer(LiteralStr, represent_literal_str)
def escape(s):
return s.replace("{{", "{{`{{").replace("}}", "}}`}}")
def fix_expr(rules):
"""Remove trailing whitespaces and line breaks, which happen to creep in
due to yaml import specifics;
convert multiline expressions to literal style, |-"""
for rule in rules:
rule['expr'] = rule['expr'].rstrip()
if '\n' in rule['expr']:
rule['expr'] = LiteralStr(rule['expr'])
def yaml_str_repr(struct, indent=4):
"""represent yaml as a string"""
text = yaml.dump(
struct,
width=1000, # to disable line wrapping
default_flow_style=False # to disable multiple items on single line
)
text = escape(text) # escape {{ and }} for helm
text = textwrap.indent(text, ' ' * indent)[indent - 1:] # indent everything, and remove very first line extra indentation
return text
def add_rules_conditions(rules, indent=4):
"""Add if wrapper for rules, listed in alert_condition_map"""
rule_condition = '{{- if %s }}\n'
for alert_name in alert_condition_map:
line_start = ' ' * indent + '- alert: '
if line_start + alert_name in rules:
rule_text = rule_condition % alert_condition_map[alert_name]
# add if condition
index = rules.index(line_start + alert_name)
rules = rules[:index] + rule_text + rules[index:]
# add end of if
try:
next_index = rules.index(line_start, index + len(rule_text) + 1)
except ValueError:
# we found the last alert in file if there are no alerts after it
next_index = len(rules)
rules = rules[:next_index] + '{{- end }}\n' + rules[next_index:]
return rules
def write_group_to_file(group, url, destination):
fix_expr(group['rules'])
# prepare rules string representation
rules = yaml_str_repr(group)
# add replacements of custom variables and include their initialisation in case it's needed
init_line = ''
for line in replacement_map:
if line in rules:
rules = rules.replace(line, replacement_map[line]['replacement'])
init_line += '\n' + replacement_map[line]['init']
# append per-alert rules
rules = add_rules_conditions(rules)
# initialize header
lines = header % {
'name': group['name'],
'url': url,
'condition': condition_map.get(group['name'], ''),
'init_line': init_line,
}
# rules themselves
lines += rules
# footer
lines += '{{- end }}'
filename = group['name'] + '.yaml'
new_filename = "%s/%s" % (destination, filename)
# make sure directories to store the file exist
makedirs(destination, exist_ok=True)
# recreate the file
with open(new_filename, 'w') as f:
f.write(lines)
print("Generated %s" % new_filename)
def main():
init_yaml_styles()
# read the rules, create a new template file per group
for chart in charts:
print("Generating rules from %s" % chart['source'])
raw_text = requests.get(chart['source']).text
yaml_text = yaml.load(raw_text)
# etcd workaround, their file don't have spec level
groups = yaml_text['spec']['groups'] if yaml_text.get('spec') else yaml_text['groups']
for group in groups:
write_group_to_file(group, chart['source'], chart['destination'])
print("Finished")
if __name__ == '__main__':
main()
|
