Skip to content

Commit 3320f64

Browse files
petewallpetewall
committed
Wip
Signed-off-by: Pete Wall <[email protected]>
1 parent 0c43e86 commit 3320f64

File tree

13 files changed

+984
-168
lines changed

13 files changed

+984
-168
lines changed

‎charts/k8s-monitoring/charts/feature-kubernetes-manifests/README.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,5 +51,10 @@ Be sure perform actual integration testing in a live environment in the main [k8
5151
| image.registry | string | `"ghcr.io"` | |
5252
| image.repository | string | `"grafana/helm-chart-toolbox-kubectl"` | |
5353
| image.tag | string | `"0.1.2"` | |
54+
| kinds.cronjobs.gather | bool | `true` | |
55+
| kinds.daemonsets.gather | bool | `true` | |
56+
| kinds.deployments.gather | bool | `true` | |
57+
| kinds.pods.gather | bool | `true` | |
58+
| kinds.statefulsets.gather | bool | `true` | |
5459
| namespaces | list | `[]` | |
55-
| pods.gather | bool | `true` | |
60+
| refreshInterval | int | `3600` | |

‎charts/k8s-monitoring/charts/feature-kubernetes-manifests/collect-manifests.sh‎

Lines changed: 155 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ if [[ "${script_name}" == "bash" || "${script_name}" == "-bash" ]]; then
66
script_name="script.sh"
77
fi
88

9+
DefaultRefreshInterval=60
10+
911
usage() {
1012
echo "Usage: ${script_name} [OPTIONS]"
1113
echo ""
@@ -19,13 +21,14 @@ usage() {
1921
echo " -n, --namespaces <list> Comma or space separated list of namespaces to scan."
2022
echo " When omitted, all namespaces are scanned."
2123
echo " -p, --pod-filters <list> Comma or space separated list of jq selectors to drop"
22-
echo " from the pod JSON."
23-
echo " Default: .status"
24+
echo " from the pod JSON. Default: \".status\""
25+
echo " --refresh-interval <sec> How frequently to refresh manifests. Default: \"${DefaultRefreshInterval}\""
2426
echo " -h, --help Show this help message."
2527
}
2628

27-
namespaces_arg=""
28-
pod_filters_arg=""
29+
podNamespaces=()
30+
podFilters=(".status")
31+
refreshInterval="${DefaultRefreshInterval}"
2932

3033
while [[ $# -gt 0 ]]; do
3134
case "$1" in
@@ -35,7 +38,15 @@ while [[ $# -gt 0 ]]; do
3538
usage
3639
exit 1
3740
fi
38-
namespaces_arg="$2"
41+
42+
sanitized="${2//$'\n'/ }"
43+
sanitized="${sanitized//,/ }"
44+
read -ra parsedNamespaces <<< "${sanitized}"
45+
for ns in "${parsedNamespaces[@]}"; do
46+
[[ -n "${ns}" ]] || continue
47+
podNamespaces+=("${ns}")
48+
done
49+
3950
shift 2
4051
;;
4152
-p|--pod-filters)
@@ -44,7 +55,25 @@ while [[ $# -gt 0 ]]; do
4455
usage
4556
exit 1
4657
fi
47-
pod_filters_arg="$2"
58+
59+
podFilters=()
60+
sanitized="${2//$'\n'/ }"
61+
sanitized="${sanitized//,/ }"
62+
read -ra parsedPodFilters <<< "${sanitized}"
63+
for filter in "${parsedPodFilters[@]}"; do
64+
[[ -n "${filter}" ]] || continue
65+
podFilters+=("${filter}")
66+
done
67+
68+
shift 2
69+
;;
70+
--refresh-interval)
71+
if [[ $# -lt 2 ]]; then
72+
echo "Error: --pod-refresh requires an argument." >&2
73+
usage
74+
exit 1
75+
fi
76+
refreshInterval="$2"
4877
shift 2
4978
;;
5079
-h|--help)
@@ -70,32 +99,7 @@ fi
7099

71100
mkdir -p "${MANIFEST_DIR}"
72101

73-
pod_namespaces=()
74-
if [[ -n "${namespaces_arg}" ]]; then
75-
sanitized="${namespaces_arg//$'\n'/ }"
76-
sanitized="${sanitized//,/ }"
77-
read -ra parsed_namespaces <<< "${sanitized}"
78-
for ns in "${parsed_namespaces[@]}"; do
79-
[[ -n "${ns}" ]] || continue
80-
pod_namespaces+=("${ns}")
81-
done
82-
fi
83-
84-
default_pod_filters=(".status")
85-
pod_filters=()
86-
if [[ -n "${pod_filters_arg}" ]]; then
87-
sanitized="${pod_filters_arg//$'\n'/ }"
88-
sanitized="${sanitized//,/ }"
89-
read -ra parsed_pod_filters <<< "${sanitized}"
90-
for filter in "${parsed_pod_filters[@]}"; do
91-
[[ -n "${filter}" ]] || continue
92-
pod_filters+=("${filter}")
93-
done
94-
fi
95-
96-
if [[ ${#pod_filters[@]} -eq 0 ]]; then
97-
pod_filters=("${default_pod_filters[@]}")
98-
fi
102+
watchPids=()
99103

100104
build_jq_filter() {
101105
local program="."
@@ -108,63 +112,158 @@ build_jq_filter() {
108112

109113
collect_pod_manifest() {
110114
local namespace="$1"
111-
local pod_name="$2"
112-
pod_output_filter="$(build_jq_filter "${pod_filters[@]}")"
115+
local podName="$2"
113116

114-
[[ -n "${namespace}" && -n "${pod_name}" ]] || return 0
117+
[[ -n "${namespace}" && -n "${podName}" ]] || return 0
115118

116119
local namespace_dir="${MANIFEST_DIR}/pods/${namespace}"
117120
mkdir -p "${namespace_dir}"
118121

119-
local output_file="${namespace_dir}/${pod_name}.json"
120-
local tmp_file="${output_file}.tmp"
122+
local outputFile="${namespace_dir}/${podName}.json"
123+
local tmpFile="${outputFile}.tmp"
121124

122-
if kubectl get pod --namespace "${namespace}" "${pod_name}" -o json | jq --compact-output "${pod_output_filter}" > "${tmp_file}"; then
123-
echo "Storing pod manifest \"${namespace}/${pod_name}\""
124-
mv "${tmp_file}" "${output_file}"
125+
pod_output_filter="$(build_jq_filter "${podFilters[@]}")"
126+
if kubectl get pod --namespace "${namespace}" "${podName}" -o json | jq --compact-output "${pod_output_filter}" > "${tmpFile}"; then
127+
if [[ ! -f "${outputFile}" ]] || ! cmp -s "${tmpFile}" "${outputFile}"; then
128+
echo "Storing pod manifest \"${namespace}/${podName}\""
129+
mv "${tmpFile}" "${outputFile}"
130+
else
131+
echo "No changes to pod manifest \"${namespace}/${podName}\""
132+
rm -f "${tmpFile}"
133+
fi
125134
else
126-
echo "Failed to collect manifest for pod ${namespace}/${pod_name}" >&2
127-
rm -f "${tmp_file}"
135+
echo "Failed to collect manifest for pod ${namespace}/${podName}" >&2
136+
rm -f "${tmpFile}"
137+
fi
138+
}
139+
140+
remove_pod_manifest() {
141+
local namespace="$1"
142+
local podName="$2"
143+
144+
[[ -n "${namespace}" && -n "${podName}" ]] || return
145+
146+
local outputFile="${MANIFEST_DIR}/pods/${namespace}/${podName}.json"
147+
if [[ -f "${outputFile}" ]]; then
148+
rm -f "${outputFile}"
149+
echo "Removed pod manifest \"${namespace}/${podName}\""
128150
fi
129151
}
130152

131153
collect_all_pod_manifests() {
132-
if pod_entries=$(kubectl get pods --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}'); then
154+
if podEntries=$(kubectl get pods --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}'); then
133155
while IFS= read -r entry; do
134156
[[ -n "${entry}" ]] || continue
135-
read -r namespace pod_name _ <<< "${entry}"
136-
if [[ -z "${namespace}" || -z "${pod_name}" ]]; then
157+
read -r namespace podName _ <<< "${entry}"
158+
if [[ -z "${namespace}" || -z "${podName}" ]]; then
137159
continue
138160
fi
139-
collect_pod_manifest "${namespace}" "${pod_name}"
140-
done <<< "${pod_entries}"
161+
collect_pod_manifest "${namespace}" "${podName}"
162+
done <<< "${podEntries}"
141163
else
142164
echo "Failed to list pods across all namespaces." >&2
143165
fi
144166
}
145167

146168
collect_pod_manifests_by_namespace() {
147-
for namespace in "${pod_namespaces[@]}"; do
169+
for namespace in "${podNamespaces[@]}"; do
148170
[[ -n "${namespace}" ]] || continue
149171

150-
if ! pod_names=$(kubectl get pods --namespace "${namespace}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); then
172+
if ! podNames=$(kubectl get pods --namespace "${namespace}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); then
151173
echo "Failed to list pods in namespace ${namespace}" >&2
152174
continue
153175
fi
154176

155-
while IFS= read -r pod_name; do
156-
[[ -n "${pod_name}" ]] || continue
157-
collect_pod_manifest "${namespace}" "${pod_name}"
158-
done <<< "${pod_names}"
177+
while IFS= read -r podName; do
178+
[[ -n "${podName}" ]] || continue
179+
collect_pod_manifest "${namespace}" "${podName}"
180+
done <<< "${podNames}"
159181
done
160182
}
161183

162-
while true; do
163-
if [[ ${#pod_namespaces[@]} -eq 0 ]]; then
184+
refresh_pod_manifests() {
185+
if [[ ${#podNamespaces[@]} -eq 0 ]]; then
164186
collect_all_pod_manifests
165187
else
166188
collect_pod_manifests_by_namespace
167189
fi
190+
}
191+
192+
handle_pod_watch_event() {
193+
local eventType="$1"
194+
local namespace="$2"
195+
local podName="$3"
196+
197+
[[ -n "${event_type}" && -n "${namespace}" && -n "${podName}" ]] || return
198+
199+
echo "Pod event: ${namespace}/${podName}: ${eventType}"
200+
case "${eventType}" in
201+
ADDED|MODIFIED)
202+
collect_pod_manifest "${namespace}" "${podName}"
203+
;;
204+
DELETED)
205+
remove_pod_manifest "${namespace}" "${podName}"
206+
;;
207+
*)
208+
;;
209+
esac
210+
}
211+
212+
watch_pods() {
213+
local kubectl_args=("$@")
214+
echo "Starting pod watcher: kubectl get pods ${kubectl_args[*]}"
215+
216+
while true; do
217+
if ! kubectl get pods "${kubectl_args[@]}" --watch --output-watch-events -o json \
218+
| jq --unbuffered -r 'select(.object.metadata.namespace != null and .object.metadata.name != null and .type != null) | "\(.type) \(.object.metadata.namespace) \(.object.metadata.name)"' \
219+
| while read -r eventType namespace podName; do
220+
handle_pod_watch_event "${eventType}" "${namespace}" "${podName}"
221+
done; then
222+
echo "Pod watch ended unexpectedly for args: ${kubectl_args[*]}" >&2
223+
sleep 5
224+
fi
225+
done
226+
}
227+
228+
start_pod_watches() {
229+
if [[ ${#podNamespaces[@]} -eq 0 ]]; then
230+
watch_pods "--all-namespaces" &
231+
watchPids+=("$!")
232+
else
233+
for namespace in "${podNamespaces[@]}"; do
234+
[[ -n "${namespace}" ]] || continue
235+
watch_pods "--namespace" "${namespace}" &
236+
watchPids+=("$!")
237+
done
238+
fi
239+
}
240+
241+
stop_pod_watches() {
242+
if [[ ${#watchPids[@]} -eq 0 ]]; then
243+
return
244+
fi
245+
246+
for pid in "${watchPids[@]}"; do
247+
[[ -n "${pid}" ]] || continue
248+
kill "${pid}" 2>/dev/null || true
249+
done
250+
251+
watchPids=()
252+
}
253+
254+
trap stop_pod_watches EXIT
255+
256+
start_pod_watches
257+
258+
loop_delay="${POD_LOOP_DELAY:-5}"
259+
lastFullSync=0
260+
261+
while true; do
262+
currentTime=$(date +%s)
263+
if (( currentTime - lastFullSync >= refreshInterval )); then
264+
refresh_pod_manifests
265+
lastFullSync="${currentTime}"
266+
fi
168267

169-
sleep 60
268+
sleep "${loop_delay}"
170269
done

‎charts/k8s-monitoring/charts/feature-kubernetes-manifests/templates/_sidecar.tpl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
{{- if .Values.namespaces }}
1515
- --namespaces
1616
- {{ .Values.namespaces | join "," }}
17+
{{- end }}
18+
{{- if .Values.refreshInterval }}
19+
- --refresh-interval
20+
- {{ .Values.refreshInterval | quote }}
1721
{{- end }}
1822
env:
1923
- name: MANIFEST_DIR

‎charts/k8s-monitoring/charts/feature-kubernetes-manifests/values.schema.json‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,56 @@
2828
}
2929
}
3030
},
31-
"namespaces": {
32-
"type": "array"
33-
},
34-
"pods": {
31+
"kinds": {
3532
"type": "object",
3633
"properties": {
37-
"gather": {
38-
"type": "boolean"
34+
"cronjobs": {
35+
"type": "object",
36+
"properties": {
37+
"gather": {
38+
"type": "boolean"
39+
}
40+
}
41+
},
42+
"daemonsets": {
43+
"type": "object",
44+
"properties": {
45+
"gather": {
46+
"type": "boolean"
47+
}
48+
}
49+
},
50+
"deployments": {
51+
"type": "object",
52+
"properties": {
53+
"gather": {
54+
"type": "boolean"
55+
}
56+
}
57+
},
58+
"pods": {
59+
"type": "object",
60+
"properties": {
61+
"gather": {
62+
"type": "boolean"
63+
}
64+
}
65+
},
66+
"statefulsets": {
67+
"type": "object",
68+
"properties": {
69+
"gather": {
70+
"type": "boolean"
71+
}
72+
}
3973
}
4074
}
75+
},
76+
"namespaces": {
77+
"type": "array"
78+
},
79+
"refreshInterval": {
80+
"type": "integer"
4181
}
4282
}
4383
}

‎charts/k8s-monitoring/charts/feature-kubernetes-manifests/values.yaml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,19 @@
11
---
22
namespaces: []
33

4-
pods:
5-
gather: true
4+
refreshInterval: 3600
5+
6+
kinds:
7+
pods:
8+
gather: true
9+
deployments:
10+
gather: true
11+
statefulsets:
12+
gather: true
13+
daemonsets:
14+
gather: true
15+
cronjobs:
16+
gather: true
617

718
# The image to run to get the Kubernetes manifests from this cluster. It must contain
819
# `kubectl` and `jq` at a minimum.

0 commit comments

Comments
 (0)