--- apiVersion: v1 data: nginx_error_status.py: |- #!/usr/bin/env python3 # -- coding: utf-8 -- import os import sys import requests import json es_index_dict = { "ingress-nginx-prod-*": "51b4d440-734f-11ee-942a-c96909d7cda5" } os.environ['ES_URL'] = "http://elasticsearch-es-http.mid:9200" #os.environ['ES_URL'] = "http://es.uenpay.com" os.environ['ES_USER'] = "kibana" os.environ['ES_PASSWORD'] = "123456" os.environ['DURATION'] = "5m" os.environ['define_description'] = "nginx状态码4xx或5xx" os.environ['QUERY_CONDITION'] = "(status:5* OR status:4*)" os.environ['METRIC_NAME'] = "nginx_error_status" def add_custom_labels(metric_name, base_labels, count): custom_labels = {} for key, value in os.environ.items(): if key.startswith('define_'): label_key = key.replace('define_', '', 1) custom_labels[label_key] = value all_labels = {**base_labels, **custom_labels} label_parts = [f"{k}=\"{v}\"" for k, v in all_labels.items()] labels_str = ",".join(label_parts) print(f'{metric_name}{{{labels_str}}} {count}') def es_query(es_index_name, query_condition): result = None headers = {'Content-Type': 'application/json'} es_auth = (os.getenv('ES_USER'), os.getenv('ES_PASSWORD')) es_url = f"{os.getenv('ES_URL')}/{es_index_name}/_search" duration = ("now-" + os.getenv('DURATION','15m')) body_template = { "size": 10000, "query": { "bool": { "filter": { "range": { "@timestamp": { "gte": duration } } }, "must": [ { "query_string": { "query": query_condition } } ] } } } try: r = requests.post(url=es_url, headers=headers, auth=es_auth, json=body_template) if r.status_code == 200: result = r.json() except Exception as err: print(f"An error occurred in es_query: {err}", file=sys.stderr) return result def counting(): query_condition = os.getenv('QUERY_CONDITION') metric_name = os.getenv('METRIC_NAME') for index_name in es_index_dict: try: result = es_query(index_name, query_condition) if result and 'hits' in result.keys(): count = result['hits']['total']['value'] for hit in result['hits']['hits']: if '_source' in hit and 'domain_name' in hit['_source']: domain_name = hit["_source"]["domain_name"] status = hit["_source"]["status"] request_method = hit["_source"]["request_method"] uri = hit["_source"]["uri"] es_index_id = es_index_dict[index_name] base_labels = {'domain_name': domain_name, 'status': status, 'request_method': request_method, 'uri': uri, 'es_index_id': es_index_id, 'query_condition': query_condition.replace(" ", "%20").replace('"', '')} add_custom_labels(metric_name, base_labels, 1) else: print('%s %d' % (metric_name, count)) except Exception as err: print(f"An error occurred in couting: {err}", file=sys.stderr) if __name__ == '__main__': counting() Null_pointer_exception-prod.py: |- #!/usr/bin/env python3 # -- coding: utf-8 -- import os import sys import requests import json es_index_dict = { "mpos-java-prod-*": "3d1ecb70-c0c6-11ee-942a-c96909d7cda5" } os.environ['ES_URL'] = "http://elasticsearch-es-http.mid:9200" #os.environ['ES_URL'] = "http://es.uenpay.com" os.environ['ES_USER'] = "kibana" os.environ['ES_PASSWORD'] = "123456" os.environ['DURATION'] = "15m" os.environ['define_description'] = "mpos服务空指针异常" os.environ['QUERY_CONDITION'] = "\"Exception error message:null\"" os.environ['METRIC_NAME'] = "java_error" def add_custom_labels(metric_name, base_labels, count): custom_labels = {} for key, value in os.environ.items(): if key.startswith('define_'): label_key = key.replace('define_', '', 1) custom_labels[label_key] = value all_labels = {**base_labels, **custom_labels} label_parts = [f"{k}=\"{v}\"" for k, v in all_labels.items()] labels_str = ",".join(label_parts) print(f'{metric_name}{{{labels_str}}} {count}') def es_query(es_index_name, query_condition): result = None headers = {'Content-Type': 'application/json'} es_auth = (os.getenv('ES_USER'), os.getenv('ES_PASSWORD')) es_url = f"{os.getenv('ES_URL')}/{es_index_name}/_search" duration = ("now-" + os.getenv('DURATION','15m')) body_template = { "size": 10000, "query": { "bool": { "filter": { "range": { "@timestamp": { "gte": duration } } }, "must": [ { "query_string": { "query": query_condition } } ] } } } try: r = requests.post(url=es_url, headers=headers, auth=es_auth, json=body_template) if r.status_code == 200: result = r.json() except Exception as err: print(f"An error occurred in es_query: {err}", file=sys.stderr) return result def counting(): query_condition = os.getenv('QUERY_CONDITION') metric_name = os.getenv('METRIC_NAME') for index_name in es_index_dict: try: result = es_query(index_name, query_condition) if result and 'hits' in result.keys(): count = result['hits']['total']['value'] for hit in result['hits']['hits']: if '_source' in hit and 'message' in hit['_source']: project = hit["_source"]["fields"]["project"] es_index_id = es_index_dict[index_name] base_labels = {'project': project, 'es_index_id': es_index_id, 'query_condition': query_condition.replace(" ", "%20").replace('"', '')} add_custom_labels(metric_name, base_labels, count) else: print('%s %d' % (metric_name, count)) except Exception as err: print(f"An error occurred in couting: {err}", file=sys.stderr) if __name__ == '__main__': counting() kind: ConfigMap metadata: name: script-file namespace: monitoring --- apiVersion: apps/v1 kind: Deployment metadata: name: script-monitor1 namespace: monitoring labels: name: script-monitor1 app: script-monitor1 spec: replicas: 1 selector: matchLabels: name: script-monitor1 template: metadata: annotations: app_name: script-monitor1 prometheus.com/path: /metrics prometheus.com/port: '8000' prometheus.com/scheme: http prometheus.com/scraped: 'true' labels: app: script-monitor1 name: script-monitor1 spec: containers: - name: script-monitor1 #image: harbor.uenpay.com/base/script-monitor:20240523 image: registry.cn-hangzhou.aliyuncs.com/zhengyu1992/script-monitor:20240523 imagePullPolicy: Always env: - name: INTERVAL_TIME value: '120' - name: TIMEOUT_TIME value: '90' volumeMounts: - name: data mountPath: /opt/monitor terminationGracePeriodSeconds: 0 securityContext: runAsUser: 0 volumes: - name: data configMap: name: script-file1 --- apiVersion: v1 kind: Service metadata: name: script-monitor1 namespace: monitoring labels: app: script-monitor1 spec: selector: app: script-monitor1 ports: - name: nacos-nodeport port: 8000 targetPort: 8000