From 9cb51800466091d8fe058b004b0103f1dc23439e Mon Sep 17 00:00:00 2001 From: Melcom van Eeden Date: Mon, 12 Apr 2021 16:12:21 +0200 Subject: [PATCH 1/4] Fixed the conf type assertion and exclude mapping --- .run/agent-check-disk.run.xml | 13 + cmd/agent/dist/datadog.yaml | 782 ++++++++++++++++++++++++ pkg/collector/corechecks/system/disk.go | 17 +- 3 files changed, 807 insertions(+), 5 deletions(-) create mode 100644 .run/agent-check-disk.run.xml create mode 100644 cmd/agent/dist/datadog.yaml diff --git a/.run/agent-check-disk.run.xml b/.run/agent-check-disk.run.xml new file mode 100644 index 000000000000..374e8536dbd0 --- /dev/null +++ b/.run/agent-check-disk.run.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/cmd/agent/dist/datadog.yaml b/cmd/agent/dist/datadog.yaml new file mode 100644 index 000000000000..58d48d003ee8 --- /dev/null +++ b/cmd/agent/dist/datadog.yaml @@ -0,0 +1,782 @@ +# The host of the StackState receiver to send Agent data to +sts_url: http://localhost + +# The StackState api key to associate your Agent's data with your organization. +api_key: API_KEY + +# If you need a proxy to connect to the Internet, provide it here (default: +# disabled). You can use the 'no_proxy' list to specify hosts that should +# bypass the proxy. These settings might impact your checks requests, please +# refer to the specific check documentation for more details. Environment +# variables STS_PROXY_HTTP, STS_PROXY_HTTPS and STS_PROXY_NO_PROXY (space-separated string) +# will override the values set here. See https://docs.stackstate.com/agent/proxy/. +# +# proxy: +# http: http://user:password@proxy_for_http:port +# https: http://user:password@proxy_for_https:port +# no_proxy: +# - host1 +# - host2 + +# Setting this option to "true" will tell the agent to skip validation of SSL/TLS certificates. +# This may be necessary if the agent is running behind a proxy. +# skip_ssl_validation: false + +# Setting this option to "true" will force the agent to only use TLS 1.2 when +# pushing data to the url specified in "sts_url". +# force_tls_12: false + +# Force the hostname to whatever you want. (default: auto-detected) +# hostname: mymachine.mydomain + +# Setting this option to "true" will tell the agent to skip validation of the Hostname. +# skip_hostname_validation: false + +# Make the agent use "hostname -f" on unix-based systems as a last resort +# way of determining the hostname instead of Golang "os.Hostname()" +# This will be enabled by default in version 6.6 +# More information at https://dtdg.co/flag-hostname-fqdn +# hostname_fqdn: false + +# Set the host's tags (optional) +# tags: +# - mytag +# - env:prod +# - role:database + +# Split tag values according to a given separator. +# Only applies to host tags, tags coming from container integrations. +# Does not apply to tags on dogstatsd metrics, and tags collected by other +# integrations. +# This option is useful when the native tags do not support repeating multiple +# tags with the same name and different values. +# +# Example use-case: +# +# With a raw collected tag "foo:1;2;3" +# Using the following configuration: +# +# tag_value_split_separator: +# foo: ; +# +# will result in the raw tag being transformed into "foo:1", "foo:2", "foo:3" tags + +# Configure the level of granularity of tags to send for checks and dogstatsd metrics and events. +# Choices are: +# - low: add tags about low-cardinality objects (clusters, hosts, deployments, container images, ...) +# - orchestrator: add tags about pod, (in Kubernetes), or task (in ECS or Mesos) -level of cardinality +# - high: add tags about high-cardinality objects (individual containers, user IDs in requests, ...) +# WARNING: sending container tags for dogstatsd metrics may create more metrics +# (one per container instead of one per host). This may impact your custom metrics billing. +# +# checks_tag_cardinality: orchestrator +# dogstatsd_tag_cardinality: low + +# Histogram and Historate configuration +# +# Configure which aggregated value to compute. Possible values are: min, max, +# median, avg, sum and count. +# +# histogram_aggregates: ["max", "median", "avg", "count"] +# +# Configure which percentiles will be computed. Must be a list of float +# between 0 and 1. +# Warning: percentiles must be specified as yaml strings +# +# histogram_percentiles: ["0.95"] +# +# Copy histogram values to distributions for true global distributions (in beta) +# This will increase the number of custom metrics created +# histogram_copy_to_distribution: false +# +# A prefix to add to distribution metrics created when histogram_copy_to_distributions is true +# histogram_copy_to_distribution_prefix: "" + +# Forwarder timeout in seconds +# forwarder_timeout: 20 + +# The forwarder retries failed requests. Use this setting to change the +# maximum length of the forwarder's retry queue (each request in the queue +# takes no more than 2MB in memory) +# forwarder_retry_queue_max_size: 30 + +# The number of workers used by the forwarder. Please note each worker will +# open an outbound HTTP connection towards StackState's metrics intake at every +# flush. +# forwarder_num_workers: 1 + +# Collect AWS EC2 custom tags as agent tags +# collect_ec2_tags: false + +# Collect Google Cloud Engine metadata as agent tags +# collect_gce_tags: true + +# The path containing check configuration files +# By default, uses the conf.d folder located in the agent configuration folder. +confd_path: ../stackstate-agent/dev/dist/conf.d + +# Additional path where to search for Python checks +# By default, uses the checks.d folder located in the agent configuration folder. +# additional_checksd: + +# The port for the go_expvar server +# expvar_port: 5000 + +# The port on which the IPC api listens +# cmd_port: 5001 + +# The port for the browser GUI to be served +# Setting 'GUI_port: -1' turns off the GUI completely +# Default is '5002' on Windows and macOS ; turned off on Linux +# GUI_port: -1 + +# The agent can expose its health check on a dedicated http port. +# This is useful for orchestrators that support http probes. +# Default is 0 (disabled), set a valid port number (eg. 5555) to enable. +# health_port: 0 + +# The `check_runners` refers to the number of concurrent check runners available +# for check instance execution. The scheduler will attempt to spread the +# instances over the collection interval and will _at most_ be running the +# number of check runners instances concurrently. Setting the value to 1 +# would result in checks running sequentially. +# +# NOTE: due to the nature of the python interpreter and its global interpreter +# lock (GIL) only one native thread may be running on the python interpreter at +# a time. Concurrent check runners imply concurrently executing checks must contend +# for the GIL. This has the side-effect of increasing wall-time execution times for +# the checks. The aggregate system/user-time for all checks however, should be +# decreased. Concurrency works particularly well with I/O bound tasks (or at least +# IO intensive) as the GIL will typically be released while waiting for IO, allowing +# for other checks to grab a hold of the GIL and continue execution. For CPU-bound +# checks with a low activity, it's likely that the check instances will contend for +# the GIL, which will cause a CPU overhead compared to running these check instances +# sequentially (i.e. on one check runner). +# +# This is a sensitive setting and we do NOT recommend changing the default number +# of check runners in the general case. The level of concurrency has effects on +# the agent's: RSS memory, CPU load, resource contention overhead, etc. +# +# check_runners: 4 + +# Metadata collection should always be enabled, except if you are running several +# agents/dsd instances per host. In that case, only one agent should have it on. +# WARNING: disabling it on every agent will lead to display and billing issues +# enable_metadata_collection: true + +# Enable the gohai collection of systems data +enable_gohai: false + +# IPC api server timeout in seconds +# server_timeout: 15 + +# Some environments may have the procfs file system mounted in a miscellaneous +# location. The procfs_path configuration parameter provides a mechanism to +# override the standard default location: '/proc' - this setting will trickle +# down to integrations and affect their behavior if they rely on the psutil +# python package. +# procfs_path: /proc + +# Disable Python3 validation of python checks +# disable_py3_validation: false + +# BETA: Encrypted Secrets (Linux only) +# +# This feature is in beta and its options or behaviour might break between +# minor or bugfix releases of the Agent. +# +# The agent can call an external command to fetch secrets. The command will be +# executed maximum once per instance containing an encrypted password. +# Secrets are cached by the agent, this will avoid executing again the +# secret_backend_command to fetch an already known secret (useful when combine +# with Autodiscovery). This feature is still in beta. +# +# For more information see: https://github.com/StackVista/stackstate-agent/blob/master/docs/agent/secrets.md +# +# Path to the script to execute. The script must belong to the same user used +# to run the agent. Executable right must be given to the agent and no rights +# for 'group' or 'other'. +# secret_backend_command: /path/to/command +# +# A list of arguments to give to the command at each run (optional) +# secret_backend_arguments: +# - argument1 +# - argument2 +# +# The size in bytes of the buffer used to store the command answer (apply to +# both stdout and stderr) +# secret_backend_output_max_size: 1024 +# +# The timeout to execute the command in second +# secret_backend_timeout: 5 + + +# Metadata providers, add or remove from the list to enable or disable collection. +# Intervals are expressed in seconds. You can also set a provider's interval to 0 +# to disable it. +metadata_providers: + - name: agent_checks + interval: 600 + +# DogStatsd +# +# If you don't want to enable the DogStatsd server, set this option to false +# use_dogstatsd: true +# +# Make sure your client is sending to the same UDP port +# dogstatsd_port: 8125 +# +# The host to bind to receive external metrics (used only by the dogstatsd +# server for now). For dogstatsd this is ignored if +# 'dogstatsd_non_local_traffic' is set to true +# bind_host: localhost +# +# Dogstatsd can also listen for metrics on a Unix Socket (*nix only). +# Set to a valid filesystem path to enable. +# dogstatsd_socket: /var/run/dogstatsd/dsd.sock +# +# When using Unix Socket, dogstatsd can tag metrics with container metadata. +# If running dogstatsd in a container, host PID mode (e.g. with --pid=host) is required. +# dogstatsd_origin_detection: false +# +# The buffer size use to receive statsd packet, in bytes +# dogstatsd_buffer_size: 8192 +# +# Whether dogstatsd should listen to non local UDP traffic +# dogstatsd_non_local_traffic: false +# +# Publish dogstatsd's internal stats as Go expvars +# dogstatsd_stats_enable: false +# +# How many items in the dogstatsd's stats circular buffer +# dogstatsd_stats_buffer: 10 +# +# The port for the go_expvar server +# dogstatsd_stats_port: 5000 +# +# The number of bytes allocated to dogstatsd's socket receive buffer (POSIX +# system only). By default, this value is set by the system. If you need to +# increase the size of this buffer but keep the OS default value the same, you +# can set dogstatsd's receive buffer size here. The maximum accepted value +# might change depending on the OS. +# dogstatsd_so_rcvbuf: +# +# Additional tags to append to all metrics, events and service checks received by +# this dogstatsd server. Useful for tagging all dogstatsd metrics reporting from +# a single host without resorting to host tags. +# dogstatsd_tags: +# - name:value +# +# If you want to forward every packet received by the dogstatsd server +# to another statsd server, uncomment these lines. +# WARNING: Make sure that forwarded packets are regular statsd packets and not "dogstatsd" packets, +# as your other statsd server might not be able to handle them. +# statsd_forward_host: address_of_own_statsd_server +# statsd_forward_port: 8125 +# +# If you want all statsd metrics coming from this host to be namespaced +# you can configure the namspace below. Each metric received will be prefixed +# with the namespace before it's sent to StackState. +# statsd_metric_namespace: + +# Logs agent +# +# Logs agent is disabled by default +# logs_enabled: false +# +# Enable logs collection for all containers, disabled by default +# logs_config: +# container_collect_all: false +# + +# JMX +# +# jmx_pipe_path: +# jmx_pipe_name: sts-auto_discovery +# +# If you only run Autodiscovery tests, jmxfetch might fail to pick up custom_jar_paths +# set in the check templates. If that is the case, you can force custom jars here. +# jmx_custom_jars: +# - /jmx-jars/jboss-cli-client.jar +# +# When running in a memory cgroup, openjdk 8u131 and higher can automatically adjust +# its heap memory usage in accordance to the cgroup/container's memory limit. +# Default is false: we'll set a Xmx of 200MB if none is configured. +# Note: older openjdk versions and other jvms might fail to start if this option is set +# +# jmx_use_cgroup_memory_limit: true +# +# Number of JMX restarts allowed in the restart-interval before giving up +# jmx_max_restarts: 3 +# +# Duration of the restart interval in seconds +# jmx_restart_interval: 5 +# +# JMXFetch collects multiples instances concurrently. The following options may +# help fine-tune the level of concurrency and timeouts that come into play during the +# collection of metrics from configured instances: +# +# Defines the maximum level of concurrency. Higher concurrency will increase CPU +# utilization during metric collection. Lower concurrency will result in lower CPU +# usage but may increase the total collection time - a value of 1 will process +# instances serially. The total collection is allowed to take up to `jmx_collection_timeout` +# seconds. +# jmx_thread_pool_size: 3 +# +# Defines the maximum waiting period in seconds before timing up on metric collection. +# jmx_collection_timeout: 60 +# +# Defines the maximum level of concurrency. Higher concurrency will increase CPU +# utilization during reconnection. Lower concurrency will result in lower CPU +# usage but may increase the total reconnection time - a value of 1 will process +# instance reconnections serially. In total, reconnections are allowed to take up to +# `jmx_reconnection_timeout` seconds. +# jmx_reconnection_thread_pool_size: 3 +# +# Determines the maximum waiting period in seconds before timing up on instance reconnection. +# jmx_reconnection_timeout: 10 +# + +# Autoconfig +# +# Directory containing configuration templates +# autoconf_template_dir: /stackstate/check_configs +# +# The providers the Agent should call to collect checks configurations. +# Please note the File Configuration Provider is enabled by default and cannot +# be configured. +# config_providers: + +## The kubelet provider handles templates embedded in pod annotations, see +## https://docs.stackstate.com/guides/autodiscovery/#template-source-kubernetes-pod-annotations +# - name: kubelet +# polling: true + +## The docker provider handles templates embedded in container labels, see +## https://docs.stackstate.com/guides/autodiscovery/#template-source-docker-label-annotations +# - name: docker +# polling: true + +## The clustercheck provider retrieves cluster-level check configurations +## from the cluster-agent +# - name: clusterchecks +# grace_time_seconds: 60# - name: etcd +# polling: true +# template_dir: /stackstate/check_configs +# template_url: http://127.0.0.1 +# username: +# password: + +# - name: consul +# polling: true +# template_dir: /stackstate/check_configs +# template_url: http://127.0.0.1 +# ca_file: +# ca_path: +# cert_file: +# key_file: +# username: +# password: +# token: + +# - name: zookeeper +# polling: true +# template_dir: /stackstate/check_configs +# template_url: 127.0.0.1 +# username: +# password: + +## You can also add additional config providers by name using their default settings, +## and pooling enabled. This list is available as an environment variable binding. +# +# extra_config_providers: +# - clusterchecks + + +# Logging +# +# log_level: info +# log_file: /var/log/stackstate-agent/agent.log + +# Set to 'true' to output logs in JSON format +# log_format_json: false + +# Set to 'false' to disable logging to stdout +# log_to_console: true + +# Set to 'true' to disable logging to the log file +# disable_file_logging: false + +# Set to 'true' to enable logging to syslog. +# Note: Even if this option is set to 'false', the service launcher of your environment +# may redirect the agent process' stdout/stderr to syslog. In that case, if you wish +# to disable logging to syslog entirely, please set 'log_to_console' to 'false' as well. +# log_to_syslog: false +# +# If 'syslog_uri' is left undefined/empty, a local domain socket connection will be attempted +# +# syslog_uri: +# +# Set to 'true' to output in an RFC 5424-compliant format +# +# syslog_rfc: false +# +# If TLS enabled, you must specify a path to a PEM certificate here +# +# syslog_pem: /path/to/certificate.pem +# +# If TLS enabled, you must specify a path to a private key here +# +# syslog_key: /path/to/key.pem +# +# If TLS enabled, you may enforce TLS verification here (defaults to true) +# +# syslog_tls_verify: true +# + +# Autodiscovery +# +# Change the root directory to look at to get cgroup statistics. Useful when running inside a +# container with host directories mounted on a different folder. +# Default if environment variable "DOCKER_STS_AGENT" is set +# "/host/sys/fs/cgroup" and "/sys/fs/cgroup" if not. +# +# container_cgroup_root: /host/sys/fs/cgroup/ +# +# Change the root directory to look at to get proc statistics. Useful when running inside a +# container with host directories mounted on a different folder. +# Default if environment variable "DOCKER_STS_AGENT" is set +# "/host/proc" and "/proc" if not. +# +# container_proc_root: /host/proc +# +# Choose "auto" if you want to let the agent find any relevant listener on your host +# At the moment, the only auto listener supported is docker +# If you have already set docker anywhere in the listeners, the auto listener is ignored +# listeners: +# - name: auto +# - name: docker +# +## You can also add additional listeners by name using their default settings. +## This list is available as an environment variable binding. +# +# extra_listeners: +# - kubelet +# +# Exclude containers from metrics and AD based on their name or image: +# An excluded container will not get any individual container metric reported for it. +# Please note that the `docker.containers.running`, `.stopped`, `.running.total` and +# `.stopped.total` metrics are not affected by these settings and always count all +# containers. This does not affect your per-container billing. +# +# How it works: include first. +# If a container matches an exclude rule, it won't be included unless it first matches an include rule. +# +# Rules are regexp. +# +# Examples: +# exclude all, except containers based on the 'ubuntu' image or the 'debian' image. +# ac_exclude: ["image:.*"] +# ac_include: ["image:ubuntu", "image:debian"] +# +# include all, except containers based on the 'ubuntu' image. +# ac_exclude: ["image:ubuntu"] +# ac_include: [] +# +# exclude all debian images except containers with a name starting with 'frontend'. +# ac_exclude: ["image:debian"] +# ac_include: ["name:frontend.*"] +# +# ac_exclude: [] +# ac_include: [] +# +# +# Exclude default pause containers from orchestrators. +# +# By default the agent will not monitor kubernetes/openshift pause +# container. They will still be counted in the container count (just like +# excluded containers) since ignoring them would give a wrong impression +# about the docker daemon load. +# +# exclude_pause_container: true + +# Exclude default containers from DockerCloud: +# The following configuration will instruct the agent to ignore the containers from Docker Cloud. +# You can remove the ones you want to collect. +# ac_exclude: ["image:dockercloud/network-daemon","image:dockercloud/cleanup","image:dockercloud/logrotate","image:dockercloud/events","image:dockercloud/ntpd"] +# ac_include: [] +# +# You can also use the regex to ignore them all: +# ac_exclude: ["image:dockercloud/*"] +# ac_include: [] +# +# The default timeout value when connecting to the docker daemon +# is 5 seconds. It can be configured with this option. +# docker_query_timeout: 5 +# +# The default interval in second to check for new autodiscovery configurations +# On all registered configuration providers +# ad_config_poll_interval: 10 +# + +# Container detection +# +# On hosts with mixed workloads, non-containernized processes can +# mistakenly be detected as containerized. This option can be used to +# tune the detection logic to your system and avoid false-positives. +# +# container_cgroup_prefix: "/docker/" +# +# Docker tag extraction +# +# We can extract container label or environment variables +# as metric tags. If you prefix your tag name with +, it +# will only be added to high cardinality metrics (docker check) +# +# docker_labels_as_tags: +# label_name: tag_name +# high_cardinality_label_name: +tag_name +# docker_env_as_tags: +# ENVVAR_NAME: tag_name +# +# Example: +# docker_labels_as_tags: +# com.docker.compose.service: service_name +# com.docker.compose.project: +project_name +# + +# Kubernetes tag extraction +# +# We can extract pod labels and annotations as metric tags. If you prefix your +# tag name with +, it will only be added to high cardinality metrics +# +# kubernetes_pod_labels_as_tags: +# app: kube_app +# pod-template-hash: +kube_pod-template-hash +# +# kubernetes_pod_annotations_as_tags: +# app: kube_app +# pod-template-hash: +kube_pod-template-hash +# + +# ECS integration +# +# The ECS agent container should be autodetected when running with the +# default (ecs-agent) name. Else, you can change the container name the +# agent will look for, or force a fixed url: +# ecs_agent_container_name: ecs-agent +# ecs_agent_url: http://localhost:51678 +# +# Fargate clusters use other endpoints and are not affected by these options. +# + +# CRI integration +# +# To activate the CRI check you'll need to indicate the path of the +# CRI runtime you're using (and mount it in the container if needed) +# cri_socket_path: /var/run/containerd/containerd.sock +# +# You can configure the initial connection timeout (in seconds) +# cri_connection_timeout: 1 +# +# You can configure the timeout (in seconds) for querying the CRI +# cri_query_timeout: 5 +# + +# Containerd integration +# +# To activate the Containerd check you'll need to indicate the path of the +# Containerd socket you're using (and mount it in the container if needed) +# cri_socket_path: /var/run/containerd/containerd.sock +# +# You can configure the timeout (in seconds) for querying the Containerd API +# cri_query_timeout: 5 +# +# Activating the Containerd check will also activate the CRI check, as it contains an additional subset of useful metrics. +# +# You can specify the namespace that Containerd is using on your system. +# As the Containerd check only supports Kubernetes, the default value is `k8s.io` +# https://github.com/containerd/cri/blob/release/1.2/pkg/constants/constants.go#L22-L23 +# containerd_namespace: k8s.io + +# Kubernetes kubelet connectivity +# +# The kubelet host and port should be autodetected when running inside a pod. +# If you run into connectivity issues, you can set these options according to +# your cluster setup: +# kubernetes_kubelet_host: autodetected +# kubernetes_http_kubelet_port: 10255 +# kubernetes_https_kubelet_port: 10250 +# +# When using HTTPS, we verify the kubelet's certificate, you can tune this: +# kubelet_tls_verify: true +# kubelet_client_ca: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt +# +# If authentication is needed, the agent will use the pod's serviceaccount's +# credentials. If you want to use a different account, or are running the agent +# on the host, you can set the credentials to use here: +# kubelet_auth_token_path: /path/to/file +# kubelet_client_crt: /path/to/key +# kubelet_client_key: /path/to/key +# +# On some kubelet versions, containers can take up to a second to +# register in the podlist. This option allows to wait for up to a given +# number of seconds (in 250ms chunks) when a container does not exist in the podlist. +# kubelet_wait_on_missing_container: 0 +# +# Polling frequency in seconds of the agent to the kubelet "/pods" endpoint +# kubelet_cache_pods_duration: 5 +# + +# Kubernetes apiserver integration +# +# When running in a pod, the agent will automatically use the pod's serviceaccount +# to authenticate with the apiserver. If you wish to install the agent out of a pod +# or customise connection parameters, you can provide the path to a KubeConfig file +# see https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/ +# +# kubernetes_kubeconfig_path: /path/to/file +# +# By default, communication with the apiserver is in json format. Setting the following +# option to true will allow communication in the binary protobuf format, with a potential +# performance improvement on both the agent and the apiserver. +# kubernetes_apiserver_use_protobuf: false +# +# In order to collect Kubernetes service names, the agent needs certain rights (see RBAC documentation in +# [docker readme](https://github.com/StackVista/stackstate-agent/blob/master/Dockerfiles/agent/README.md#kubernetes)). +# You can disable this option or set how often (in seconds) the agent refreshes the internal mapping of services to +# ContainerIDs with the following options: +# kubernetes_collect_metadata_tags: true +# kubernetes_metadata_tag_update_freq: 60 +# kubernetes_apiserver_client_timeout: 10 +# +# To collect Kubernetes events, leader election must be enabled and collect_kubernetes_events set to true. +# Only the leader will collect events. More details about events [here](https://github.com/StackVista/stackstate-agent/blob/master/Dockerfiles/agent/README.md#event-collection). +# collect_kubernetes_events: false +# To collect Kubernetes metrics, leader election must be enabled and collect_kubernetes_metrics set to true. +# collect_kubernetes_metrics: false +# To collect Kubernetes topology, leader election must be enabled and collect_kubernetes_topology set to true. +# collect_kubernetes_topology: true +# +# +# Leader Election settings, more details about leader election [here](https://github.com/StackVista/stackstate-agent/blob/master/Dockerfiles/agent/README.md#leader-election) +# To enable the leader election on this node, set the leader_election variable to true. +# leader_election: false +# The leader election lease is an integer in seconds. +# leader_lease_duration: 60 +# +# Node labels that should be collected and their name in host tags. Off by default. +# Some of these labels are redundant with metadata collected by +# cloud provider crawlers (AWS, GCE, Azure) +# +# kubernetes_node_labels_as_tags: +# kubernetes.io/hostname: nodename +# beta.kubernetes.io/os: os +# +# Kubernetes cluster identifier used to avoid host alias collisions. Empty by default. +# clustername: cluster_identifier + +# Docker Swarm +# To collect Docker Swarm topology, collect_swarm_topology set to true. +# collect_swarm_topology: true + +# Process agent specific settings +# +process_config: + # A string indicating the enabled state of the Process Agent. + # If "false" (the default) it will only collect containers. + # If "true" it will collect containers and processes. + # If "disabled" it will be disabled altogether and won't start. + enabled: "true" + # Enable/disable network tracing + process_sts_url: + # The full path to the file where process-agent logs will be written. + # log_file: + # The interval, in seconds, at which we will run each check. If you want consistent + # behavior between real-time you may set the Container/ProcessRT intervals to 10. + # Defaults to 10s for normal checks and 2s for others. + intervals: + container: 30 + process: 30 + connections: 30 + ## Relation Cache is used to keep state of the known network relations for this agent, with items expiring after relation_cache_duration minutes. The default is 5 minutes. + # relation_cache_duration_min: 5 + ## Process Cache is used to keep state of the known processes for this agent, with items expiring after process_cache_duration minutes. The default is 5 minutes. + # process_cache_duration_min: 5 + ## Filters are used to filter out processes or network connections based on some condition + # filters: + ## The Short-Lived Processes filter is used to filter out process that are observed for less than x seconds, with the default being 60 seconds. + # short_lived_processes: + # enabled: true + # qualifier_secs: 60 + ## The Short-Lived Relations filter is used to filter out network relations that are observed for less than x seconds, with the default being 60 seconds. + ## Short-Lived network relations are defined as network connections that do not occur frequently between processes / services. Multiple short-lived connections + ## between the same processes / services are considered a Long-Lived network relation, while a once-off network connection is filtered out and not reported to StackState. + # short_lived_network_relations: + # enabled: true + # qualifier_secs: 60 + + ## Process blacklisting is used to filter out unwanted processes and not report them to StackState. + process_blacklist: + # patterns: + # - "regex-pattern" + ## Inclusions override the blacklist patterns, these are used to include process that consume a lot of resources. + ## Each inclusion type: top_cpu, top_io_read, top_io_write and top_mem have an amount of processes that get's considered + ## as the top x resource using processes. For top_cpu and top_mem there is a threshold that needs to be met first, meaning + ## the process needs to consume more resource % than the threshold. + inclusions: + amount_top_cpu_pct_usage: 3 + cpu_pct_usage_threshold: 20 + amount_top_io_read_usage: 3 + amount_top_io_write_usage: 3 + amount_top_mem_usage: 3 + mem_usage_threshold: 35 + # How many check results to buffer in memory when POST fails. The default is usually fine. + queue_size: 10 +# The maximum number of file descriptors to open when collecting net connections. +# Only change if you are running out of file descriptors from the Agent. +# max_proc_fds: +# The maximum number of processes or containers per message. +# Only change if the defaults are causing issues. +# max_per_message: +# Overrides the path to the Agent bin used for getting the hostname. The default is usually fine. +# sts_agent_bin: +# Overrides of the environment we pass to fetch the hostname. The default is usually fine. +# sts_agent_env: +# +# Network tracer specific settings +# +network_tracer_config: + network_tracing_enabled: "true" + initial_connections_from_proc: "true" + +# Trace Agent Specific Settings +# +apm_config: + # Whether or not the APM Agent should run + enabled: true + apm_sts_url: + # The environment tag that Traces should be tagged with + # Will inherit from "env" tag if "none" is applied here + # env: none + # The port that the Receiver should listen on + receiver_port: 8126 + # Whether the Trace Agent should listen for non local traffic + # Only enable if Traces are being sent to this Agent from another host/container + apm_non_local_traffic: true +# Extra global sample rate to apply on all the traces +# This sample rate is combined to the sample rate from the sampler logic, still promoting interesting traces +# From 1 (no extra rate) to 0 (don't sample at all) +# extra_sample_rate: 1.0 +# Maximum number of traces per second to sample. +# The limit is applied over an average over a few minutes ; much bigger spikes are possible. +# Set to 0 to disable the limit. +# max_traces_per_second: 10 +# A blacklist of regular expressions can be provided to disable certain traces based on their resource name +# all entries must be surrounded by double quotes and separated by commas +# Example: ["(GET|POST) /healthcheck", "GET /V1"] +# ignore_resources: [] +## features retrieves the features supported by the StackState backend so that we can toggle agent functionality +#features: +# retry_interval_millis: 5000 +# max_retries: 10 diff --git a/pkg/collector/corechecks/system/disk.go b/pkg/collector/corechecks/system/disk.go index 9c4e8be7aa87..e20829bab79a 100644 --- a/pkg/collector/corechecks/system/disk.go +++ b/pkg/collector/corechecks/system/disk.go @@ -92,16 +92,23 @@ func (c *DiskCheck) instanceConfigure(data integration.Data) error { } excludedFilesystems, found := conf["excluded_filesystems"] - if excludedFilesystems, ok := excludedFilesystems.([]string); found && ok { - c.cfg.excludedFilesystems = excludedFilesystems + if excludedFilesystems, ok := excludedFilesystems.([]interface{}); found && ok { + excludedList := make([]string, len(excludedFilesystems)) + for i, ex := range excludedFilesystems { + excludedList[i] = ex.(string) + } + c.cfg.excludedFilesystems = excludedList } - // Force exclusion of CDROM (iso9660) from disk check c.cfg.excludedFilesystems = append(c.cfg.excludedFilesystems, "iso9660") excludedDisks, found := conf["excluded_disks"] - if excludedDisks, ok := excludedDisks.([]string); found && ok { - c.cfg.excludedDisks = excludedDisks + if excludedDisks, ok := excludedDisks.([]interface{}); found && ok { + excludedList := make([]string, len(excludedDisks)) + for i, ex := range excludedDisks { + excludedList[i] = ex.(string) + } + c.cfg.excludedDisks = excludedList } excludedDiskRe, found := conf["excluded_disk_re"] From ab94127699627ce0f5d48a778cd83cde0a742e99 Mon Sep 17 00:00:00 2001 From: Melcom van Eeden Date: Tue, 13 Apr 2021 09:24:19 +0200 Subject: [PATCH 2/4] Release notes updated --- stackstate-changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stackstate-changelog.md b/stackstate-changelog.md index eb1cbeb48c06..6c65bb906be2 100644 --- a/stackstate-changelog.md +++ b/stackstate-changelog.md @@ -21,6 +21,8 @@ - Kubelet check should not fail for Kubernetes 1.18+ (due to deprecated `/spec` API endpoint) [(STAC-12307)](https://stackstate.atlassian.net/browse/STAC-12307) - Remove the tag for process components with high I/O or CPU. [(STAC-12306)](https://stackstate.atlassian.net/browse/STAC-12306) + - Fixed the core check for the Excluded Filesystems to use the conf file variable. + - Fixed the core check for the Excluded Disks to use the conf file variable. ## 2.10.0 (2021-02-25) From 95e2e63bb2043478880f7b4a69c625174862400b Mon Sep 17 00:00:00 2001 From: Melcom van Eeden Date: Tue, 13 Apr 2021 11:03:00 +0200 Subject: [PATCH 3/4] Added base config file system exclusions && Update the release notes && Added unit tests --- .../dist/conf.d/disk.d/conf.yaml.default | 5 ++- pkg/collector/corechecks/system/disk_test.go | 38 +++++++++++++++++++ stackstate-changelog.md | 6 ++- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/cmd/agent/dist/conf.d/disk.d/conf.yaml.default b/cmd/agent/dist/conf.d/disk.d/conf.yaml.default index 36536f58d58e..28c3c24a3c13 100644 --- a/cmd/agent/dist/conf.d/disk.d/conf.yaml.default +++ b/cmd/agent/dist/conf.d/disk.d/conf.yaml.default @@ -7,8 +7,9 @@ instances: # The (optional) excluded_filesystems parameter will instruct the check to # ignore disks using these filesystems. Note: On some linux distributions, # rootfs will be found and tagged as a device, add rootfs here to exclude. - # excluded_filesystems: - # - tmpfs + excluded_filesystems: + - tmpfs + - squashfs # The (optional) excluded_disks parameter will instruct the check to # ignore this list of disks. diff --git a/pkg/collector/corechecks/system/disk_test.go b/pkg/collector/corechecks/system/disk_test.go index 5e5d637b33ab..d4a23e9efa54 100644 --- a/pkg/collector/corechecks/system/disk_test.go +++ b/pkg/collector/corechecks/system/disk_test.go @@ -264,3 +264,41 @@ func TestDiskCheckTags(t *testing.T) { mock.AssertNumberOfCalls(t, "Rate", expectedRates) mock.AssertNumberOfCalls(t, "Commit", 1) } + +func TestExcludedDiskFSFromConfig(t *testing.T) { + for _, tc := range []struct{ + test string + config integration.Data + excludedDisks []string + excludedFileSystems []string + }{ + { + test: "No file system and disk exclusions", + config: integration.Data("use_mount: true"), + excludedDisks: []string(nil), + excludedFileSystems: []string{"iso9660"}, + }, + { + test: "Exclude file systems", + config: integration.Data("use_mount: true\nexcluded_filesystems: \n - tmpfs\n - squashfs"), + excludedDisks: []string(nil), + excludedFileSystems: []string{"iso9660", "tmpfs", "squashfs"}, + }, + { + test: "Exclude disks", + config: integration.Data("use_mount: true\nexcluded_disks: \n - /dev/nvme0n1p1\n - /dev/sda1\n - /dev/sda2"), + excludedDisks: []string{"/dev/nvme0n1p1", "/dev/sda1", "/dev/sda2"}, + excludedFileSystems: []string{"iso9660"}, + }, + }{ + t.Run(tc.test, func(t *testing.T) { + diskPartitions = diskSampler + diskCheck := diskFactory().(*DiskCheck) + err := diskCheck.Configure(tc.config, nil) + + assert.NoError(t, err) + assert.ElementsMatch(t, diskCheck.cfg.excludedDisks, tc.excludedDisks) + assert.ElementsMatch(t, diskCheck.cfg.excludedFilesystems, tc.excludedFileSystems) + }) + } +} diff --git a/stackstate-changelog.md b/stackstate-changelog.md index 6c65bb906be2..f9ef603a00de 100644 --- a/stackstate-changelog.md +++ b/stackstate-changelog.md @@ -19,10 +19,12 @@ **Bugfix** +- Disk Integration + - Fixed the excluded filesystems and excluded disks failing to use the conf file. +- Integrations: - Kubelet check should not fail for Kubernetes 1.18+ (due to deprecated `/spec` API endpoint) [(STAC-12307)](https://stackstate.atlassian.net/browse/STAC-12307) - Remove the tag for process components with high I/O or CPU. [(STAC-12306)](https://stackstate.atlassian.net/browse/STAC-12306) - - Fixed the core check for the Excluded Filesystems to use the conf file variable. - - Fixed the core check for the Excluded Disks to use the conf file variable. + ## 2.10.0 (2021-02-25) From 18e219e597734192cf7bac02a741894ddba844d5 Mon Sep 17 00:00:00 2001 From: Melcom van Eeden Date: Tue, 13 Apr 2021 14:48:29 +0200 Subject: [PATCH 4/4] Removed nil values --- pkg/collector/corechecks/system/disk_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/collector/corechecks/system/disk_test.go b/pkg/collector/corechecks/system/disk_test.go index d4a23e9efa54..069ff0ca396c 100644 --- a/pkg/collector/corechecks/system/disk_test.go +++ b/pkg/collector/corechecks/system/disk_test.go @@ -275,13 +275,11 @@ func TestExcludedDiskFSFromConfig(t *testing.T) { { test: "No file system and disk exclusions", config: integration.Data("use_mount: true"), - excludedDisks: []string(nil), excludedFileSystems: []string{"iso9660"}, }, { test: "Exclude file systems", config: integration.Data("use_mount: true\nexcluded_filesystems: \n - tmpfs\n - squashfs"), - excludedDisks: []string(nil), excludedFileSystems: []string{"iso9660", "tmpfs", "squashfs"}, }, {