diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 954f75f844..b1469c0ac2 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -63744,6 +63744,199 @@ components: x-enum-varnames: - BLOCK - DROP_NEWEST + ObservabilityPipelineClickhouseDestination: + description: |- + The `clickhouse` destination sends log events to a ClickHouse database table over HTTP. + + **Supported pipeline types:** logs. + properties: + auth: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationAuth" + batch: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatch" + batch_encoding: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatchEncoding" + compression: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompression" + database: + description: Optional ClickHouse database name. If omitted, the user's default database on the ClickHouse server is used. + example: my_database + type: string + date_time_best_effort: + description: When `true`, enables flexible DateTime parsing on the ClickHouse server side. + example: false + type: boolean + endpoint_url_key: + description: |- + Name of the environment variable or secret that contains the ClickHouse HTTP endpoint URL. + Defaults to `DESTINATION_CLICKHOUSE_ENDPOINT_URL` (prefixed with `DD_OP_` at runtime). + example: CLICKHOUSE_ENDPOINT_URL + type: string + format: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationFormat" + id: + description: The unique identifier for this component. + example: clickhouse-destination + type: string + inputs: + description: A list of component IDs whose output is used as the `input` for this component. + example: ["filter-processor"] + items: + description: The ID of a component whose output is used as input for this destination. + type: string + type: array + skip_unknown_fields: + description: |- + When `true`, fields not present in the target table schema are dropped instead of causing insert errors. + When unset, the ClickHouse server's own `input_format_skip_unknown_fields` setting applies. + example: true + nullable: true + type: boolean + table: + description: Target ClickHouse table name. Events are inserted into this table. + example: application_logs + type: string + tls: + $ref: "#/components/schemas/ObservabilityPipelineTls" + type: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationType" + required: + - id + - type + - inputs + - table + type: object + x-pipeline-types: [logs] + ObservabilityPipelineClickhouseDestinationAuth: + description: |- + HTTP Basic Authentication credentials for the ClickHouse destination. + When `strategy` is `basic`, provide `username_key` and `password_key` that reference environment variables or secrets containing the credentials. + properties: + password_key: + description: Name of the environment variable or secret that contains the ClickHouse password. + example: CLICKHOUSE_PASSWORD + type: string + strategy: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationAuthStrategy" + username_key: + description: Name of the environment variable or secret that contains the ClickHouse username. + example: CLICKHOUSE_USERNAME + type: string + required: + - strategy + type: object + ObservabilityPipelineClickhouseDestinationAuthStrategy: + description: The authentication strategy for ClickHouse HTTP requests. Only `basic` is supported. + enum: + - basic + example: basic + type: string + x-enum-varnames: + - BASIC + ObservabilityPipelineClickhouseDestinationBatch: + description: Batching configuration for ClickHouse inserts. + properties: + max_events: + description: Maximum number of events per batch before it is flushed. + example: 1000 + format: int64 + minimum: 1 + type: integer + timeout_secs: + description: Maximum number of seconds to wait before flushing a partial batch. + example: 1 + format: int64 + maximum: 65535 + minimum: 1 + type: integer + type: object + ObservabilityPipelineClickhouseDestinationBatchEncoding: + description: |- + Batch encoding configuration for the ClickHouse destination. + Required when `format` is `arrow_stream`. The `codec` field must be set to `arrow_stream`. + properties: + allow_nullable_fields: + description: |- + When `true`, null values are allowed for non-nullable fields in the ClickHouse schema. + When `false` (default), missing values for non-nullable columns cause encoding errors. + example: false + type: boolean + codec: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatchEncodingCodec" + required: + - codec + type: object + ObservabilityPipelineClickhouseDestinationBatchEncodingCodec: + description: The codec used for batch encoding. Only `arrow_stream` is supported. + enum: + - arrow_stream + example: arrow_stream + type: string + x-enum-varnames: + - ARROW_STREAM + ObservabilityPipelineClickhouseDestinationCompression: + description: |- + Compression setting for outbound HTTP requests to ClickHouse. + Can be specified as a shorthand string (`"gzip"` or `"none"`) or as an object + with an `algorithm` field and an optional `level` (gzip only, 1–9). + oneOf: + - $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompressionAlgorithm" + - $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompressionObject" + ObservabilityPipelineClickhouseDestinationCompressionAlgorithm: + description: The compression algorithm applied to outbound HTTP requests. + enum: + - gzip + - none + example: gzip + type: string + x-enum-varnames: + - GZIP + - NONE + ObservabilityPipelineClickhouseDestinationCompressionObject: + description: |- + Structured compression configuration for the ClickHouse destination. + Use `algorithm` to specify the compression type and `level` (optional, gzip only) to control compression strength. + properties: + algorithm: + $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompressionAlgorithm" + level: + description: Compression level (1–9). Only applicable when `algorithm` is `gzip`. + example: 6 + format: int64 + maximum: 9 + minimum: 1 + type: integer + required: + - algorithm + type: object + ObservabilityPipelineClickhouseDestinationFormat: + description: |- + Insert format for events sent to ClickHouse. + - `json_each_row`: Maps event fields to columns by name (ClickHouse `JSONEachRow`). + - `json_as_object`: Inserts each event into a single `Object('json')` / `JSON` column (ClickHouse `JSONAsObject`). + - `json_as_string`: Inserts each event into a single `String`-typed column as raw JSON (ClickHouse `JSONAsString`). + - `arrow_stream`: Batches events using Apache Arrow IPC streaming format. Requires `batch_encoding`. + enum: + - json_each_row + - json_as_object + - json_as_string + - arrow_stream + example: json_each_row + type: string + x-enum-varnames: + - JSON_EACH_ROW + - JSON_AS_OBJECT + - JSON_AS_STRING + - ARROW_STREAM + ObservabilityPipelineClickhouseDestinationType: + default: clickhouse + description: The destination type. The value must be `clickhouse`. + enum: + - clickhouse + example: clickhouse + type: string + x-enum-varnames: + - CLICKHOUSE ObservabilityPipelineCloudPremDestination: description: |- The `cloud_prem` destination sends logs to Datadog CloudPrem. @@ -63843,6 +64036,7 @@ components: - $ref: "#/components/schemas/ObservabilityPipelineAmazonS3GenericDestination" - $ref: "#/components/schemas/ObservabilityPipelineAmazonSecurityLakeDestination" - $ref: "#/components/schemas/AzureStorageDestination" + - $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestination" - $ref: "#/components/schemas/ObservabilityPipelineCloudPremDestination" - $ref: "#/components/schemas/ObservabilityPipelineCrowdStrikeNextGenSiemDestination" - $ref: "#/components/schemas/ObservabilityPipelineDatadogLogsDestination" diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index d5921157a1..d1d4e51d06 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -26590,6 +26590,83 @@ datadog\_api\_client.v2.model.observability\_pipeline\_buffer\_options\_when\_fu :members: :show-inheritance: +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination module +------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_auth module +------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_auth\_strategy module +----------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth_strategy + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_batch module +-------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_batch\_encoding module +------------------------------------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_batch\_encoding\_codec module +------------------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding_codec + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_compression module +-------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_compression\_algorithm module +------------------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_algorithm + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_compression\_object module +---------------------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_object + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_format module +--------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_format + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.observability\_pipeline\_clickhouse\_destination\_type module +------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_type + :members: + :show-inheritance: + datadog\_api\_client.v2.model.observability\_pipeline\_cloud\_prem\_destination module -------------------------------------------------------------------------------------- diff --git a/examples/v2/observability-pipelines/ValidatePipeline_1238441879.py b/examples/v2/observability-pipelines/ValidatePipeline_1238441879.py new file mode 100644 index 0000000000..1218ea91d2 --- /dev/null +++ b/examples/v2/observability-pipelines/ValidatePipeline_1238441879.py @@ -0,0 +1,101 @@ +""" +Validate an observability pipeline with ClickHouse destination returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination import ( + ObservabilityPipelineClickhouseDestination, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth import ( + ObservabilityPipelineClickhouseDestinationAuth, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth_strategy import ( + ObservabilityPipelineClickhouseDestinationAuthStrategy, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch import ( + ObservabilityPipelineClickhouseDestinationBatch, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_type import ( + ObservabilityPipelineClickhouseDestinationType, +) +from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) +from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( + ObservabilityPipelineDatadogAgentSource, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import ( + ObservabilityPipelineDatadogAgentSourceType, +) +from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor +from datadog_api_client.v2.model.observability_pipeline_filter_processor_type import ( + ObservabilityPipelineFilterProcessorType, +) +from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec +from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData + +body = ObservabilityPipelineSpec( + data=ObservabilityPipelineSpecData( + attributes=ObservabilityPipelineDataAttributes( + config=ObservabilityPipelineConfig( + destinations=[ + ObservabilityPipelineClickhouseDestination( + id="clickhouse-destination", + inputs=[ + "my-processor-group", + ], + type=ObservabilityPipelineClickhouseDestinationType.CLICKHOUSE, + table="application_logs", + database="my_database", + compression="gzip", + auth=ObservabilityPipelineClickhouseDestinationAuth( + strategy=ObservabilityPipelineClickhouseDestinationAuthStrategy.BASIC, + username_key="CLICKHOUSE_USERNAME", + password_key="CLICKHOUSE_PASSWORD", + ), + batch=ObservabilityPipelineClickhouseDestinationBatch( + max_events=1000, + timeout_secs=1, + ), + ), + ], + processor_groups=[ + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", + include="service:my-service", + inputs=[ + "datadog-agent-source", + ], + processors=[ + ObservabilityPipelineFilterProcessor( + enabled=True, + id="filter-processor", + include="status:error", + type=ObservabilityPipelineFilterProcessorType.FILTER, + ), + ], + ), + ], + sources=[ + ObservabilityPipelineDatadogAgentSource( + id="datadog-agent-source", + type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT, + ), + ], + ), + name="Pipeline with ClickHouse Destination", + ), + type="pipelines", + ), +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = ObservabilityPipelinesApi(api_client) + response = api_instance.validate_pipeline(body=body) + + print(response) diff --git a/examples/v2/observability-pipelines/ValidatePipeline_620504950.py b/examples/v2/observability-pipelines/ValidatePipeline_620504950.py new file mode 100644 index 0000000000..27759307d0 --- /dev/null +++ b/examples/v2/observability-pipelines/ValidatePipeline_620504950.py @@ -0,0 +1,115 @@ +""" +Validate an observability pipeline with ClickHouse destination arrow_stream format returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination import ( + ObservabilityPipelineClickhouseDestination, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth import ( + ObservabilityPipelineClickhouseDestinationAuth, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth_strategy import ( + ObservabilityPipelineClickhouseDestinationAuthStrategy, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch import ( + ObservabilityPipelineClickhouseDestinationBatch, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding import ( + ObservabilityPipelineClickhouseDestinationBatchEncoding, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding_codec import ( + ObservabilityPipelineClickhouseDestinationBatchEncodingCodec, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_format import ( + ObservabilityPipelineClickhouseDestinationFormat, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_type import ( + ObservabilityPipelineClickhouseDestinationType, +) +from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) +from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( + ObservabilityPipelineDatadogAgentSource, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import ( + ObservabilityPipelineDatadogAgentSourceType, +) +from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor +from datadog_api_client.v2.model.observability_pipeline_filter_processor_type import ( + ObservabilityPipelineFilterProcessorType, +) +from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec +from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData + +body = ObservabilityPipelineSpec( + data=ObservabilityPipelineSpecData( + attributes=ObservabilityPipelineDataAttributes( + config=ObservabilityPipelineConfig( + destinations=[ + ObservabilityPipelineClickhouseDestination( + id="clickhouse-destination", + inputs=[ + "my-processor-group", + ], + type=ObservabilityPipelineClickhouseDestinationType.CLICKHOUSE, + table="application_logs", + database="my_database", + format=ObservabilityPipelineClickhouseDestinationFormat.ARROW_STREAM, + batch_encoding=ObservabilityPipelineClickhouseDestinationBatchEncoding( + codec=ObservabilityPipelineClickhouseDestinationBatchEncodingCodec.ARROW_STREAM, + allow_nullable_fields=False, + ), + compression="gzip", + auth=ObservabilityPipelineClickhouseDestinationAuth( + strategy=ObservabilityPipelineClickhouseDestinationAuthStrategy.BASIC, + username_key="CLICKHOUSE_USERNAME", + password_key="CLICKHOUSE_PASSWORD", + ), + batch=ObservabilityPipelineClickhouseDestinationBatch( + max_events=1000, + timeout_secs=1, + ), + ), + ], + processor_groups=[ + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", + include="service:my-service", + inputs=[ + "datadog-agent-source", + ], + processors=[ + ObservabilityPipelineFilterProcessor( + enabled=True, + id="filter-processor", + include="status:error", + type=ObservabilityPipelineFilterProcessorType.FILTER, + ), + ], + ), + ], + sources=[ + ObservabilityPipelineDatadogAgentSource( + id="datadog-agent-source", + type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT, + ), + ], + ), + name="Pipeline with ClickHouse Destination Arrow Stream", + ), + type="pipelines", + ), +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = ObservabilityPipelinesApi(api_client) + response = api_instance.validate_pipeline(body=body) + + print(response) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination.py new file mode 100644 index 0000000000..f11f39f9e7 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination.py @@ -0,0 +1,205 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + none_type, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth import ( + ObservabilityPipelineClickhouseDestinationAuth, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch import ( + ObservabilityPipelineClickhouseDestinationBatch, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding import ( + ObservabilityPipelineClickhouseDestinationBatchEncoding, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression import ( + ObservabilityPipelineClickhouseDestinationCompression, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_format import ( + ObservabilityPipelineClickhouseDestinationFormat, + ) + from datadog_api_client.v2.model.observability_pipeline_tls import ObservabilityPipelineTls + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_type import ( + ObservabilityPipelineClickhouseDestinationType, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_object import ( + ObservabilityPipelineClickhouseDestinationCompressionObject, + ) + + +class ObservabilityPipelineClickhouseDestination(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth import ( + ObservabilityPipelineClickhouseDestinationAuth, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch import ( + ObservabilityPipelineClickhouseDestinationBatch, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding import ( + ObservabilityPipelineClickhouseDestinationBatchEncoding, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression import ( + ObservabilityPipelineClickhouseDestinationCompression, + ) + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_format import ( + ObservabilityPipelineClickhouseDestinationFormat, + ) + from datadog_api_client.v2.model.observability_pipeline_tls import ObservabilityPipelineTls + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_type import ( + ObservabilityPipelineClickhouseDestinationType, + ) + + return { + "auth": (ObservabilityPipelineClickhouseDestinationAuth,), + "batch": (ObservabilityPipelineClickhouseDestinationBatch,), + "batch_encoding": (ObservabilityPipelineClickhouseDestinationBatchEncoding,), + "compression": (ObservabilityPipelineClickhouseDestinationCompression,), + "database": (str,), + "date_time_best_effort": (bool,), + "endpoint_url_key": (str,), + "format": (ObservabilityPipelineClickhouseDestinationFormat,), + "id": (str,), + "inputs": ([str],), + "skip_unknown_fields": (bool, none_type), + "table": (str,), + "tls": (ObservabilityPipelineTls,), + "type": (ObservabilityPipelineClickhouseDestinationType,), + } + + attribute_map = { + "auth": "auth", + "batch": "batch", + "batch_encoding": "batch_encoding", + "compression": "compression", + "database": "database", + "date_time_best_effort": "date_time_best_effort", + "endpoint_url_key": "endpoint_url_key", + "format": "format", + "id": "id", + "inputs": "inputs", + "skip_unknown_fields": "skip_unknown_fields", + "table": "table", + "tls": "tls", + "type": "type", + } + + def __init__( + self_, + id: str, + inputs: List[str], + table: str, + type: ObservabilityPipelineClickhouseDestinationType, + auth: Union[ObservabilityPipelineClickhouseDestinationAuth, UnsetType] = unset, + batch: Union[ObservabilityPipelineClickhouseDestinationBatch, UnsetType] = unset, + batch_encoding: Union[ObservabilityPipelineClickhouseDestinationBatchEncoding, UnsetType] = unset, + compression: Union[ + ObservabilityPipelineClickhouseDestinationCompression, + str, + ObservabilityPipelineClickhouseDestinationCompressionObject, + UnsetType, + ] = unset, + database: Union[str, UnsetType] = unset, + date_time_best_effort: Union[bool, UnsetType] = unset, + endpoint_url_key: Union[str, UnsetType] = unset, + format: Union[ObservabilityPipelineClickhouseDestinationFormat, UnsetType] = unset, + skip_unknown_fields: Union[bool, none_type, UnsetType] = unset, + tls: Union[ObservabilityPipelineTls, UnsetType] = unset, + **kwargs, + ): + """ + The ``clickhouse`` destination sends log events to a ClickHouse database table over HTTP. + + **Supported pipeline types:** logs. + + :param auth: HTTP Basic Authentication credentials for the ClickHouse destination. + When ``strategy`` is ``basic`` , provide ``username_key`` and ``password_key`` that reference environment variables or secrets containing the credentials. + :type auth: ObservabilityPipelineClickhouseDestinationAuth, optional + + :param batch: Batching configuration for ClickHouse inserts. + :type batch: ObservabilityPipelineClickhouseDestinationBatch, optional + + :param batch_encoding: Batch encoding configuration for the ClickHouse destination. + Required when ``format`` is ``arrow_stream``. The ``codec`` field must be set to ``arrow_stream``. + :type batch_encoding: ObservabilityPipelineClickhouseDestinationBatchEncoding, optional + + :param compression: Compression setting for outbound HTTP requests to ClickHouse. + Can be specified as a shorthand string ( ``"gzip"`` or ``"none"`` ) or as an object + with an ``algorithm`` field and an optional ``level`` (gzip only, 1–9). + :type compression: ObservabilityPipelineClickhouseDestinationCompression, optional + + :param database: Optional ClickHouse database name. If omitted, the user's default database on the ClickHouse server is used. + :type database: str, optional + + :param date_time_best_effort: When ``true`` , enables flexible DateTime parsing on the ClickHouse server side. + :type date_time_best_effort: bool, optional + + :param endpoint_url_key: Name of the environment variable or secret that contains the ClickHouse HTTP endpoint URL. + Defaults to ``DESTINATION_CLICKHOUSE_ENDPOINT_URL`` (prefixed with ``DD_OP_`` at runtime). + :type endpoint_url_key: str, optional + + :param format: Insert format for events sent to ClickHouse. + + * ``json_each_row`` : Maps event fields to columns by name (ClickHouse ``JSONEachRow`` ). + * ``json_as_object`` : Inserts each event into a single ``Object('json')`` / ``JSON`` column (ClickHouse ``JSONAsObject`` ). + * ``json_as_string`` : Inserts each event into a single ``String`` -typed column as raw JSON (ClickHouse ``JSONAsString`` ). + * ``arrow_stream`` : Batches events using Apache Arrow IPC streaming format. Requires ``batch_encoding``. + :type format: ObservabilityPipelineClickhouseDestinationFormat, optional + + :param id: The unique identifier for this component. + :type id: str + + :param inputs: A list of component IDs whose output is used as the ``input`` for this component. + :type inputs: [str] + + :param skip_unknown_fields: When ``true`` , fields not present in the target table schema are dropped instead of causing insert errors. + When unset, the ClickHouse server's own ``input_format_skip_unknown_fields`` setting applies. + :type skip_unknown_fields: bool, none_type, optional + + :param table: Target ClickHouse table name. Events are inserted into this table. + :type table: str + + :param tls: Configuration for enabling TLS encryption between the pipeline component and external services. + :type tls: ObservabilityPipelineTls, optional + + :param type: The destination type. The value must be ``clickhouse``. + :type type: ObservabilityPipelineClickhouseDestinationType + """ + if auth is not unset: + kwargs["auth"] = auth + if batch is not unset: + kwargs["batch"] = batch + if batch_encoding is not unset: + kwargs["batch_encoding"] = batch_encoding + if compression is not unset: + kwargs["compression"] = compression + if database is not unset: + kwargs["database"] = database + if date_time_best_effort is not unset: + kwargs["date_time_best_effort"] = date_time_best_effort + if endpoint_url_key is not unset: + kwargs["endpoint_url_key"] = endpoint_url_key + if format is not unset: + kwargs["format"] = format + if skip_unknown_fields is not unset: + kwargs["skip_unknown_fields"] = skip_unknown_fields + if tls is not unset: + kwargs["tls"] = tls + super().__init__(kwargs) + + self_.id = id + self_.inputs = inputs + self_.table = table + self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_auth.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_auth.py new file mode 100644 index 0000000000..1370465c59 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_auth.py @@ -0,0 +1,67 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth_strategy import ( + ObservabilityPipelineClickhouseDestinationAuthStrategy, + ) + + +class ObservabilityPipelineClickhouseDestinationAuth(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth_strategy import ( + ObservabilityPipelineClickhouseDestinationAuthStrategy, + ) + + return { + "password_key": (str,), + "strategy": (ObservabilityPipelineClickhouseDestinationAuthStrategy,), + "username_key": (str,), + } + + attribute_map = { + "password_key": "password_key", + "strategy": "strategy", + "username_key": "username_key", + } + + def __init__( + self_, + strategy: ObservabilityPipelineClickhouseDestinationAuthStrategy, + password_key: Union[str, UnsetType] = unset, + username_key: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + HTTP Basic Authentication credentials for the ClickHouse destination. + When ``strategy`` is ``basic`` , provide ``username_key`` and ``password_key`` that reference environment variables or secrets containing the credentials. + + :param password_key: Name of the environment variable or secret that contains the ClickHouse password. + :type password_key: str, optional + + :param strategy: The authentication strategy for ClickHouse HTTP requests. Only ``basic`` is supported. + :type strategy: ObservabilityPipelineClickhouseDestinationAuthStrategy + + :param username_key: Name of the environment variable or secret that contains the ClickHouse username. + :type username_key: str, optional + """ + if password_key is not unset: + kwargs["password_key"] = password_key + if username_key is not unset: + kwargs["username_key"] = username_key + super().__init__(kwargs) + + self_.strategy = strategy diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_auth_strategy.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_auth_strategy.py new file mode 100644 index 0000000000..1b9080dba2 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_auth_strategy.py @@ -0,0 +1,37 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class ObservabilityPipelineClickhouseDestinationAuthStrategy(ModelSimple): + """ + The authentication strategy for ClickHouse HTTP requests. Only `basic` is supported. + + :param value: If omitted defaults to "basic". Must be one of ["basic"]. + :type value: str + """ + + allowed_values = { + "basic", + } + BASIC: ClassVar["ObservabilityPipelineClickhouseDestinationAuthStrategy"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +ObservabilityPipelineClickhouseDestinationAuthStrategy.BASIC = ObservabilityPipelineClickhouseDestinationAuthStrategy( + "basic" +) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch.py new file mode 100644 index 0000000000..569cddfbba --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch.py @@ -0,0 +1,55 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class ObservabilityPipelineClickhouseDestinationBatch(ModelNormal): + validations = { + "max_events": { + "inclusive_minimum": 1, + }, + "timeout_secs": { + "inclusive_maximum": 65535, + "inclusive_minimum": 1, + }, + } + + @cached_property + def openapi_types(_): + return { + "max_events": (int,), + "timeout_secs": (int,), + } + + attribute_map = { + "max_events": "max_events", + "timeout_secs": "timeout_secs", + } + + def __init__( + self_, max_events: Union[int, UnsetType] = unset, timeout_secs: Union[int, UnsetType] = unset, **kwargs + ): + """ + Batching configuration for ClickHouse inserts. + + :param max_events: Maximum number of events per batch before it is flushed. + :type max_events: int, optional + + :param timeout_secs: Maximum number of seconds to wait before flushing a partial batch. + :type timeout_secs: int, optional + """ + if max_events is not unset: + kwargs["max_events"] = max_events + if timeout_secs is not unset: + kwargs["timeout_secs"] = timeout_secs + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch_encoding.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch_encoding.py new file mode 100644 index 0000000000..70e5b5311c --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch_encoding.py @@ -0,0 +1,60 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding_codec import ( + ObservabilityPipelineClickhouseDestinationBatchEncodingCodec, + ) + + +class ObservabilityPipelineClickhouseDestinationBatchEncoding(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding_codec import ( + ObservabilityPipelineClickhouseDestinationBatchEncodingCodec, + ) + + return { + "allow_nullable_fields": (bool,), + "codec": (ObservabilityPipelineClickhouseDestinationBatchEncodingCodec,), + } + + attribute_map = { + "allow_nullable_fields": "allow_nullable_fields", + "codec": "codec", + } + + def __init__( + self_, + codec: ObservabilityPipelineClickhouseDestinationBatchEncodingCodec, + allow_nullable_fields: Union[bool, UnsetType] = unset, + **kwargs, + ): + """ + Batch encoding configuration for the ClickHouse destination. + Required when ``format`` is ``arrow_stream``. The ``codec`` field must be set to ``arrow_stream``. + + :param allow_nullable_fields: When ``true`` , null values are allowed for non-nullable fields in the ClickHouse schema. + When ``false`` (default), missing values for non-nullable columns cause encoding errors. + :type allow_nullable_fields: bool, optional + + :param codec: The codec used for batch encoding. Only ``arrow_stream`` is supported. + :type codec: ObservabilityPipelineClickhouseDestinationBatchEncodingCodec + """ + if allow_nullable_fields is not unset: + kwargs["allow_nullable_fields"] = allow_nullable_fields + super().__init__(kwargs) + + self_.codec = codec diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch_encoding_codec.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch_encoding_codec.py new file mode 100644 index 0000000000..d254441f3f --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_batch_encoding_codec.py @@ -0,0 +1,37 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class ObservabilityPipelineClickhouseDestinationBatchEncodingCodec(ModelSimple): + """ + The codec used for batch encoding. Only `arrow_stream` is supported. + + :param value: If omitted defaults to "arrow_stream". Must be one of ["arrow_stream"]. + :type value: str + """ + + allowed_values = { + "arrow_stream", + } + ARROW_STREAM: ClassVar["ObservabilityPipelineClickhouseDestinationBatchEncodingCodec"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +ObservabilityPipelineClickhouseDestinationBatchEncodingCodec.ARROW_STREAM = ( + ObservabilityPipelineClickhouseDestinationBatchEncodingCodec("arrow_stream") +) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression.py new file mode 100644 index 0000000000..a39baea814 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression.py @@ -0,0 +1,46 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelComposed, + cached_property, +) + + +class ObservabilityPipelineClickhouseDestinationCompression(ModelComposed): + def __init__(self, **kwargs): + """ + Compression setting for outbound HTTP requests to ClickHouse. + Can be specified as a shorthand string ( ``"gzip"`` or ``"none"`` ) or as an object + with an ``algorithm`` field and an optional ``level`` (gzip only, 1–9). + + :param algorithm: The compression algorithm applied to outbound HTTP requests. + :type algorithm: ObservabilityPipelineClickhouseDestinationCompressionAlgorithm + + :param level: Compression level (1–9). Only applicable when `algorithm` is `gzip`. + :type level: int, optional + """ + super().__init__(kwargs) + + @cached_property + def _composed_schemas(_): + # we need this here to make our import statements work + # we must store _composed_schemas in here so the code is only run + # when we invoke this method. If we kept this at the class + # level we would get an error because the class level + # code would be run when this module is imported, and these composed + # classes don't exist yet because their module has not finished + # loading + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_object import ( + ObservabilityPipelineClickhouseDestinationCompressionObject, + ) + + return { + "oneOf": [ + str, + ObservabilityPipelineClickhouseDestinationCompressionObject, + ], + } diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression_algorithm.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression_algorithm.py new file mode 100644 index 0000000000..4f9928e853 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression_algorithm.py @@ -0,0 +1,42 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class ObservabilityPipelineClickhouseDestinationCompressionAlgorithm(ModelSimple): + """ + The compression algorithm applied to outbound HTTP requests. + + :param value: Must be one of ["gzip", "none"]. + :type value: str + """ + + allowed_values = { + "gzip", + "none", + } + GZIP: ClassVar["ObservabilityPipelineClickhouseDestinationCompressionAlgorithm"] + NONE: ClassVar["ObservabilityPipelineClickhouseDestinationCompressionAlgorithm"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +ObservabilityPipelineClickhouseDestinationCompressionAlgorithm.GZIP = ( + ObservabilityPipelineClickhouseDestinationCompressionAlgorithm("gzip") +) +ObservabilityPipelineClickhouseDestinationCompressionAlgorithm.NONE = ( + ObservabilityPipelineClickhouseDestinationCompressionAlgorithm("none") +) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression_object.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression_object.py new file mode 100644 index 0000000000..36b699ea2e --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_compression_object.py @@ -0,0 +1,66 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_algorithm import ( + ObservabilityPipelineClickhouseDestinationCompressionAlgorithm, + ) + + +class ObservabilityPipelineClickhouseDestinationCompressionObject(ModelNormal): + validations = { + "level": { + "inclusive_maximum": 9, + "inclusive_minimum": 1, + }, + } + + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_algorithm import ( + ObservabilityPipelineClickhouseDestinationCompressionAlgorithm, + ) + + return { + "algorithm": (ObservabilityPipelineClickhouseDestinationCompressionAlgorithm,), + "level": (int,), + } + + attribute_map = { + "algorithm": "algorithm", + "level": "level", + } + + def __init__( + self_, + algorithm: ObservabilityPipelineClickhouseDestinationCompressionAlgorithm, + level: Union[int, UnsetType] = unset, + **kwargs, + ): + """ + Structured compression configuration for the ClickHouse destination. + Use ``algorithm`` to specify the compression type and ``level`` (optional, gzip only) to control compression strength. + + :param algorithm: The compression algorithm applied to outbound HTTP requests. + :type algorithm: ObservabilityPipelineClickhouseDestinationCompressionAlgorithm + + :param level: Compression level (1–9). Only applicable when ``algorithm`` is ``gzip``. + :type level: int, optional + """ + if level is not unset: + kwargs["level"] = level + super().__init__(kwargs) + + self_.algorithm = algorithm diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_format.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_format.py new file mode 100644 index 0000000000..6206685a30 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_format.py @@ -0,0 +1,56 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class ObservabilityPipelineClickhouseDestinationFormat(ModelSimple): + """ + Insert format for events sent to ClickHouse. + - `json_each_row`: Maps event fields to columns by name (ClickHouse `JSONEachRow`). + - `json_as_object`: Inserts each event into a single `Object('json')` / `JSON` column (ClickHouse `JSONAsObject`). + - `json_as_string`: Inserts each event into a single `String`-typed column as raw JSON (ClickHouse `JSONAsString`). + - `arrow_stream`: Batches events using Apache Arrow IPC streaming format. Requires `batch_encoding`. + + :param value: Must be one of ["json_each_row", "json_as_object", "json_as_string", "arrow_stream"]. + :type value: str + """ + + allowed_values = { + "json_each_row", + "json_as_object", + "json_as_string", + "arrow_stream", + } + JSON_EACH_ROW: ClassVar["ObservabilityPipelineClickhouseDestinationFormat"] + JSON_AS_OBJECT: ClassVar["ObservabilityPipelineClickhouseDestinationFormat"] + JSON_AS_STRING: ClassVar["ObservabilityPipelineClickhouseDestinationFormat"] + ARROW_STREAM: ClassVar["ObservabilityPipelineClickhouseDestinationFormat"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +ObservabilityPipelineClickhouseDestinationFormat.JSON_EACH_ROW = ObservabilityPipelineClickhouseDestinationFormat( + "json_each_row" +) +ObservabilityPipelineClickhouseDestinationFormat.JSON_AS_OBJECT = ObservabilityPipelineClickhouseDestinationFormat( + "json_as_object" +) +ObservabilityPipelineClickhouseDestinationFormat.JSON_AS_STRING = ObservabilityPipelineClickhouseDestinationFormat( + "json_as_string" +) +ObservabilityPipelineClickhouseDestinationFormat.ARROW_STREAM = ObservabilityPipelineClickhouseDestinationFormat( + "arrow_stream" +) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_type.py b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_type.py new file mode 100644 index 0000000000..6124b1f94b --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_clickhouse_destination_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class ObservabilityPipelineClickhouseDestinationType(ModelSimple): + """ + The destination type. The value must be `clickhouse`. + + :param value: If omitted defaults to "clickhouse". Must be one of ["clickhouse"]. + :type value: str + """ + + allowed_values = { + "clickhouse", + } + CLICKHOUSE: ClassVar["ObservabilityPipelineClickhouseDestinationType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +ObservabilityPipelineClickhouseDestinationType.CLICKHOUSE = ObservabilityPipelineClickhouseDestinationType("clickhouse") diff --git a/src/datadog_api_client/v2/model/observability_pipeline_config.py b/src/datadog_api_client/v2/model/observability_pipeline_config.py index afd1a78d88..b507450be7 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_config.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_config.py @@ -45,6 +45,9 @@ ObservabilityPipelineAmazonSecurityLakeDestination, ) from datadog_api_client.v2.model.azure_storage_destination import AzureStorageDestination + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination import ( + ObservabilityPipelineClickhouseDestination, + ) from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination import ( ObservabilityPipelineCloudPremDestination, ) @@ -185,6 +188,7 @@ def __init__( ObservabilityPipelineAmazonS3GenericDestination, ObservabilityPipelineAmazonSecurityLakeDestination, AzureStorageDestination, + ObservabilityPipelineClickhouseDestination, ObservabilityPipelineCloudPremDestination, ObservabilityPipelineCrowdStrikeNextGenSiemDestination, ObservabilityPipelineDatadogLogsDestination, diff --git a/src/datadog_api_client/v2/model/observability_pipeline_config_destination_item.py b/src/datadog_api_client/v2/model/observability_pipeline_config_destination_item.py index be0e39e2c7..34adc41705 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_config_destination_item.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_config_destination_item.py @@ -106,6 +106,33 @@ def __init__(self, **kwargs): :param container_name: The name of the Azure Blob Storage container to store logs in. :type container_name: str + :param batch: Batching configuration for ClickHouse inserts. + :type batch: ObservabilityPipelineClickhouseDestinationBatch, optional + + :param batch_encoding: Batch encoding configuration for the ClickHouse destination. + Required when `format` is `arrow_stream`. The `codec` field must be set to `arrow_stream`. + :type batch_encoding: ObservabilityPipelineClickhouseDestinationBatchEncoding, optional + + :param database: Optional ClickHouse database name. If omitted, the user's default database on the ClickHouse server is used. + :type database: str, optional + + :param date_time_best_effort: When `true`, enables flexible DateTime parsing on the ClickHouse server side. + :type date_time_best_effort: bool, optional + + :param format: Insert format for events sent to ClickHouse. + - `json_each_row`: Maps event fields to columns by name (ClickHouse `JSONEachRow`). + - `json_as_object`: Inserts each event into a single `Object('json')` / `JSON` column (ClickHouse `JSONAsObject`). + - `json_as_string`: Inserts each event into a single `String`-typed column as raw JSON (ClickHouse `JSONAsString`). + - `arrow_stream`: Batches events using Apache Arrow IPC streaming format. Requires `batch_encoding`. + :type format: ObservabilityPipelineClickhouseDestinationFormat, optional + + :param skip_unknown_fields: When `true`, fields not present in the target table schema are dropped instead of causing insert errors. + When unset, the ClickHouse server's own `input_format_skip_unknown_fields` setting applies. + :type skip_unknown_fields: bool, none_type, optional + + :param table: Target ClickHouse table name. Events are inserted into this table. + :type table: str + :param routes: A list of routing rules that forward matching logs to Datadog using dedicated API keys. :type routes: [ObservabilityPipelineDatadogLogsDestinationRoute], optional @@ -166,9 +193,6 @@ def __init__(self, **kwargs): :param dcr_immutable_id: The immutable ID of the Data Collection Rule (DCR). :type dcr_immutable_id: str - :param table: The name of the Log Analytics table where logs are sent. - :type table: str - :param tenant_id: Azure AD tenant ID. :type tenant_id: str @@ -263,6 +287,9 @@ def _composed_schemas(_): ObservabilityPipelineAmazonSecurityLakeDestination, ) from datadog_api_client.v2.model.azure_storage_destination import AzureStorageDestination + from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination import ( + ObservabilityPipelineClickhouseDestination, + ) from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination import ( ObservabilityPipelineCloudPremDestination, ) @@ -328,6 +355,7 @@ def _composed_schemas(_): ObservabilityPipelineAmazonS3GenericDestination, ObservabilityPipelineAmazonSecurityLakeDestination, AzureStorageDestination, + ObservabilityPipelineClickhouseDestination, ObservabilityPipelineCloudPremDestination, ObservabilityPipelineCrowdStrikeNextGenSiemDestination, ObservabilityPipelineDatadogLogsDestination, diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index 9eabca83ad..b464e36452 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -5042,6 +5042,39 @@ from datadog_api_client.v2.model.observability_pipeline_buffer_options_when_full import ( ObservabilityPipelineBufferOptionsWhenFull, ) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination import ( + ObservabilityPipelineClickhouseDestination, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth import ( + ObservabilityPipelineClickhouseDestinationAuth, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_auth_strategy import ( + ObservabilityPipelineClickhouseDestinationAuthStrategy, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch import ( + ObservabilityPipelineClickhouseDestinationBatch, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding import ( + ObservabilityPipelineClickhouseDestinationBatchEncoding, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_batch_encoding_codec import ( + ObservabilityPipelineClickhouseDestinationBatchEncodingCodec, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression import ( + ObservabilityPipelineClickhouseDestinationCompression, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_algorithm import ( + ObservabilityPipelineClickhouseDestinationCompressionAlgorithm, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_compression_object import ( + ObservabilityPipelineClickhouseDestinationCompressionObject, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_format import ( + ObservabilityPipelineClickhouseDestinationFormat, +) +from datadog_api_client.v2.model.observability_pipeline_clickhouse_destination_type import ( + ObservabilityPipelineClickhouseDestinationType, +) from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination import ( ObservabilityPipelineCloudPremDestination, ) @@ -13195,6 +13228,17 @@ "ObservabilityPipelineBufferOptionsDiskType", "ObservabilityPipelineBufferOptionsMemoryType", "ObservabilityPipelineBufferOptionsWhenFull", + "ObservabilityPipelineClickhouseDestination", + "ObservabilityPipelineClickhouseDestinationAuth", + "ObservabilityPipelineClickhouseDestinationAuthStrategy", + "ObservabilityPipelineClickhouseDestinationBatch", + "ObservabilityPipelineClickhouseDestinationBatchEncoding", + "ObservabilityPipelineClickhouseDestinationBatchEncodingCodec", + "ObservabilityPipelineClickhouseDestinationCompression", + "ObservabilityPipelineClickhouseDestinationCompressionAlgorithm", + "ObservabilityPipelineClickhouseDestinationCompressionObject", + "ObservabilityPipelineClickhouseDestinationFormat", + "ObservabilityPipelineClickhouseDestinationType", "ObservabilityPipelineCloudPremDestination", "ObservabilityPipelineCloudPremDestinationType", "ObservabilityPipelineConfig", diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_arrow_stream_format_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_arrow_stream_format_returns_ok_response.frozen new file mode 100644 index 0000000000..663907587b --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_arrow_stream_format_returns_ok_response.frozen @@ -0,0 +1 @@ +2026-06-22T20:44:30.778Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_arrow_stream_format_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_arrow_stream_format_returns_ok_response.yaml new file mode 100644 index 0000000000..a0d62d2cf7 --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_arrow_stream_format_returns_ok_response.yaml @@ -0,0 +1,23 @@ +interactions: +- request: + body: '{"data":{"attributes":{"config":{"destinations":[{"auth":{"password_key":"CLICKHOUSE_PASSWORD","strategy":"basic","username_key":"CLICKHOUSE_USERNAME"},"batch":{"max_events":1000,"timeout_secs":1},"batch_encoding":{"allow_nullable_fields":false,"codec":"arrow_stream"},"compression":"gzip","database":"my_database","format":"arrow_stream","id":"clickhouse-destination","inputs":["my-processor-group"],"table":"application_logs","type":"clickhouse"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Pipeline + with ClickHouse Destination Arrow Stream"},"type":"pipelines"}}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate + response: + body: + string: '{"errors":[]} + + ' + headers: + content-type: + - application/vnd.api+json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_returns_ok_response.frozen new file mode 100644 index 0000000000..9b0a433186 --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_returns_ok_response.frozen @@ -0,0 +1 @@ +2026-06-22T20:44:32.514Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_returns_ok_response.yaml new file mode 100644 index 0000000000..518c42c0f5 --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_with_clickhouse_destination_returns_ok_response.yaml @@ -0,0 +1,23 @@ +interactions: +- request: + body: '{"data":{"attributes":{"config":{"destinations":[{"auth":{"password_key":"CLICKHOUSE_PASSWORD","strategy":"basic","username_key":"CLICKHOUSE_USERNAME"},"batch":{"max_events":1000,"timeout_secs":1},"compression":"gzip","database":"my_database","id":"clickhouse-destination","inputs":["my-processor-group"],"table":"application_logs","type":"clickhouse"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Pipeline + with ClickHouse Destination"},"type":"pipelines"}}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate + response: + body: + string: '{"errors":[]} + + ' + headers: + content-type: + - application/vnd.api+json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/v2/features/observability_pipelines.feature b/tests/v2/features/observability_pipelines.feature index 487d6971f8..a57aed2a03 100644 --- a/tests/v2/features/observability_pipelines.feature +++ b/tests/v2/features/observability_pipelines.feature @@ -176,6 +176,22 @@ Feature: Observability Pipelines Then the response status is 200 OK And the response "errors" has length 0 + @team:DataDog/observability-pipelines + Scenario: Validate an observability pipeline with ClickHouse destination arrow_stream format returns "OK" response + Given new "ValidatePipeline" request + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "clickhouse-destination", "inputs": ["my-processor-group"], "type": "clickhouse", "table": "application_logs", "database": "my_database", "format": "arrow_stream", "batch_encoding": {"codec": "arrow_stream", "allow_nullable_fields": false}, "compression": "gzip", "auth": {"strategy": "basic", "username_key": "CLICKHOUSE_USERNAME", "password_key": "CLICKHOUSE_PASSWORD"}, "batch": {"max_events": 1000, "timeout_secs": 1}}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with ClickHouse Destination Arrow Stream"}, "type": "pipelines"}} + When the request is sent + Then the response status is 200 OK + And the response "errors" has length 0 + + @team:DataDog/observability-pipelines + Scenario: Validate an observability pipeline with ClickHouse destination returns "OK" response + Given new "ValidatePipeline" request + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "clickhouse-destination", "inputs": ["my-processor-group"], "type": "clickhouse", "table": "application_logs", "database": "my_database", "compression": "gzip", "auth": {"strategy": "basic", "username_key": "CLICKHOUSE_USERNAME", "password_key": "CLICKHOUSE_PASSWORD"}, "batch": {"max_events": 1000, "timeout_secs": 1}}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with ClickHouse Destination"}, "type": "pipelines"}} + When the request is sent + Then the response status is 200 OK + And the response "errors" has length 0 + @team:DataDog/observability-pipelines Scenario: Validate an observability pipeline with HTTP server source valid_tokens returns "OK" response Given new "ValidatePipeline" request