diff --git a/Makefile b/Makefile
index 7928342..c791fd6 100644
--- a/Makefile
+++ b/Makefile
@@ -27,20 +27,31 @@ build_and_deploy: builddocker deploy
# https://docs.docker.com/engine/reference/commandline/docker/
# https://docs.docker.com/compose/reference/
deploy:
+ @echo "================================"
+ @echo "Make deploy"
echo XTCPPATH=${XTCPPATH}
XTCPPATH=${XTCPPATH} \
docker compose \
--file build/containers/redpanda/docker-compose.yml \
up -d --remove-orphans
+down:
+ @echo "================================"
+ @echo "Make down"
+ XTCPPATH=${XTCPPATH} \
+ docker compose \
+ --file build/containers/redpanda/docker-compose.yml \
+ down
+
#--env-file docker-compose-enviroment-variables \
builddocker: builddocker_xtcp builddocker_clickhouse
builddocker_xtcp:
+ @echo "================================"
+ @echo "Make builddocker_xtcp"
docker build \
--build-arg XTCPPATH=${XTCPPATH} \
- --build-arg PWD=${PWD} \
--build-arg COMMIT=${COMMIT} \
--build-arg DATE=${DATE} \
--build-arg VERSION=${VERSION} \
@@ -49,9 +60,10 @@ builddocker_xtcp:
${XTCPPATH}
builddocker_clickhouse:
+ @echo "================================"
+ @echo "Make builddocker_clickhouse"
docker build \
--build-arg XTCPPATH=${XTCPPATH} \
- --build-arg PWD=${PWD} \
--build-arg VERSION=${VERSION} \
--file build/containers/clickhouse/Containerfile \
--tag xtcp_clickhouse:${VERSION} --tag xtcp_clickhouse:latest \
@@ -83,4 +95,21 @@ test:
bench:
go test -bench=. ./pkg/xtcpnl/
+followxtcp:
+ docker logs xtcp-xtcp2-1 --follow
+
+ch:
+ docker exec -it xtcp-clickhouse-1 bash
+
+ch_prom:
+ curl --silent http://localhost:9363/metrics | grep -v "#" | grep -i kafka
+
+clear_docker_volumes:
+ docker volume rm redpanda-quickstart-one-broker_redpanda-0 || true
+ docker volume rm redpanda_redpanda-0 || true
+ docker volume rm xtcp_nsq_data || true
+ docker volume rm xtcp_redpanda-0 || true
+
+ docker volume ls
+
# end
\ No newline at end of file
diff --git a/README.md b/README.md
index 31f8503..4cc1650 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,7 @@
-# xtcp2
\ No newline at end of file
+# xtcp2
+
+Please run:
+- "make" to build the docker containers and then start the set of containers
+- "deploy" to start the containers if you've already built them
+
+See also: "cat Makefile"
\ No newline at end of file
diff --git a/build/containers/clickhouse/Containerfile b/build/containers/clickhouse/Containerfile
index 9c9fa89..01b32d4 100644
--- a/build/containers/clickhouse/Containerfile
+++ b/build/containers/clickhouse/Containerfile
@@ -8,14 +8,12 @@
# https://github.com/ClickHouse/ClickHouse/blob/master/docker/server/entrypoint.sh
ARG XTCPPATH
-ARG PWD
ARG VERSION
# https://hub.docker.com/r/clickhouse/clickhouse-server/tags
FROM clickhouse/clickhouse-server:24.8.4.13-alpine
ARG XTCPPATH
-ARG PWD
ARG VERSION
RUN echo XTCPPATH:${XTCPPATH} VERSION:${VERSION}
@@ -28,15 +26,30 @@ RUN echo XTCPPATH:${XTCPPATH} VERSION:${VERSION}
RUN apk add busybox-extras
COPY --chmod=544 \
- ${XTCPPATH}/build/containers/clickhouse/docker_related_config.xml \
+ ./build/containers/clickhouse/config.xml \
+ /etc/clickhouse-server/config.xml
+
+COPY --chmod=544 \
+ ./build/containers/clickhouse/docker_related_config.xml \
/etc/clickhouse-server/config.d/docker_related_config.xml
COPY --chmod=544 \
- ${XTCPPATH}/pkg/xtcppb/flatxtcppb.proto \
+ ./pkg/xtcppb/flatxtcppb.proto \
/var/lib/clickhouse/format_schemas/flatxtcppb.proto
COPY --chmod=544 \
- ${XTCPPATH}/VERSION \
+ ./VERSION \
/VERSION
+# http
+EXPOSE 8123
+# native
+EXPOSE 9000
+# mysql
+EXPOSE 9004
+# postgres
+EXPOSE 9005
+# prometheus
+EXPOSE 9363
+
# end
\ No newline at end of file
diff --git a/build/containers/clickhouse/config.xml b/build/containers/clickhouse/config.xml
new file mode 100644
index 0000000..79b56cd
--- /dev/null
+++ b/build/containers/clickhouse/config.xml
@@ -0,0 +1,1635 @@
+
+
+
+
+ warning
+ /var/log/clickhouse-server/clickhouse-server.log
+ /var/log/clickhouse-server/clickhouse-server.err.log
+
+ 1000M
+ 10
+
+
+
+
+
+
+
+
+
+
+
+
+
+ https://{bucket}.s3.amazonaws.com
+
+
+ https://{bucket}.storage.googleapis.com
+
+
+ https://{bucket}.oss.aliyuncs.com
+
+
+
+
+
+
+ Access-Control-Allow-Origin
+ *
+
+
+ Access-Control-Allow-Headers
+ origin, x-requested-with, x-clickhouse-format, x-clickhouse-user, x-clickhouse-key, Authorization
+
+
+ Access-Control-Allow-Methods
+ POST, GET, OPTIONS
+
+
+ Access-Control-Max-Age
+ 86400
+
+
+
+
+
+
+
+ 8123
+
+
+ 9000
+
+
+ 9004
+
+
+ 9005
+
+
+
+
+
+
+
+
+
+
+
+ 9009
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 4096
+
+
+
+ 30
+
+
+
+
+ false
+
+
+ /path/to/ssl_cert_file
+ /path/to/ssl_key_file
+
+
+ false
+
+
+ /path/to/ssl_ca_cert_file
+
+
+
+ stream_gzip,none
+
+
+
+ 1
+
+
+ -1
+ -1
+
+
+ false
+
+
+
+
+
+
+
+
+
+ none
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ RejectCertificateHandler
+
+
+
+
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ RejectCertificateHandler
+
+
+
+
+
+
+
+
+ 0
+ 2
+
+
+
+ 100
+
+
+ 0
+
+
+
+ 10000
+
+
+
+
+
+
+
+
+ 0.9
+
+
+ 4194304
+
+
+ 0
+
+
+
+
+
+ 8589934592
+
+
+ 5368709120
+
+
+ 5368709120
+
+
+ 1000
+
+
+ 134217728
+
+
+ 10000
+
+
+ /var/lib/clickhouse/caches/
+
+ false
+
+
+ /var/lib/clickhouse/
+
+
+
+
+
+
+ /var/lib/clickhouse/tmp/
+
+
+ 1
+ 1
+ 1
+
+
+ sha256_password
+
+
+ 12
+
+
+
+
+
+
+
+
+ /var/lib/clickhouse/user_files/
+
+
+
+
+
+
+
+
+
+
+
+
+ users.xml
+
+
+
+ /var/lib/clickhouse/access/
+
+
+
+
+
+
+
+ true
+
+
+ true
+
+
+ true
+
+
+ true
+
+
+ true
+
+
+ 600
+
+
+
+ default
+
+
+ SQL_
+
+
+
+
+
+
+
+
+ default
+
+
+
+
+
+
+
+
+ true
+
+
+ false
+
+ ' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ clickhouse-jdbc-bridge &
+
+ * [CentOS/RHEL]
+ export MVN_URL=https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc-bridge/
+ export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ clickhouse-jdbc-bridge &
+
+ Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information.
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 3600
+
+
+
+ 3600
+
+
+ 60
+
+
+
+
+
+
+
+
+
+ /metrics
+ 9363
+ true
+ true
+ true
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+
+
+
+
+
+
+
+ 7500
+
+ 1048576
+
+ 8192
+
+ 524288
+
+ false
+
+
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+ 1048576
+ 8192
+ 524288
+
+ false
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ 1000
+ false
+
+
+
+
+ system
+
+ 7000
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+
+
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+
+ system
+
+
+
+ 1000
+ 1024
+ 1024
+ 512
+ true
+
+
+
+
+
+
+
+
+
+
+ system
+
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+ event_date
+ event_date + INTERVAL 3 DAY
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+ event_date + INTERVAL 30 DAY
+
+
+
+
+
+
+
+
+
+ *_dictionary.*ml
+
+
+ true
+
+
+ true
+
+
+ *_function.*ml
+
+
+
+
+
+
+
+
+
+ 10000000000
+ 0.01
+ lz4
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /clickhouse/task_queue/ddl
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /var/lib/clickhouse/format_schemas/
+
+
+ /usr/share/clickhouse/protos/
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1073741824
+ 1024
+ 1048576
+ 30000000
+
+
+
+ backups
+
+
+ true
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/build/containers/clickhouse/docker_related_config.xml b/build/containers/clickhouse/docker_related_config.xml
index 5e60af7..b4bf944 100644
--- a/build/containers/clickhouse/docker_related_config.xml
+++ b/build/containers/clickhouse/docker_related_config.xml
@@ -13,6 +13,8 @@
all
+
+ zstd
\ No newline at end of file
diff --git a/build/containers/clickhouse/initdb.d/.gitignore b/build/containers/clickhouse/initdb.d/.gitignore
new file mode 100644
index 0000000..e69de29
diff --git a/build/containers/clickhouse/initdb.d/date b/build/containers/clickhouse/initdb.d/date
index f5c0591..7ff77b4 100644
--- a/build/containers/clickhouse/initdb.d/date
+++ b/build/containers/clickhouse/initdb.d/date
@@ -1 +1 @@
-date2024_09_11_00_16_01
+date2024_09_18_23_49_35
diff --git a/build/containers/clickhouse/initdb.d/date_utc b/build/containers/clickhouse/initdb.d/date_utc
new file mode 100644
index 0000000..7ff77b4
--- /dev/null
+++ b/build/containers/clickhouse/initdb.d/date_utc
@@ -0,0 +1 @@
+date2024_09_18_23_49_35
diff --git a/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.bash b/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.bash
deleted file mode 100755
index 1cbc963..0000000
--- a/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.bash
+++ /dev/null
@@ -1,407 +0,0 @@
-#!/bin/bash
-
-# apparently /usr/bin/bash doesn't exist in the container
-
-set -e
-
-d=$(date +date%Y_%m_%d_%H_%M_%S)
-w=$(whoami)
-
-#usermod -d /home/das das
-
-# https://clickhouse.com/docs/en/interfaces/formats#protobuf
-# https://clickhouse.com/docs/en/interfaces/formats#protobufsingle
-cp /docker-entrypoint-initdb.d/flatxtcppb.proto /var/lib/clickhouse/format_schemas/flatxtcppb.proto
-
-# https://protobuf.dev/programming-guides/encoding/#structure
-
-echo "${d}" > /docker-entrypoint-initdb.d/date
-echo "${w}" > /docker-entrypoint-initdb.d/whoami
-
-clickhouse client -n <<-EOSQL
- SELECT now();
-
- --------------------------------------------------------------------------------------------------
-
- -- Reload protobufs
- -- https://clickhouse.com/docs/en/interfaces/formats#drop-protobuf-cache
- SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf;
-
- --------------------------------------------------------------------------------------------------
-
- DROP DATABASE IF EXISTS xtcp;
- CREATE DATABASE IF NOT EXISTS xtcp;
-
- --------------------------------------------------------------------------------------------------
-
- DROP TABLE IF EXISTS xtcp.xtcp_records;
- CREATE TABLE IF NOT EXISTS xtcp.xtcp_records
- (
- -- https://clickhouse.com/docs/en/sql-reference/data-types/datetime64
- sec DateTime64(3,'UTC'),
- nsec Int64,
-
- -- https://clickhouse.com/docs/en/sql-reference/data-types/lowcardinality
- hostname LowCardinality(String),
- tag LowCardinality(String),
- record_count UInt64,
- netlinker_id UInt32,
-
- inet_diag_msg_family UInt32,
- inet_diag_msg_state UInt32,
- -- inet_diag_msg_family LowCardinality(UInt32),
- -- inet_diag_msg_state LowCardinality(UInt32),
- inet_diag_msg_timer UInt32,
- inet_diag_msg_retrans UInt32,
-
- inet_diag_msg_socket_source_port UInt32,
- inet_diag_msg_socket_destination_port UInt32,
- inet_diag_msg_socket_source String,
- inet_diag_msg_socket_source_ipv4 Nullable(IPv4),
- inet_diag_msg_socket_source_ipv6 Nullable(IPv6),
- inet_diag_msg_socket_destination String,
- inet_diag_msg_socket_destination_ipv4 Nullable(IPv4),
- inet_diag_msg_socket_destination_ipv6 Nullable(IPv6),
- inet_diag_msg_socket_interface UInt32,
- inet_diag_msg_socket_cookie UInt64,
- inet_diag_msg_socket_dest_asn UInt64,
- inet_diag_msg_socket_next_hop_asn UInt64,
- inet_diag_msg_socket_source_asn UInt64,
-
- inet_diag_msg_expires UInt32,
- inet_diag_msg_rqueue UInt32,
- inet_diag_msg_wqueue UInt32,
- inet_diag_msg_uid UInt32,
- inet_diag_msg_inode UInt32,
-
- mem_info_rmem UInt32,
- mem_info_wmem UInt32,
- mem_info_fmem UInt32,
- mem_info_tmem UInt32,
-
- tcp_info_state UInt32,
- tcp_info_ca_state UInt32,
- -- tcp_info_state LowCardinality(UInt32),
- -- tcp_info_ca_state LowCardinality(UInt32),
- tcp_info_retransmits UInt32,
- tcp_info_probes UInt32,
- tcp_info_backoff UInt32,
- tcp_info_options UInt32,
- tcp_info_send_scale UInt32,
- tcp_info_rcv_scale UInt32,
- tcp_info_delivery_rate_app_limited UInt32,
- tcp_info_fast_open_client_failed UInt32,
- tcp_info_rto UInt32,
- tcp_info_ato UInt32,
- tcp_info_snd_mss UInt32,
- tcp_info_rcv_mss UInt32,
- tcp_info_unacked UInt32,
- tcp_info_sacked UInt32,
- tcp_info_lost UInt32,
- tcp_info_retrans UInt32,
- tcp_info_fackets UInt32,
- tcp_info_last_data_sent UInt32,
- tcp_info_last_ack_sent UInt32,
- tcp_info_last_data_recv UInt32,
- tcp_info_last_ack_recv UInt32,
- tcp_info_pmtu UInt32,
- -- tcp_info_pmtu LowCardinality(UInt32),
- tcp_info_rcv_ssthresh UInt32,
- tcp_info_rtt UInt32,
- tcp_info_rtt_var UInt32,
- tcp_info_snd_ssthresh UInt32,
- tcp_info_snd_cwnd UInt32,
- tcp_info_adv_mss UInt32,
- tcp_info_reordering UInt32,
- tcp_info_rcv_rtt UInt32,
- tcp_info_rcv_space UInt32,
- tcp_info_total_retrans UInt32,
- tcp_info_pacing_rate UInt64,
- tcp_info_max_pacing_rate UInt64,
- tcp_info_bytes_acked UInt64,
- tcp_info_bytes_received UInt64,
- tcp_info_segs_out UInt32,
- tcp_info_segs_in UInt32,
- tcp_info_not_sent_bytes UInt32,
- tcp_info_min_rtt UInt32,
- tcp_info_data_segs_in UInt32,
- tcp_info_data_segs_out UInt32,
- tcp_info_delivery_rate UInt64,
- tcp_info_busy_time UInt64,
- tcp_info_rwnd_limited UInt64,
- tcp_info_sndbuf_limited UInt64,
- tcp_info_delivered UInt32,
- tcp_info_delivered_ce UInt32,
- tcp_info_bytes_sent UInt64,
- tcp_info_bytes_retrans UInt64,
- tcp_info_dsack_dups UInt32,
- tcp_info_reord_seen UInt32,
- tcp_info_rcv_ooopack UInt32,
- tcp_info_snd_wnd UInt32,
-
- congestion_algorithm_string LowCardinality(String),
- congestion_algorithm_enum LowCardinality(String),
-
- type_of_service UInt32,
- traffic_class UInt32,
- -- type_of_service LowCardinality(UInt32),
- -- traffic_class LowCardinality(UInt32),
-
- sk_mem_info_rmem_alloc UInt32,
- sk_mem_info_rcv_buf UInt32,
- sk_mem_info_wmem_alloc UInt32,
- sk_mem_info_snd_buf UInt32,
- sk_mem_info_fwd_alloc UInt32,
- sk_mem_info_wmem_queued UInt32,
- sk_mem_info_optmem UInt32,
- sk_mem_info_backlog UInt32,
- sk_mem_info_drops UInt32,
-
- shutdown_state UInt32,
- -- shutdown_state LowCardinality(UInt32),
-
- vegas_info_enabled UInt32,
- -- vegas_info_enabled LowCardinality(UInt32),
- vegas_info_rtt_cnt UInt32,
- vegas_info_rtt UInt32,
- vegas_info_min_rtt UInt32,
-
- dctcp_info_enabled UInt32,
- -- dctcp_info_enabled LowCardinality(UInt32),
- dctcp_info_ce_state UInt32,
- dctcp_info_alpha UInt32,
- dctcp_info_ab_ecn UInt32,
- dctcp_info_ab_tot UInt32,
-
- bbr_info_bw_lo UInt32,
- bbr_info_bw_hi UInt32,
- bbr_info_min_rtt UInt32,
- bbr_info_pacing_gain UInt32,
- bbr_info_cwnd_gain UInt32,
-
- class_id UInt32, -- LowCardinality?
- sock_opt UInt32, -- LowCardinality?
- c_group UInt64,
-
- )
- ENGINE = MergeTree
- -- ENGINE = ReplicatedMergeTree
- --PARTITION BY toYYYYMMDD(sec)
- ORDER BY (sec, nsec, hostname)
- TTL toDateTime(sec) + INTERVAL 2 MONTH DELETE;
-
- --------------------------------------------------------------------------------------------------
-
- -- Not using Nested!
- -- https://clickhouse.com/docs/en/sql-reference/data-types/nested-data-structures/nested
- -- Not using Nullable, because it uses more space and apparently "almost always negatively affects performance"
- -- https://clickhouse.com/docs/en/sql-reference/data-types/nullable
-
- CREATE TABLE IF NOT EXISTS xtcp.flat_xtcp_records_kafka
- (
- -- https://clickhouse.com/docs/en/sql-reference/data-types/datetime64
- sec DateTime64(3,'UTC'),
- nsec Int64,
-
- -- https://clickhouse.com/docs/en/sql-reference/data-types/lowcardinality
- hostname LowCardinality(String),
- tag LowCardinality(String),
-
- inet_diag_msg_family UInt32,
- inet_diag_msg_state UInt32,
- -- inet_diag_msg_family LowCardinality(UInt32),
- -- inet_diag_msg_state LowCardinality(UInt32),
- inet_diag_msg_timer UInt32,
- inet_diag_msg_retrans UInt32,
-
- inet_diag_msg_socket_source_port UInt32,
- inet_diag_msg_socket_destination_port UInt32,
- inet_diag_msg_socket_source String,
- inet_diag_msg_socket_source_ipv4 Nullable(IPv4),
- inet_diag_msg_socket_source_ipv6 Nullable(IPv6),
- inet_diag_msg_socket_destination String,
- inet_diag_msg_socket_destination_ipv4 Nullable(IPv4),
- inet_diag_msg_socket_destination_ipv6 Nullable(IPv6),
- inet_diag_msg_socket_interface UInt32,
- inet_diag_msg_socket_cookie UInt64,
- inet_diag_msg_socket_dest_asn UInt64,
- inet_diag_msg_socket_next_hop_asn UInt64,
- inet_diag_msg_socket_source_asn UInt64,
-
- inet_diag_msg_expires UInt32,
- inet_diag_msg_rqueue UInt32,
- inet_diag_msg_wqueue UInt32,
- inet_diag_msg_uid UInt32,
- inet_diag_msg_inode UInt32,
-
- mem_info_rmem UInt32,
- mem_info_wmem UInt32,
- mem_info_fmem UInt32,
- mem_info_tmem UInt32,
-
- tcp_info_state UInt32,
- tcp_info_ca_state UInt32,
- -- tcp_info_state LowCardinality(UInt32),
- -- tcp_info_ca_state LowCardinality(UInt32),
- tcp_info_retransmits UInt32,
- tcp_info_probes UInt32,
- tcp_info_backoff UInt32,
- tcp_info_options UInt32,
- tcp_info_send_scale UInt32,
- tcp_info_rcv_scale UInt32,
- tcp_info_delivery_rate_app_limited UInt32,
- tcp_info_fast_open_client_failed UInt32,
- tcp_info_rto UInt32,
- tcp_info_ato UInt32,
- tcp_info_snd_mss UInt32,
- tcp_info_rcv_mss UInt32,
- tcp_info_unacked UInt32,
- tcp_info_sacked UInt32,
- tcp_info_lost UInt32,
- tcp_info_retrans UInt32,
- tcp_info_fackets UInt32,
- tcp_info_last_data_sent UInt32,
- tcp_info_last_ack_sent UInt32,
- tcp_info_last_data_recv UInt32,
- tcp_info_last_ack_recv UInt32,
- tcp_info_pmtu UInt32,
- -- tcp_info_pmtu LowCardinality(UInt32),
- tcp_info_rcv_ssthresh UInt32,
- tcp_info_rtt UInt32,
- tcp_info_rtt_var UInt32,
- tcp_info_snd_ssthresh UInt32,
- tcp_info_snd_cwnd UInt32,
- tcp_info_adv_mss UInt32,
- tcp_info_reordering UInt32,
- tcp_info_rcv_rtt UInt32,
- tcp_info_rcv_space UInt32,
- tcp_info_total_retrans UInt32,
- tcp_info_pacing_rate UInt64,
- tcp_info_max_pacing_rate UInt64,
- tcp_info_bytes_acked UInt64,
- tcp_info_bytes_received UInt64,
- tcp_info_segs_out UInt32,
- tcp_info_segs_in UInt32,
- tcp_info_not_sent_bytes UInt32,
- tcp_info_min_rtt UInt32,
- tcp_info_data_segs_in UInt32,
- tcp_info_data_segs_out UInt32,
- tcp_info_delivery_rate UInt64,
- tcp_info_busy_time UInt64,
- tcp_info_rwnd_limited UInt64,
- tcp_info_sndbuf_limited UInt64,
- tcp_info_delivered UInt32,
- tcp_info_delivered_ce UInt32,
- tcp_info_bytes_sent UInt64,
- tcp_info_bytes_retrans UInt64,
- tcp_info_dsack_dups UInt32,
- tcp_info_reord_seen UInt32,
- tcp_info_rcv_ooopack UInt32,
- tcp_info_snd_wnd UInt32,
-
- congestion_algorithm_string LowCardinality(String),
- congestion_algorithm_enum LowCardinality(String),
-
- type_of_service UInt32,
- traffic_class UInt32,
- -- type_of_service LowCardinality(UInt32),
- -- traffic_class LowCardinality(UInt32),
-
- sk_mem_info_rmem_alloc UInt32,
- sk_mem_info_rcv_buf UInt32,
- sk_mem_info_wmem_alloc UInt32,
- sk_mem_info_snd_buf UInt32,
- sk_mem_info_fwd_alloc UInt32,
- sk_mem_info_wmem_queued UInt32,
- sk_mem_info_optmem UInt32,
- sk_mem_info_backlog UInt32,
- sk_mem_info_drops UInt32,
-
- shutdown_state UInt32,
- -- shutdown_state LowCardinality(UInt32),
-
- vegas_info_enabled UInt32,
- -- vegas_info_enabled LowCardinality(UInt32),
- vegas_info_rtt_cnt UInt32,
- vegas_info_rtt UInt32,
- vegas_info_min_rtt UInt32,
-
- dctcp_info_enabled UInt32,
- -- dctcp_info_enabled LowCardinality(UInt32),
- dctcp_info_ce_state UInt32,
- dctcp_info_alpha UInt32,
- dctcp_info_ab_ecn UInt32,
- dctcp_info_ab_tot UInt32,
-
- bbr_info_bw_lo UInt32,
- bbr_info_bw_hi UInt32,
- bbr_info_min_rtt UInt32,
- bbr_info_pacing_gain UInt32,
- bbr_info_cwnd_gain UInt32,
-
- class_id UInt32, -- LowCardinality?
- sock_opt UInt32, -- LowCardinality?
- c_group UInt64,
-
- )
- ENGINE = Kafka
- SETTINGS
- kafka_broker_list = 'redpanda-0:9092',
- kafka_topic_list = 'xtcp',
- kafka_format = 'ProtobufSingle',
- kafka_schema = 'flatxtcppb.proto:flat_xtcp_record',
- kafka_group_name = 'xtcp';
-
- --------------------------------------------------------------------------------------------------
-
- -- https://clickhouse.com/docs/en/integrations/kafka/kafka-table-engine#6-create-the-materialized-view
- DROP VIEW IF EXISTS xtcp.xtcp_records_mv;
-
- CREATE MATERIALIZED VIEW IF NOT EXISTS xtcp.xtcp_records_mv TO xtcp.xtcp_records
- AS SELECT * FROM xtcp.flat_xtcp_records_kafka;
-
- -- DETACH TABLE xtcp.xtcp_records_kafka;
- -- ATTACH TABLE xtcp.xtcp_records_kafka;
-
- -- SELECT * FROM xtcp.flat_xtcp_records_kafka SETTINGS stream_like_engine_allow_direct_select = 1;
- -- DESCRIBE TABLE xtcp.flat_xtcp_records_kafka SETTINGS stream_like_engine_allow_direct_select = 1;
-
-EOSQL
-
-# https://github.com/ClickHouse/ClickHouse/blob/master/docker/server/README.md#how-to-extend-this-image
-
-# https://github.com/ClickHouse/ClickHouse/blob/master/docker/server/entrypoint.sh
-
-# https://stackoverflow.com/questions/75079434/clickhouse-protobuf-output-format
-
-# https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#format_schema_path
-
-# https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka#configuration
-
-# https://clickhouse.com/docs/en/engines/table-engines/integrations/nats
-# CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
-# (
-# name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1],
-# name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2],
-# ...
-# ) ENGINE = NATS SETTINGS
-# nats_url = 'host:port',
-# nats_subjects = 'subject1,subject2,...',
-# nats_format = 'data_format'[,]
-# [nats_schema = '',]
-# [nats_num_consumers = N,]
-# [nats_queue_group = 'group_name',]
-# [nats_secure = false,]
-# [nats_max_reconnect = N,]
-# [nats_reconnect_wait = N,]
-# [nats_server_list = 'host1:port1,host2:port2,...',]
-# [nats_skip_broken_messages = N,]
-# [nats_max_block_size = N,]
-# [nats_flush_interval_ms = N,]
-# [nats_username = 'user',]
-# [nats_password = 'password',]
-# [nats_token = 'clickhouse',]
-# [nats_credential_file = '/var/nats_credentials',]
-# [nats_startup_connect_tries = '5']
-# [nats_max_rows_per_message = 1,]
-# [nats_handle_error_mode = 'default']
\ No newline at end of file
diff --git a/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.sh b/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.sh
deleted file mode 120000
index c11fea0..0000000
--- a/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.sh
+++ /dev/null
@@ -1 +0,0 @@
-init_xtcp_clickhouse.bash
\ No newline at end of file
diff --git a/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.sh b/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.sh
new file mode 100755
index 0000000..e11dab5
--- /dev/null
+++ b/build/containers/clickhouse/initdb.d/init_xtcp_clickhouse.sh
@@ -0,0 +1,469 @@
+#!/bin/bash
+# apparently /usr/bin/bash doesn't exist in the container
+
+#
+# This is the clickhouse database table creation script for xtcp2
+#
+
+set -e
+
+if [ "$EUID" -ne 0 ]
+then
+ echo "Please run as root"
+ exit 1
+fi
+
+rm -rf /docker-entrypoint-initdb.d/date
+rm -rf /docker-entrypoint-initdb.d/date_utc
+rm -rf /docker-entrypoint-initdb.d/whoami
+rm -rf /docker-entrypoint-initdb.d/success
+rm -rf /docker-entrypoint-initdb.d/xtcp.flat_xtcp_records_kafka*
+rm -rf /docker-entrypoint-initdb.d/xtcp.xtcp_records*
+
+d=$(date +date%Y_%m_%d_%H_%M_%S)
+du=$(date --utc +date%Y_%m_%d_%H_%M_%S)
+w=$(whoami)
+
+#usermod -d /home/das das
+
+# https://clickhouse.com/docs/en/interfaces/formats#protobuf
+# https://clickhouse.com/docs/en/interfaces/formats#protobufsingle
+# The Containerfile now copies the .proto to the correct location
+#cp /docker-entrypoint-initdb.d/flatxtcppb.proto /var/lib/clickhouse/format_schemas/flatxtcppb.proto
+
+# https://protobuf.dev/programming-guides/encoding/#structure
+
+echo "${d}" > /docker-entrypoint-initdb.d/date
+echo "${du}" > /docker-entrypoint-initdb.d/date_utc
+echo "${w}" > /docker-entrypoint-initdb.d/whoami
+
+# TODO
+#CODEC(T64, ZSTD(1))
+#https://clickhouse.com/blog/optimize-clickhouse-codecs-compression-schema
+
+# https://altinity.com/blog/2019-7-new-encodings-to-improve-clickhouse
+# https://altinity.com/blog/clickhouse-for-time-series
+
+clickhouse client -n <<-EOSQL
+ SELECT now();
+
+ --------------------------------------------------------------------------------------------------
+
+ -- Reload protobufs
+ -- https://clickhouse.com/docs/en/interfaces/formats#drop-protobuf-cache
+ SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf;
+
+ --------------------------------------------------------------------------------------------------
+
+ DROP DATABASE IF EXISTS xtcp;
+ CREATE DATABASE IF NOT EXISTS xtcp;
+
+ --------------------------------------------------------------------------------------------------
+
+ DROP TABLE IF EXISTS xtcp.xtcp_records;
+ CREATE TABLE IF NOT EXISTS xtcp.xtcp_records
+ (
+ -- https://clickhouse.com/docs/en/sql-reference/data-types/datetime64
+ sec DateTime64(3,'UTC') CODEC(DoubleDelta, LZ4),
+ nsec Int64,
+
+ -- https://clickhouse.com/docs/en/sql-reference/data-types/lowcardinality
+ hostname LowCardinality(String),
+ tag LowCardinality(String),
+ record_count UInt64 CODEC(DoubleDelta, LZ4),
+ netlinker_id UInt32 CODEC(LZ4),
+
+ inet_diag_msg_family UInt32 CODEC(LZ4),
+ inet_diag_msg_state UInt32 CODEC(LZ4),
+ -- inet_diag_msg_family LowCardinality(UInt32),
+ -- inet_diag_msg_state LowCardinality(UInt32),
+ inet_diag_msg_timer UInt32 CODEC(LZ4),
+ inet_diag_msg_retrans UInt32 CODEC(LZ4),
+
+ inet_diag_msg_socket_source_port UInt32 CODEC(LZ4),
+ inet_diag_msg_socket_destination_port UInt32 CODEC(LZ4),
+ inet_diag_msg_socket_source String CODEC(ZSTD),
+ inet_diag_msg_socket_source_ipv4 Nullable(IPv4),
+ inet_diag_msg_socket_source_ipv6 Nullable(IPv6),
+ inet_diag_msg_socket_destination String CODEC(ZSTD),
+ inet_diag_msg_socket_destination_ipv4 Nullable(IPv4),
+ inet_diag_msg_socket_destination_ipv6 Nullable(IPv6),
+ inet_diag_msg_socket_interface UInt32 CODEC(LZ4),
+ inet_diag_msg_socket_cookie UInt64 CODEC(LZ4),
+ inet_diag_msg_socket_dest_asn UInt64 CODEC(LZ4),
+ inet_diag_msg_socket_next_hop_asn UInt64 CODEC(LZ4),
+ inet_diag_msg_socket_source_asn UInt64 CODEC(LZ4),
+
+ inet_diag_msg_expires UInt32 CODEC(LZ4),
+ inet_diag_msg_rqueue UInt32 CODEC(LZ4),
+ inet_diag_msg_wqueue UInt32 CODEC(LZ4),
+ inet_diag_msg_uid UInt32 CODEC(LZ4),
+ inet_diag_msg_inode UInt32 CODEC(LZ4),
+
+ mem_info_rmem UInt32 CODEC(LZ4),
+ mem_info_wmem UInt32 CODEC(LZ4),
+ mem_info_fmem UInt32 CODEC(LZ4),
+ mem_info_tmem UInt32 CODEC(LZ4),
+
+ tcp_info_state UInt32 CODEC(LZ4),
+ tcp_info_ca_state UInt32 CODEC(LZ4),
+ -- tcp_info_state LowCardinality(UInt32),
+ -- tcp_info_ca_state LowCardinality(UInt32),
+ tcp_info_retransmits UInt32 CODEC(LZ4),
+ tcp_info_probes UInt32 CODEC(LZ4),
+ tcp_info_backoff UInt32 CODEC(LZ4),
+ tcp_info_options UInt32 CODEC(LZ4),
+ tcp_info_send_scale UInt32 CODEC(LZ4),
+ tcp_info_rcv_scale UInt32 CODEC(LZ4),
+ tcp_info_delivery_rate_app_limited UInt32 CODEC(LZ4),
+ tcp_info_fast_open_client_failed UInt32 CODEC(LZ4),
+ tcp_info_rto UInt32 CODEC(LZ4),
+ tcp_info_ato UInt32 CODEC(LZ4),
+ tcp_info_snd_mss UInt32 CODEC(LZ4),
+ tcp_info_rcv_mss UInt32 CODEC(LZ4),
+ tcp_info_unacked UInt32 CODEC(LZ4),
+ tcp_info_sacked UInt32 CODEC(LZ4),
+ tcp_info_lost UInt32 CODEC(LZ4),
+ tcp_info_retrans UInt32 CODEC(LZ4),
+ tcp_info_fackets UInt32 CODEC(LZ4),
+ tcp_info_last_data_sent UInt32 CODEC(LZ4),
+ tcp_info_last_ack_sent UInt32 CODEC(LZ4),
+ tcp_info_last_data_recv UInt32 CODEC(LZ4),
+ tcp_info_last_ack_recv UInt32 CODEC(LZ4),
+ tcp_info_pmtu UInt32 CODEC(LZ4),
+ -- tcp_info_pmtu LowCardinality(UInt32),
+ tcp_info_rcv_ssthresh UInt32 CODEC(LZ4),
+ tcp_info_rtt UInt32 CODEC(LZ4),
+ tcp_info_rtt_var UInt32 CODEC(LZ4),
+ tcp_info_snd_ssthresh UInt32 CODEC(LZ4),
+ tcp_info_snd_cwnd UInt32 CODEC(LZ4),
+ tcp_info_adv_mss UInt32 CODEC(LZ4),
+ tcp_info_reordering UInt32 CODEC(LZ4),
+ tcp_info_rcv_rtt UInt32 CODEC(LZ4),
+ tcp_info_rcv_space UInt32 CODEC(LZ4),
+ tcp_info_total_retrans UInt32 CODEC(LZ4),
+ tcp_info_pacing_rate UInt64 CODEC(LZ4),
+ tcp_info_max_pacing_rate UInt64 CODEC(LZ4),
+ tcp_info_bytes_acked UInt64 CODEC(LZ4),
+ tcp_info_bytes_received UInt64 CODEC(LZ4),
+ tcp_info_segs_out UInt32 CODEC(LZ4),
+ tcp_info_segs_in UInt32 CODEC(LZ4),
+ tcp_info_not_sent_bytes UInt32 CODEC(LZ4),
+ tcp_info_min_rtt UInt32 CODEC(LZ4),
+ tcp_info_data_segs_in UInt32 CODEC(LZ4),
+ tcp_info_data_segs_out UInt32 CODEC(LZ4),
+ tcp_info_delivery_rate UInt64 CODEC(LZ4),
+ tcp_info_busy_time UInt64 CODEC(LZ4),
+ tcp_info_rwnd_limited UInt64 CODEC(LZ4),
+ tcp_info_sndbuf_limited UInt64 CODEC(LZ4),
+ tcp_info_delivered UInt32 CODEC(LZ4),
+ tcp_info_delivered_ce UInt32 CODEC(LZ4),
+ tcp_info_bytes_sent UInt64 CODEC(LZ4),
+ tcp_info_bytes_retrans UInt64 CODEC(LZ4),
+ tcp_info_dsack_dups UInt32 CODEC(LZ4),
+ tcp_info_reord_seen UInt32 CODEC(LZ4),
+ tcp_info_rcv_ooopack UInt32 CODEC(LZ4),
+ tcp_info_snd_wnd UInt32 CODEC(LZ4),
+
+ congestion_algorithm_string LowCardinality(String),
+ congestion_algorithm_enum LowCardinality(String),
+
+ type_of_service UInt32 CODEC(LZ4),
+ traffic_class UInt32 CODEC(LZ4),
+ -- type_of_service LowCardinality(UInt32),
+ -- traffic_class LowCardinality(UInt32),
+
+ sk_mem_info_rmem_alloc UInt32 CODEC(LZ4),
+ sk_mem_info_rcv_buf UInt32 CODEC(LZ4),
+ sk_mem_info_wmem_alloc UInt32 CODEC(LZ4),
+ sk_mem_info_snd_buf UInt32 CODEC(LZ4),
+ sk_mem_info_fwd_alloc UInt32 CODEC(LZ4),
+ sk_mem_info_wmem_queued UInt32 CODEC(LZ4),
+ sk_mem_info_optmem UInt32 CODEC(LZ4),
+ sk_mem_info_backlog UInt32 CODEC(LZ4),
+ sk_mem_info_drops UInt32 CODEC(LZ4),
+
+ shutdown_state UInt32 CODEC(LZ4),
+ -- shutdown_state LowCardinality(UInt32),
+
+ vegas_info_enabled UInt32 CODEC(LZ4),
+ -- vegas_info_enabled LowCardinality(UInt32),
+ vegas_info_rtt_cnt UInt32 CODEC(LZ4),
+ vegas_info_rtt UInt32 CODEC(LZ4),
+ vegas_info_min_rtt UInt32 CODEC(LZ4),
+
+ dctcp_info_enabled UInt32 CODEC(LZ4),
+ -- dctcp_info_enabled LowCardinality(UInt32),
+ dctcp_info_ce_state UInt32 CODEC(LZ4),
+ dctcp_info_alpha UInt32 CODEC(LZ4),
+ dctcp_info_ab_ecn UInt32 CODEC(LZ4),
+ dctcp_info_ab_tot UInt32 CODEC(LZ4),
+
+ bbr_info_bw_lo UInt32 CODEC(LZ4),
+ bbr_info_bw_hi UInt32 CODEC(LZ4),
+ bbr_info_min_rtt UInt32 CODEC(LZ4),
+ bbr_info_pacing_gain UInt32 CODEC(LZ4),
+ bbr_info_cwnd_gain UInt32 CODEC(LZ4),
+
+ class_id UInt32 CODEC(LZ4), -- LowCardinality?
+ sock_opt UInt32 CODEC(LZ4), -- LowCardinality?
+ c_group UInt64 CODEC(LZ4),
+
+ )
+ ENGINE = MergeTree
+ -- ENGINE = ReplicatedMergeTree
+ --PARTITION BY toYYYYMMDD(sec)
+ ORDER BY (sec, nsec, hostname, netlinker_id, record_count)
+ TTL toDateTime(sec) + INTERVAL 2 MONTH DELETE;
+
+ --------------------------------------------------------------------------------------------------
+
+ -- Not using Nested!
+ -- https://clickhouse.com/docs/en/sql-reference/data-types/nested-data-structures/nested
+ -- Not using Nullable, because it uses more space and apparently "almost always negatively affects performance"
+ -- https://clickhouse.com/docs/en/sql-reference/data-types/nullable
+
+ CREATE TABLE IF NOT EXISTS xtcp.flat_xtcp_records_kafka
+ (
+ -- https://clickhouse.com/docs/en/sql-reference/data-types/datetime64
+ sec DateTime64(3,'UTC') CODEC(DoubleDelta, LZ4),
+ nsec Int64,
+
+ -- https://clickhouse.com/docs/en/sql-reference/data-types/lowcardinality
+ hostname LowCardinality(String),
+ tag LowCardinality(String),
+ record_count UInt64 CODEC(DoubleDelta, LZ4),
+ netlinker_id UInt32 CODEC(LZ4),
+
+ inet_diag_msg_family UInt32 CODEC(LZ4),
+ inet_diag_msg_state UInt32 CODEC(LZ4),
+ -- inet_diag_msg_family LowCardinality(UInt32),
+ -- inet_diag_msg_state LowCardinality(UInt32),
+ inet_diag_msg_timer UInt32 CODEC(LZ4),
+ inet_diag_msg_retrans UInt32 CODEC(LZ4),
+
+ inet_diag_msg_socket_source_port UInt32 CODEC(LZ4),
+ inet_diag_msg_socket_destination_port UInt32 CODEC(LZ4),
+ inet_diag_msg_socket_source String CODEC(ZSTD),
+ inet_diag_msg_socket_source_ipv4 Nullable(IPv4),
+ inet_diag_msg_socket_source_ipv6 Nullable(IPv6),
+ inet_diag_msg_socket_destination String CODEC(ZSTD),
+ inet_diag_msg_socket_destination_ipv4 Nullable(IPv4),
+ inet_diag_msg_socket_destination_ipv6 Nullable(IPv6),
+ inet_diag_msg_socket_interface UInt32 CODEC(LZ4),
+ inet_diag_msg_socket_cookie UInt64 CODEC(LZ4),
+ inet_diag_msg_socket_dest_asn UInt64 CODEC(LZ4),
+ inet_diag_msg_socket_next_hop_asn UInt64 CODEC(LZ4),
+ inet_diag_msg_socket_source_asn UInt64 CODEC(LZ4),
+
+ inet_diag_msg_expires UInt32 CODEC(LZ4),
+ inet_diag_msg_rqueue UInt32 CODEC(LZ4),
+ inet_diag_msg_wqueue UInt32 CODEC(LZ4),
+ inet_diag_msg_uid UInt32 CODEC(LZ4),
+ inet_diag_msg_inode UInt32 CODEC(LZ4),
+
+ mem_info_rmem UInt32 CODEC(LZ4),
+ mem_info_wmem UInt32 CODEC(LZ4),
+ mem_info_fmem UInt32 CODEC(LZ4),
+ mem_info_tmem UInt32 CODEC(LZ4),
+
+ tcp_info_state UInt32 CODEC(LZ4),
+ tcp_info_ca_state UInt32 CODEC(LZ4),
+ -- tcp_info_state LowCardinality(UInt32),
+ -- tcp_info_ca_state LowCardinality(UInt32),
+ tcp_info_retransmits UInt32 CODEC(LZ4),
+ tcp_info_probes UInt32 CODEC(LZ4),
+ tcp_info_backoff UInt32 CODEC(LZ4),
+ tcp_info_options UInt32 CODEC(LZ4),
+ tcp_info_send_scale UInt32 CODEC(LZ4),
+ tcp_info_rcv_scale UInt32 CODEC(LZ4),
+ tcp_info_delivery_rate_app_limited UInt32 CODEC(LZ4),
+ tcp_info_fast_open_client_failed UInt32 CODEC(LZ4),
+ tcp_info_rto UInt32 CODEC(LZ4),
+ tcp_info_ato UInt32 CODEC(LZ4),
+ tcp_info_snd_mss UInt32 CODEC(LZ4),
+ tcp_info_rcv_mss UInt32 CODEC(LZ4),
+ tcp_info_unacked UInt32 CODEC(LZ4),
+ tcp_info_sacked UInt32 CODEC(LZ4),
+ tcp_info_lost UInt32 CODEC(LZ4),
+ tcp_info_retrans UInt32 CODEC(LZ4),
+ tcp_info_fackets UInt32 CODEC(LZ4),
+ tcp_info_last_data_sent UInt32 CODEC(LZ4),
+ tcp_info_last_ack_sent UInt32 CODEC(LZ4),
+ tcp_info_last_data_recv UInt32 CODEC(LZ4),
+ tcp_info_last_ack_recv UInt32 CODEC(LZ4),
+ tcp_info_pmtu UInt32 CODEC(LZ4),
+ -- tcp_info_pmtu LowCardinality(UInt32),
+ tcp_info_rcv_ssthresh UInt32 CODEC(LZ4),
+ tcp_info_rtt UInt32 CODEC(LZ4),
+ tcp_info_rtt_var UInt32 CODEC(LZ4),
+ tcp_info_snd_ssthresh UInt32 CODEC(LZ4),
+ tcp_info_snd_cwnd UInt32 CODEC(LZ4),
+ tcp_info_adv_mss UInt32 CODEC(LZ4),
+ tcp_info_reordering UInt32 CODEC(LZ4),
+ tcp_info_rcv_rtt UInt32 CODEC(LZ4),
+ tcp_info_rcv_space UInt32 CODEC(LZ4),
+ tcp_info_total_retrans UInt32 CODEC(LZ4),
+ tcp_info_pacing_rate UInt64 CODEC(LZ4),
+ tcp_info_max_pacing_rate UInt64 CODEC(LZ4),
+ tcp_info_bytes_acked UInt64 CODEC(LZ4),
+ tcp_info_bytes_received UInt64 CODEC(LZ4),
+ tcp_info_segs_out UInt32 CODEC(LZ4),
+ tcp_info_segs_in UInt32 CODEC(LZ4),
+ tcp_info_not_sent_bytes UInt32 CODEC(LZ4),
+ tcp_info_min_rtt UInt32 CODEC(LZ4),
+ tcp_info_data_segs_in UInt32 CODEC(LZ4),
+ tcp_info_data_segs_out UInt32 CODEC(LZ4),
+ tcp_info_delivery_rate UInt64 CODEC(LZ4),
+ tcp_info_busy_time UInt64 CODEC(LZ4),
+ tcp_info_rwnd_limited UInt64 CODEC(LZ4),
+ tcp_info_sndbuf_limited UInt64 CODEC(LZ4),
+ tcp_info_delivered UInt32 CODEC(LZ4),
+ tcp_info_delivered_ce UInt32 CODEC(LZ4),
+ tcp_info_bytes_sent UInt64 CODEC(LZ4),
+ tcp_info_bytes_retrans UInt64 CODEC(LZ4),
+ tcp_info_dsack_dups UInt32 CODEC(LZ4),
+ tcp_info_reord_seen UInt32 CODEC(LZ4),
+ tcp_info_rcv_ooopack UInt32 CODEC(LZ4),
+ tcp_info_snd_wnd UInt32 CODEC(LZ4),
+
+ congestion_algorithm_string LowCardinality(String),
+ congestion_algorithm_enum LowCardinality(String),
+
+ type_of_service UInt32 CODEC(LZ4),
+ traffic_class UInt32 CODEC(LZ4),
+ -- type_of_service LowCardinality(UInt32),
+ -- traffic_class LowCardinality(UInt32),
+
+ sk_mem_info_rmem_alloc UInt32 CODEC(LZ4),
+ sk_mem_info_rcv_buf UInt32 CODEC(LZ4),
+ sk_mem_info_wmem_alloc UInt32 CODEC(LZ4),
+ sk_mem_info_snd_buf UInt32 CODEC(LZ4),
+ sk_mem_info_fwd_alloc UInt32 CODEC(LZ4),
+ sk_mem_info_wmem_queued UInt32 CODEC(LZ4),
+ sk_mem_info_optmem UInt32 CODEC(LZ4),
+ sk_mem_info_backlog UInt32 CODEC(LZ4),
+ sk_mem_info_drops UInt32 CODEC(LZ4),
+
+ shutdown_state UInt32 CODEC(LZ4),
+ -- shutdown_state LowCardinality(UInt32),
+
+ vegas_info_enabled UInt32 CODEC(LZ4),
+ -- vegas_info_enabled LowCardinality(UInt32),
+ vegas_info_rtt_cnt UInt32 CODEC(LZ4),
+ vegas_info_rtt UInt32 CODEC(LZ4),
+ vegas_info_min_rtt UInt32 CODEC(LZ4),
+
+ dctcp_info_enabled UInt32 CODEC(LZ4),
+ -- dctcp_info_enabled LowCardinality(UInt32),
+ dctcp_info_ce_state UInt32 CODEC(LZ4),
+ dctcp_info_alpha UInt32 CODEC(LZ4),
+ dctcp_info_ab_ecn UInt32 CODEC(LZ4),
+ dctcp_info_ab_tot UInt32 CODEC(LZ4),
+
+ bbr_info_bw_lo UInt32 CODEC(LZ4),
+ bbr_info_bw_hi UInt32 CODEC(LZ4),
+ bbr_info_min_rtt UInt32 CODEC(LZ4),
+ bbr_info_pacing_gain UInt32 CODEC(LZ4),
+ bbr_info_cwnd_gain UInt32 CODEC(LZ4),
+
+ class_id UInt32 CODEC(LZ4), -- LowCardinality?
+ sock_opt UInt32 CODEC(LZ4), -- LowCardinality?
+ c_group UInt64 CODEC(LZ4),
+
+ )
+ ENGINE = Kafka
+ SETTINGS
+ kafka_broker_list = 'redpanda-0:9092',
+ kafka_topic_list = 'xtcp',
+ kafka_format = 'ProtobufSingle',
+ kafka_schema = 'flatxtcppb.proto:flat_xtcp_record',
+ kafka_group_name = 'xtcp';
+
+ --------------------------------------------------------------------------------------------------
+
+ -- https://clickhouse.com/docs/en/integrations/kafka/kafka-table-engine#6-create-the-materialized-view
+ DROP VIEW IF EXISTS xtcp.xtcp_records_mv;
+
+ CREATE MATERIALIZED VIEW IF NOT EXISTS xtcp.xtcp_records_mv TO xtcp.xtcp_records
+ AS SELECT * FROM xtcp.flat_xtcp_records_kafka;
+
+ -- DETACH TABLE xtcp.xtcp_records_kafka;
+ -- ATTACH TABLE xtcp.xtcp_records_kafka;
+
+ -- SELECT * FROM xtcp.flat_xtcp_records_kafka SETTINGS stream_like_engine_allow_direct_select = 1;
+ -- DESCRIBE TABLE xtcp.flat_xtcp_records_kafka SETTINGS stream_like_engine_allow_direct_select = 1;
+
+ -- https://clickhouse.com/docs/en/sql-reference/statements/describe-table
+ DESCRIBE TABLE xtcp.flat_xtcp_records_kafka INTO OUTFILE '/docker-entrypoint-initdb.d/xtcp.flat_xtcp_records_kafka';
+ DESCRIBE TABLE xtcp.xtcp_records INTO OUTFILE '/docker-entrypoint-initdb.d/xtcp.xtcp_records';
+
+ DESCRIBE TABLE xtcp.flat_xtcp_records_kafka INTO OUTFILE '/docker-entrypoint-initdb.d/xtcp.flat_xtcp_records_kafka.csv' FORMAT CSV;
+ DESCRIBE TABLE xtcp.xtcp_records INTO OUTFILE '/docker-entrypoint-initdb.d/xtcp.xtcp_records.csv' FORMAT CSV;
+
+EOSQL
+
+#-----------------------------------
+# This code does a quick check that the tables for the Kafka and the real output
+# table match. The tables NEED to match.
+
+if [ ! -f /usr/bin/sha512sum ]; then
+ echo "/usr/bin/sha512sum not found";
+ exit 1;
+fi
+
+file1='/docker-entrypoint-initdb.d/xtcp.flat_xtcp_records_kafka';
+file2='/docker-entrypoint-initdb.d/xtcp.xtcp_records';
+
+sha512sum1=$(sha512sum $file1 | cut -d ' ' -f 1);
+sha512sum2=$(sha512sum $file2 | cut -d ' ' -f 1);
+
+if [ "${sha512sum1}" = "${sha512sum2}" ]; then
+ echo "DESCRIBE TABLES MATCH. Woot woot!";
+else
+ echo "DESCRIBE TABLES DO NOT MATCH!! Fix the tables!!";
+ exit 1;
+fi
+
+
+# https://github.com/ClickHouse/ClickHouse/blob/master/docker/server/README.md#how-to-extend-this-image
+
+# https://github.com/ClickHouse/ClickHouse/blob/master/docker/server/entrypoint.sh
+
+# https://stackoverflow.com/questions/75079434/clickhouse-protobuf-output-format
+
+# https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#format_schema_path
+
+# https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka#configuration
+
+# https://clickhouse.com/docs/en/engines/table-engines/integrations/nats
+# CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
+# (
+# name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1],
+# name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2],
+# ...
+# ) ENGINE = NATS SETTINGS
+# nats_url = 'host:port',
+# nats_subjects = 'subject1,subject2,...',
+# nats_format = 'data_format'[,]
+# [nats_schema = '',]
+# [nats_num_consumers = N,]
+# [nats_queue_group = 'group_name',]
+# [nats_secure = false,]
+# [nats_max_reconnect = N,]
+# [nats_reconnect_wait = N,]
+# [nats_server_list = 'host1:port1,host2:port2,...',]
+# [nats_skip_broken_messages = N,]
+# [nats_max_block_size = N,]
+# [nats_flush_interval_ms = N,]
+# [nats_username = 'user',]
+# [nats_password = 'password',]
+# [nats_token = 'clickhouse',]
+# [nats_credential_file = '/var/nats_credentials',]
+# [nats_startup_connect_tries = '5']
+# [nats_max_rows_per_message = 1,]
+# [nats_handle_error_mode = 'default']
+
+echo success > /docker-entrypoint-initdb.d/success
+
+#end
\ No newline at end of file
diff --git a/build/containers/clickhouse/initdb.d/success b/build/containers/clickhouse/initdb.d/success
new file mode 100644
index 0000000..2e9ba47
--- /dev/null
+++ b/build/containers/clickhouse/initdb.d/success
@@ -0,0 +1 @@
+success
diff --git a/build/containers/clickhouse/initdb.d/xtcp.flat_xtcp_records_kafka b/build/containers/clickhouse/initdb.d/xtcp.flat_xtcp_records_kafka
new file mode 100644
index 0000000..c14487c
--- /dev/null
+++ b/build/containers/clickhouse/initdb.d/xtcp.flat_xtcp_records_kafka
@@ -0,0 +1,119 @@
+sec DateTime64(3, \'UTC\') DoubleDelta, LZ4
+nsec Int64
+hostname LowCardinality(String)
+tag LowCardinality(String)
+record_count UInt64 DoubleDelta, LZ4
+netlinker_id UInt32 LZ4
+inet_diag_msg_family UInt32 LZ4
+inet_diag_msg_state UInt32 LZ4
+inet_diag_msg_timer UInt32 LZ4
+inet_diag_msg_retrans UInt32 LZ4
+inet_diag_msg_socket_source_port UInt32 LZ4
+inet_diag_msg_socket_destination_port UInt32 LZ4
+inet_diag_msg_socket_source String ZSTD(1)
+inet_diag_msg_socket_source_ipv4 Nullable(IPv4)
+inet_diag_msg_socket_source_ipv6 Nullable(IPv6)
+inet_diag_msg_socket_destination String ZSTD(1)
+inet_diag_msg_socket_destination_ipv4 Nullable(IPv4)
+inet_diag_msg_socket_destination_ipv6 Nullable(IPv6)
+inet_diag_msg_socket_interface UInt32 LZ4
+inet_diag_msg_socket_cookie UInt64 LZ4
+inet_diag_msg_socket_dest_asn UInt64 LZ4
+inet_diag_msg_socket_next_hop_asn UInt64 LZ4
+inet_diag_msg_socket_source_asn UInt64 LZ4
+inet_diag_msg_expires UInt32 LZ4
+inet_diag_msg_rqueue UInt32 LZ4
+inet_diag_msg_wqueue UInt32 LZ4
+inet_diag_msg_uid UInt32 LZ4
+inet_diag_msg_inode UInt32 LZ4
+mem_info_rmem UInt32 LZ4
+mem_info_wmem UInt32 LZ4
+mem_info_fmem UInt32 LZ4
+mem_info_tmem UInt32 LZ4
+tcp_info_state UInt32 LZ4
+tcp_info_ca_state UInt32 LZ4
+tcp_info_retransmits UInt32 LZ4
+tcp_info_probes UInt32 LZ4
+tcp_info_backoff UInt32 LZ4
+tcp_info_options UInt32 LZ4
+tcp_info_send_scale UInt32 LZ4
+tcp_info_rcv_scale UInt32 LZ4
+tcp_info_delivery_rate_app_limited UInt32 LZ4
+tcp_info_fast_open_client_failed UInt32 LZ4
+tcp_info_rto UInt32 LZ4
+tcp_info_ato UInt32 LZ4
+tcp_info_snd_mss UInt32 LZ4
+tcp_info_rcv_mss UInt32 LZ4
+tcp_info_unacked UInt32 LZ4
+tcp_info_sacked UInt32 LZ4
+tcp_info_lost UInt32 LZ4
+tcp_info_retrans UInt32 LZ4
+tcp_info_fackets UInt32 LZ4
+tcp_info_last_data_sent UInt32 LZ4
+tcp_info_last_ack_sent UInt32 LZ4
+tcp_info_last_data_recv UInt32 LZ4
+tcp_info_last_ack_recv UInt32 LZ4
+tcp_info_pmtu UInt32 LZ4
+tcp_info_rcv_ssthresh UInt32 LZ4
+tcp_info_rtt UInt32 LZ4
+tcp_info_rtt_var UInt32 LZ4
+tcp_info_snd_ssthresh UInt32 LZ4
+tcp_info_snd_cwnd UInt32 LZ4
+tcp_info_adv_mss UInt32 LZ4
+tcp_info_reordering UInt32 LZ4
+tcp_info_rcv_rtt UInt32 LZ4
+tcp_info_rcv_space UInt32 LZ4
+tcp_info_total_retrans UInt32 LZ4
+tcp_info_pacing_rate UInt64 LZ4
+tcp_info_max_pacing_rate UInt64 LZ4
+tcp_info_bytes_acked UInt64 LZ4
+tcp_info_bytes_received UInt64 LZ4
+tcp_info_segs_out UInt32 LZ4
+tcp_info_segs_in UInt32 LZ4
+tcp_info_not_sent_bytes UInt32 LZ4
+tcp_info_min_rtt UInt32 LZ4
+tcp_info_data_segs_in UInt32 LZ4
+tcp_info_data_segs_out UInt32 LZ4
+tcp_info_delivery_rate UInt64 LZ4
+tcp_info_busy_time UInt64 LZ4
+tcp_info_rwnd_limited UInt64 LZ4
+tcp_info_sndbuf_limited UInt64 LZ4
+tcp_info_delivered UInt32 LZ4
+tcp_info_delivered_ce UInt32 LZ4
+tcp_info_bytes_sent UInt64 LZ4
+tcp_info_bytes_retrans UInt64 LZ4
+tcp_info_dsack_dups UInt32 LZ4
+tcp_info_reord_seen UInt32 LZ4
+tcp_info_rcv_ooopack UInt32 LZ4
+tcp_info_snd_wnd UInt32 LZ4
+congestion_algorithm_string LowCardinality(String)
+congestion_algorithm_enum LowCardinality(String)
+type_of_service UInt32 LZ4
+traffic_class UInt32 LZ4
+sk_mem_info_rmem_alloc UInt32 LZ4
+sk_mem_info_rcv_buf UInt32 LZ4
+sk_mem_info_wmem_alloc UInt32 LZ4
+sk_mem_info_snd_buf UInt32 LZ4
+sk_mem_info_fwd_alloc UInt32 LZ4
+sk_mem_info_wmem_queued UInt32 LZ4
+sk_mem_info_optmem UInt32 LZ4
+sk_mem_info_backlog UInt32 LZ4
+sk_mem_info_drops UInt32 LZ4
+shutdown_state UInt32 LZ4
+vegas_info_enabled UInt32 LZ4
+vegas_info_rtt_cnt UInt32 LZ4
+vegas_info_rtt UInt32 LZ4
+vegas_info_min_rtt UInt32 LZ4
+dctcp_info_enabled UInt32 LZ4
+dctcp_info_ce_state UInt32 LZ4
+dctcp_info_alpha UInt32 LZ4
+dctcp_info_ab_ecn UInt32 LZ4
+dctcp_info_ab_tot UInt32 LZ4
+bbr_info_bw_lo UInt32 LZ4
+bbr_info_bw_hi UInt32 LZ4
+bbr_info_min_rtt UInt32 LZ4
+bbr_info_pacing_gain UInt32 LZ4
+bbr_info_cwnd_gain UInt32 LZ4
+class_id UInt32 LZ4
+sock_opt UInt32 LZ4
+c_group UInt64 LZ4
diff --git a/build/containers/clickhouse/initdb.d/xtcp.flat_xtcp_records_kafka.csv b/build/containers/clickhouse/initdb.d/xtcp.flat_xtcp_records_kafka.csv
new file mode 100644
index 0000000..dc510ec
--- /dev/null
+++ b/build/containers/clickhouse/initdb.d/xtcp.flat_xtcp_records_kafka.csv
@@ -0,0 +1,119 @@
+"sec","DateTime64(3, 'UTC')","","","","DoubleDelta, LZ4",""
+"nsec","Int64","","","","",""
+"hostname","LowCardinality(String)","","","","",""
+"tag","LowCardinality(String)","","","","",""
+"record_count","UInt64","","","","DoubleDelta, LZ4",""
+"netlinker_id","UInt32","","","","LZ4",""
+"inet_diag_msg_family","UInt32","","","","LZ4",""
+"inet_diag_msg_state","UInt32","","","","LZ4",""
+"inet_diag_msg_timer","UInt32","","","","LZ4",""
+"inet_diag_msg_retrans","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_source_port","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_destination_port","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_source","String","","","","ZSTD(1)",""
+"inet_diag_msg_socket_source_ipv4","Nullable(IPv4)","","","","",""
+"inet_diag_msg_socket_source_ipv6","Nullable(IPv6)","","","","",""
+"inet_diag_msg_socket_destination","String","","","","ZSTD(1)",""
+"inet_diag_msg_socket_destination_ipv4","Nullable(IPv4)","","","","",""
+"inet_diag_msg_socket_destination_ipv6","Nullable(IPv6)","","","","",""
+"inet_diag_msg_socket_interface","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_cookie","UInt64","","","","LZ4",""
+"inet_diag_msg_socket_dest_asn","UInt64","","","","LZ4",""
+"inet_diag_msg_socket_next_hop_asn","UInt64","","","","LZ4",""
+"inet_diag_msg_socket_source_asn","UInt64","","","","LZ4",""
+"inet_diag_msg_expires","UInt32","","","","LZ4",""
+"inet_diag_msg_rqueue","UInt32","","","","LZ4",""
+"inet_diag_msg_wqueue","UInt32","","","","LZ4",""
+"inet_diag_msg_uid","UInt32","","","","LZ4",""
+"inet_diag_msg_inode","UInt32","","","","LZ4",""
+"mem_info_rmem","UInt32","","","","LZ4",""
+"mem_info_wmem","UInt32","","","","LZ4",""
+"mem_info_fmem","UInt32","","","","LZ4",""
+"mem_info_tmem","UInt32","","","","LZ4",""
+"tcp_info_state","UInt32","","","","LZ4",""
+"tcp_info_ca_state","UInt32","","","","LZ4",""
+"tcp_info_retransmits","UInt32","","","","LZ4",""
+"tcp_info_probes","UInt32","","","","LZ4",""
+"tcp_info_backoff","UInt32","","","","LZ4",""
+"tcp_info_options","UInt32","","","","LZ4",""
+"tcp_info_send_scale","UInt32","","","","LZ4",""
+"tcp_info_rcv_scale","UInt32","","","","LZ4",""
+"tcp_info_delivery_rate_app_limited","UInt32","","","","LZ4",""
+"tcp_info_fast_open_client_failed","UInt32","","","","LZ4",""
+"tcp_info_rto","UInt32","","","","LZ4",""
+"tcp_info_ato","UInt32","","","","LZ4",""
+"tcp_info_snd_mss","UInt32","","","","LZ4",""
+"tcp_info_rcv_mss","UInt32","","","","LZ4",""
+"tcp_info_unacked","UInt32","","","","LZ4",""
+"tcp_info_sacked","UInt32","","","","LZ4",""
+"tcp_info_lost","UInt32","","","","LZ4",""
+"tcp_info_retrans","UInt32","","","","LZ4",""
+"tcp_info_fackets","UInt32","","","","LZ4",""
+"tcp_info_last_data_sent","UInt32","","","","LZ4",""
+"tcp_info_last_ack_sent","UInt32","","","","LZ4",""
+"tcp_info_last_data_recv","UInt32","","","","LZ4",""
+"tcp_info_last_ack_recv","UInt32","","","","LZ4",""
+"tcp_info_pmtu","UInt32","","","","LZ4",""
+"tcp_info_rcv_ssthresh","UInt32","","","","LZ4",""
+"tcp_info_rtt","UInt32","","","","LZ4",""
+"tcp_info_rtt_var","UInt32","","","","LZ4",""
+"tcp_info_snd_ssthresh","UInt32","","","","LZ4",""
+"tcp_info_snd_cwnd","UInt32","","","","LZ4",""
+"tcp_info_adv_mss","UInt32","","","","LZ4",""
+"tcp_info_reordering","UInt32","","","","LZ4",""
+"tcp_info_rcv_rtt","UInt32","","","","LZ4",""
+"tcp_info_rcv_space","UInt32","","","","LZ4",""
+"tcp_info_total_retrans","UInt32","","","","LZ4",""
+"tcp_info_pacing_rate","UInt64","","","","LZ4",""
+"tcp_info_max_pacing_rate","UInt64","","","","LZ4",""
+"tcp_info_bytes_acked","UInt64","","","","LZ4",""
+"tcp_info_bytes_received","UInt64","","","","LZ4",""
+"tcp_info_segs_out","UInt32","","","","LZ4",""
+"tcp_info_segs_in","UInt32","","","","LZ4",""
+"tcp_info_not_sent_bytes","UInt32","","","","LZ4",""
+"tcp_info_min_rtt","UInt32","","","","LZ4",""
+"tcp_info_data_segs_in","UInt32","","","","LZ4",""
+"tcp_info_data_segs_out","UInt32","","","","LZ4",""
+"tcp_info_delivery_rate","UInt64","","","","LZ4",""
+"tcp_info_busy_time","UInt64","","","","LZ4",""
+"tcp_info_rwnd_limited","UInt64","","","","LZ4",""
+"tcp_info_sndbuf_limited","UInt64","","","","LZ4",""
+"tcp_info_delivered","UInt32","","","","LZ4",""
+"tcp_info_delivered_ce","UInt32","","","","LZ4",""
+"tcp_info_bytes_sent","UInt64","","","","LZ4",""
+"tcp_info_bytes_retrans","UInt64","","","","LZ4",""
+"tcp_info_dsack_dups","UInt32","","","","LZ4",""
+"tcp_info_reord_seen","UInt32","","","","LZ4",""
+"tcp_info_rcv_ooopack","UInt32","","","","LZ4",""
+"tcp_info_snd_wnd","UInt32","","","","LZ4",""
+"congestion_algorithm_string","LowCardinality(String)","","","","",""
+"congestion_algorithm_enum","LowCardinality(String)","","","","",""
+"type_of_service","UInt32","","","","LZ4",""
+"traffic_class","UInt32","","","","LZ4",""
+"sk_mem_info_rmem_alloc","UInt32","","","","LZ4",""
+"sk_mem_info_rcv_buf","UInt32","","","","LZ4",""
+"sk_mem_info_wmem_alloc","UInt32","","","","LZ4",""
+"sk_mem_info_snd_buf","UInt32","","","","LZ4",""
+"sk_mem_info_fwd_alloc","UInt32","","","","LZ4",""
+"sk_mem_info_wmem_queued","UInt32","","","","LZ4",""
+"sk_mem_info_optmem","UInt32","","","","LZ4",""
+"sk_mem_info_backlog","UInt32","","","","LZ4",""
+"sk_mem_info_drops","UInt32","","","","LZ4",""
+"shutdown_state","UInt32","","","","LZ4",""
+"vegas_info_enabled","UInt32","","","","LZ4",""
+"vegas_info_rtt_cnt","UInt32","","","","LZ4",""
+"vegas_info_rtt","UInt32","","","","LZ4",""
+"vegas_info_min_rtt","UInt32","","","","LZ4",""
+"dctcp_info_enabled","UInt32","","","","LZ4",""
+"dctcp_info_ce_state","UInt32","","","","LZ4",""
+"dctcp_info_alpha","UInt32","","","","LZ4",""
+"dctcp_info_ab_ecn","UInt32","","","","LZ4",""
+"dctcp_info_ab_tot","UInt32","","","","LZ4",""
+"bbr_info_bw_lo","UInt32","","","","LZ4",""
+"bbr_info_bw_hi","UInt32","","","","LZ4",""
+"bbr_info_min_rtt","UInt32","","","","LZ4",""
+"bbr_info_pacing_gain","UInt32","","","","LZ4",""
+"bbr_info_cwnd_gain","UInt32","","","","LZ4",""
+"class_id","UInt32","","","","LZ4",""
+"sock_opt","UInt32","","","","LZ4",""
+"c_group","UInt64","","","","LZ4",""
diff --git a/build/containers/clickhouse/initdb.d/xtcp.xtcp_records b/build/containers/clickhouse/initdb.d/xtcp.xtcp_records
new file mode 100644
index 0000000..c14487c
--- /dev/null
+++ b/build/containers/clickhouse/initdb.d/xtcp.xtcp_records
@@ -0,0 +1,119 @@
+sec DateTime64(3, \'UTC\') DoubleDelta, LZ4
+nsec Int64
+hostname LowCardinality(String)
+tag LowCardinality(String)
+record_count UInt64 DoubleDelta, LZ4
+netlinker_id UInt32 LZ4
+inet_diag_msg_family UInt32 LZ4
+inet_diag_msg_state UInt32 LZ4
+inet_diag_msg_timer UInt32 LZ4
+inet_diag_msg_retrans UInt32 LZ4
+inet_diag_msg_socket_source_port UInt32 LZ4
+inet_diag_msg_socket_destination_port UInt32 LZ4
+inet_diag_msg_socket_source String ZSTD(1)
+inet_diag_msg_socket_source_ipv4 Nullable(IPv4)
+inet_diag_msg_socket_source_ipv6 Nullable(IPv6)
+inet_diag_msg_socket_destination String ZSTD(1)
+inet_diag_msg_socket_destination_ipv4 Nullable(IPv4)
+inet_diag_msg_socket_destination_ipv6 Nullable(IPv6)
+inet_diag_msg_socket_interface UInt32 LZ4
+inet_diag_msg_socket_cookie UInt64 LZ4
+inet_diag_msg_socket_dest_asn UInt64 LZ4
+inet_diag_msg_socket_next_hop_asn UInt64 LZ4
+inet_diag_msg_socket_source_asn UInt64 LZ4
+inet_diag_msg_expires UInt32 LZ4
+inet_diag_msg_rqueue UInt32 LZ4
+inet_diag_msg_wqueue UInt32 LZ4
+inet_diag_msg_uid UInt32 LZ4
+inet_diag_msg_inode UInt32 LZ4
+mem_info_rmem UInt32 LZ4
+mem_info_wmem UInt32 LZ4
+mem_info_fmem UInt32 LZ4
+mem_info_tmem UInt32 LZ4
+tcp_info_state UInt32 LZ4
+tcp_info_ca_state UInt32 LZ4
+tcp_info_retransmits UInt32 LZ4
+tcp_info_probes UInt32 LZ4
+tcp_info_backoff UInt32 LZ4
+tcp_info_options UInt32 LZ4
+tcp_info_send_scale UInt32 LZ4
+tcp_info_rcv_scale UInt32 LZ4
+tcp_info_delivery_rate_app_limited UInt32 LZ4
+tcp_info_fast_open_client_failed UInt32 LZ4
+tcp_info_rto UInt32 LZ4
+tcp_info_ato UInt32 LZ4
+tcp_info_snd_mss UInt32 LZ4
+tcp_info_rcv_mss UInt32 LZ4
+tcp_info_unacked UInt32 LZ4
+tcp_info_sacked UInt32 LZ4
+tcp_info_lost UInt32 LZ4
+tcp_info_retrans UInt32 LZ4
+tcp_info_fackets UInt32 LZ4
+tcp_info_last_data_sent UInt32 LZ4
+tcp_info_last_ack_sent UInt32 LZ4
+tcp_info_last_data_recv UInt32 LZ4
+tcp_info_last_ack_recv UInt32 LZ4
+tcp_info_pmtu UInt32 LZ4
+tcp_info_rcv_ssthresh UInt32 LZ4
+tcp_info_rtt UInt32 LZ4
+tcp_info_rtt_var UInt32 LZ4
+tcp_info_snd_ssthresh UInt32 LZ4
+tcp_info_snd_cwnd UInt32 LZ4
+tcp_info_adv_mss UInt32 LZ4
+tcp_info_reordering UInt32 LZ4
+tcp_info_rcv_rtt UInt32 LZ4
+tcp_info_rcv_space UInt32 LZ4
+tcp_info_total_retrans UInt32 LZ4
+tcp_info_pacing_rate UInt64 LZ4
+tcp_info_max_pacing_rate UInt64 LZ4
+tcp_info_bytes_acked UInt64 LZ4
+tcp_info_bytes_received UInt64 LZ4
+tcp_info_segs_out UInt32 LZ4
+tcp_info_segs_in UInt32 LZ4
+tcp_info_not_sent_bytes UInt32 LZ4
+tcp_info_min_rtt UInt32 LZ4
+tcp_info_data_segs_in UInt32 LZ4
+tcp_info_data_segs_out UInt32 LZ4
+tcp_info_delivery_rate UInt64 LZ4
+tcp_info_busy_time UInt64 LZ4
+tcp_info_rwnd_limited UInt64 LZ4
+tcp_info_sndbuf_limited UInt64 LZ4
+tcp_info_delivered UInt32 LZ4
+tcp_info_delivered_ce UInt32 LZ4
+tcp_info_bytes_sent UInt64 LZ4
+tcp_info_bytes_retrans UInt64 LZ4
+tcp_info_dsack_dups UInt32 LZ4
+tcp_info_reord_seen UInt32 LZ4
+tcp_info_rcv_ooopack UInt32 LZ4
+tcp_info_snd_wnd UInt32 LZ4
+congestion_algorithm_string LowCardinality(String)
+congestion_algorithm_enum LowCardinality(String)
+type_of_service UInt32 LZ4
+traffic_class UInt32 LZ4
+sk_mem_info_rmem_alloc UInt32 LZ4
+sk_mem_info_rcv_buf UInt32 LZ4
+sk_mem_info_wmem_alloc UInt32 LZ4
+sk_mem_info_snd_buf UInt32 LZ4
+sk_mem_info_fwd_alloc UInt32 LZ4
+sk_mem_info_wmem_queued UInt32 LZ4
+sk_mem_info_optmem UInt32 LZ4
+sk_mem_info_backlog UInt32 LZ4
+sk_mem_info_drops UInt32 LZ4
+shutdown_state UInt32 LZ4
+vegas_info_enabled UInt32 LZ4
+vegas_info_rtt_cnt UInt32 LZ4
+vegas_info_rtt UInt32 LZ4
+vegas_info_min_rtt UInt32 LZ4
+dctcp_info_enabled UInt32 LZ4
+dctcp_info_ce_state UInt32 LZ4
+dctcp_info_alpha UInt32 LZ4
+dctcp_info_ab_ecn UInt32 LZ4
+dctcp_info_ab_tot UInt32 LZ4
+bbr_info_bw_lo UInt32 LZ4
+bbr_info_bw_hi UInt32 LZ4
+bbr_info_min_rtt UInt32 LZ4
+bbr_info_pacing_gain UInt32 LZ4
+bbr_info_cwnd_gain UInt32 LZ4
+class_id UInt32 LZ4
+sock_opt UInt32 LZ4
+c_group UInt64 LZ4
diff --git a/build/containers/clickhouse/initdb.d/xtcp.xtcp_records.csv b/build/containers/clickhouse/initdb.d/xtcp.xtcp_records.csv
new file mode 100644
index 0000000..dc510ec
--- /dev/null
+++ b/build/containers/clickhouse/initdb.d/xtcp.xtcp_records.csv
@@ -0,0 +1,119 @@
+"sec","DateTime64(3, 'UTC')","","","","DoubleDelta, LZ4",""
+"nsec","Int64","","","","",""
+"hostname","LowCardinality(String)","","","","",""
+"tag","LowCardinality(String)","","","","",""
+"record_count","UInt64","","","","DoubleDelta, LZ4",""
+"netlinker_id","UInt32","","","","LZ4",""
+"inet_diag_msg_family","UInt32","","","","LZ4",""
+"inet_diag_msg_state","UInt32","","","","LZ4",""
+"inet_diag_msg_timer","UInt32","","","","LZ4",""
+"inet_diag_msg_retrans","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_source_port","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_destination_port","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_source","String","","","","ZSTD(1)",""
+"inet_diag_msg_socket_source_ipv4","Nullable(IPv4)","","","","",""
+"inet_diag_msg_socket_source_ipv6","Nullable(IPv6)","","","","",""
+"inet_diag_msg_socket_destination","String","","","","ZSTD(1)",""
+"inet_diag_msg_socket_destination_ipv4","Nullable(IPv4)","","","","",""
+"inet_diag_msg_socket_destination_ipv6","Nullable(IPv6)","","","","",""
+"inet_diag_msg_socket_interface","UInt32","","","","LZ4",""
+"inet_diag_msg_socket_cookie","UInt64","","","","LZ4",""
+"inet_diag_msg_socket_dest_asn","UInt64","","","","LZ4",""
+"inet_diag_msg_socket_next_hop_asn","UInt64","","","","LZ4",""
+"inet_diag_msg_socket_source_asn","UInt64","","","","LZ4",""
+"inet_diag_msg_expires","UInt32","","","","LZ4",""
+"inet_diag_msg_rqueue","UInt32","","","","LZ4",""
+"inet_diag_msg_wqueue","UInt32","","","","LZ4",""
+"inet_diag_msg_uid","UInt32","","","","LZ4",""
+"inet_diag_msg_inode","UInt32","","","","LZ4",""
+"mem_info_rmem","UInt32","","","","LZ4",""
+"mem_info_wmem","UInt32","","","","LZ4",""
+"mem_info_fmem","UInt32","","","","LZ4",""
+"mem_info_tmem","UInt32","","","","LZ4",""
+"tcp_info_state","UInt32","","","","LZ4",""
+"tcp_info_ca_state","UInt32","","","","LZ4",""
+"tcp_info_retransmits","UInt32","","","","LZ4",""
+"tcp_info_probes","UInt32","","","","LZ4",""
+"tcp_info_backoff","UInt32","","","","LZ4",""
+"tcp_info_options","UInt32","","","","LZ4",""
+"tcp_info_send_scale","UInt32","","","","LZ4",""
+"tcp_info_rcv_scale","UInt32","","","","LZ4",""
+"tcp_info_delivery_rate_app_limited","UInt32","","","","LZ4",""
+"tcp_info_fast_open_client_failed","UInt32","","","","LZ4",""
+"tcp_info_rto","UInt32","","","","LZ4",""
+"tcp_info_ato","UInt32","","","","LZ4",""
+"tcp_info_snd_mss","UInt32","","","","LZ4",""
+"tcp_info_rcv_mss","UInt32","","","","LZ4",""
+"tcp_info_unacked","UInt32","","","","LZ4",""
+"tcp_info_sacked","UInt32","","","","LZ4",""
+"tcp_info_lost","UInt32","","","","LZ4",""
+"tcp_info_retrans","UInt32","","","","LZ4",""
+"tcp_info_fackets","UInt32","","","","LZ4",""
+"tcp_info_last_data_sent","UInt32","","","","LZ4",""
+"tcp_info_last_ack_sent","UInt32","","","","LZ4",""
+"tcp_info_last_data_recv","UInt32","","","","LZ4",""
+"tcp_info_last_ack_recv","UInt32","","","","LZ4",""
+"tcp_info_pmtu","UInt32","","","","LZ4",""
+"tcp_info_rcv_ssthresh","UInt32","","","","LZ4",""
+"tcp_info_rtt","UInt32","","","","LZ4",""
+"tcp_info_rtt_var","UInt32","","","","LZ4",""
+"tcp_info_snd_ssthresh","UInt32","","","","LZ4",""
+"tcp_info_snd_cwnd","UInt32","","","","LZ4",""
+"tcp_info_adv_mss","UInt32","","","","LZ4",""
+"tcp_info_reordering","UInt32","","","","LZ4",""
+"tcp_info_rcv_rtt","UInt32","","","","LZ4",""
+"tcp_info_rcv_space","UInt32","","","","LZ4",""
+"tcp_info_total_retrans","UInt32","","","","LZ4",""
+"tcp_info_pacing_rate","UInt64","","","","LZ4",""
+"tcp_info_max_pacing_rate","UInt64","","","","LZ4",""
+"tcp_info_bytes_acked","UInt64","","","","LZ4",""
+"tcp_info_bytes_received","UInt64","","","","LZ4",""
+"tcp_info_segs_out","UInt32","","","","LZ4",""
+"tcp_info_segs_in","UInt32","","","","LZ4",""
+"tcp_info_not_sent_bytes","UInt32","","","","LZ4",""
+"tcp_info_min_rtt","UInt32","","","","LZ4",""
+"tcp_info_data_segs_in","UInt32","","","","LZ4",""
+"tcp_info_data_segs_out","UInt32","","","","LZ4",""
+"tcp_info_delivery_rate","UInt64","","","","LZ4",""
+"tcp_info_busy_time","UInt64","","","","LZ4",""
+"tcp_info_rwnd_limited","UInt64","","","","LZ4",""
+"tcp_info_sndbuf_limited","UInt64","","","","LZ4",""
+"tcp_info_delivered","UInt32","","","","LZ4",""
+"tcp_info_delivered_ce","UInt32","","","","LZ4",""
+"tcp_info_bytes_sent","UInt64","","","","LZ4",""
+"tcp_info_bytes_retrans","UInt64","","","","LZ4",""
+"tcp_info_dsack_dups","UInt32","","","","LZ4",""
+"tcp_info_reord_seen","UInt32","","","","LZ4",""
+"tcp_info_rcv_ooopack","UInt32","","","","LZ4",""
+"tcp_info_snd_wnd","UInt32","","","","LZ4",""
+"congestion_algorithm_string","LowCardinality(String)","","","","",""
+"congestion_algorithm_enum","LowCardinality(String)","","","","",""
+"type_of_service","UInt32","","","","LZ4",""
+"traffic_class","UInt32","","","","LZ4",""
+"sk_mem_info_rmem_alloc","UInt32","","","","LZ4",""
+"sk_mem_info_rcv_buf","UInt32","","","","LZ4",""
+"sk_mem_info_wmem_alloc","UInt32","","","","LZ4",""
+"sk_mem_info_snd_buf","UInt32","","","","LZ4",""
+"sk_mem_info_fwd_alloc","UInt32","","","","LZ4",""
+"sk_mem_info_wmem_queued","UInt32","","","","LZ4",""
+"sk_mem_info_optmem","UInt32","","","","LZ4",""
+"sk_mem_info_backlog","UInt32","","","","LZ4",""
+"sk_mem_info_drops","UInt32","","","","LZ4",""
+"shutdown_state","UInt32","","","","LZ4",""
+"vegas_info_enabled","UInt32","","","","LZ4",""
+"vegas_info_rtt_cnt","UInt32","","","","LZ4",""
+"vegas_info_rtt","UInt32","","","","LZ4",""
+"vegas_info_min_rtt","UInt32","","","","LZ4",""
+"dctcp_info_enabled","UInt32","","","","LZ4",""
+"dctcp_info_ce_state","UInt32","","","","LZ4",""
+"dctcp_info_alpha","UInt32","","","","LZ4",""
+"dctcp_info_ab_ecn","UInt32","","","","LZ4",""
+"dctcp_info_ab_tot","UInt32","","","","LZ4",""
+"bbr_info_bw_lo","UInt32","","","","LZ4",""
+"bbr_info_bw_hi","UInt32","","","","LZ4",""
+"bbr_info_min_rtt","UInt32","","","","LZ4",""
+"bbr_info_pacing_gain","UInt32","","","","LZ4",""
+"bbr_info_cwnd_gain","UInt32","","","","LZ4",""
+"class_id","UInt32","","","","LZ4",""
+"sock_opt","UInt32","","","","LZ4",""
+"c_group","UInt64","","","","LZ4",""
diff --git a/build/containers/clickhouse/sql_queries.sql b/build/containers/clickhouse/sql_queries.sql
new file mode 100644
index 0000000..bd8c93b
--- /dev/null
+++ b/build/containers/clickhouse/sql_queries.sql
@@ -0,0 +1,16 @@
+SELECT
+ sec,
+ nsec,
+ hostname,
+ tcp_info_rtt,
+ tcp_info_rtt_var,
+ tcp_info_min_rtt,
+ tcp_info_rcv_rtt,
+FROM xtcp.xtcp_records
+ORDER BY
+ sec DESC,
+ nsec DESC
+LIMIT 20;
+
+
+SELECT COUNT(*) FROM xtcp.xtcp_records;
diff --git a/build/containers/redpanda/docker-compose.yml b/build/containers/redpanda/docker-compose.yml
index d9bb6dc..5a8dd29 100644
--- a/build/containers/redpanda/docker-compose.yml
+++ b/build/containers/redpanda/docker-compose.yml
@@ -18,6 +18,7 @@ networks:
volumes:
redpanda-0: {}
+ clickhouse_db: {}
nsq_data: {}
valkey_data: {}
@@ -89,6 +90,7 @@ services:
enabled: true
urls: ["http://redpanda-0:9644"]
ports:
+ # Browse to: http://localhost:8080/topics/xtcp?p=-1&s=50&o=-2#messages
- 8080:8080
depends_on:
- redpanda-0
@@ -128,12 +130,19 @@ services:
- 8123:8123
# native
- 9001:9000
+ # mysql
+ - 9004:9004
+ # postgres
+ - 9005:9005
+ # prometheus
+ - 9363:9363
volumes:
# ch_server-1 | Processing configuration file '/etc/clickhouse-server/config.xml'.
# ch_server-1 | Merging configuration file '/etc/clickhouse-server/config.d/docker_related_config.xml'.
# https://clickhouse.com/docs/en/operations/configuration-files
- - ${XTCPPATH}/build/containers/clickhouse/db/:/var/lib/clickhouse/:z
- - ${XTCPPATH}/build/containers/clickhouse/bitnami_db/:/bitnami/clickhouse/:z
+ - clickhouse_db:/var/lib/clickhouse/
+ # - ${XTCPPATH}/build/containers/clickhouse/db/:/var/lib/clickhouse/:z
+ # - ${XTCPPATH}/build/containers/clickhouse/bitnami_db/:/bitnami/clickhouse/:z
#- ch_data:/var/lib/clickhouse/
# - type: bind
# source: ${PWD}/db/
@@ -279,7 +288,7 @@ services:
# valkey
# https://hub.docker.com/r/valkey/valkey/tags
valkey:
- image: valkey/valkey:8.0.0-rc2-alpine3.20
+ image: valkey/valkey:8-alpine3.20
volumes:
- /valkey_data:/data
ports:
diff --git a/build/containers/xtcp2/Containerfile b/build/containers/xtcp2/Containerfile
index c249707..a7bea36 100644
--- a/build/containers/xtcp2/Containerfile
+++ b/build/containers/xtcp2/Containerfile
@@ -2,7 +2,6 @@
# ./build/containers/xtcp2/Containerfile
#
ARG XTCPPATH
-ARG PWD
ARG COMMIT
ARG DATE
ARG GO_VERSION=1.23
@@ -12,12 +11,11 @@ ARG VERSION
FROM golang:${GO_VERSION} as build
ARG XTCPPATH
-ARG PWD
ARG COMMIT
ARG DATE
ARG VERSION
-RUN echo XTCPPATH:${XTCPPATH} PWD:${PWD} COMMIT:${COMMIT} DATE:${DATE} VERSION:${VERSION}
+RUN echo XTCPPATH:${XTCPPATH} COMMIT:${COMMIT} DATE:${DATE} VERSION:${VERSION}
WORKDIR /go/src
COPY . .
@@ -30,5 +28,6 @@ RUN CGO_ENABLED=0 go build \
# https://github.com/GoogleContainerTools/distroless?tab=readme-ov-file#what-images-are-available
FROM gcr.io/distroless/static-debian12:nonroot
COPY --from=build /go/bin/xtcp2 /
+# Prometheus
EXPOSE 9009
ENTRYPOINT ["/xtcp2"]
\ No newline at end of file
diff --git a/build/containers/xtcp2/Makefile b/build/containers/xtcp2/Makefile
index 36bbcab..a239a0a 100644
--- a/build/containers/xtcp2/Makefile
+++ b/build/containers/xtcp2/Makefile
@@ -14,6 +14,6 @@ buildd:
--build-arg PWD=${PWD} \
--build-arg COMMIT=${COMMIT} \
--build-arg DATE=${DATE} \
- --build-arg DATE=${VERSION} \
+ --build-arg VERSION=${VERSION} \
--tag xtcp2 \
${XTCPPATH}
\ No newline at end of file